From b40836e60f7bc27feeb8c4b7b5c05e7914d0fbad Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 10 Nov 2022 16:53:59 +0100 Subject: [PATCH 1/9] Leverage MQ file attribute in language scripts (#245) * Leverage MQ file attribute in build scripts Signed-off-by: Dennis Behm --- build-conf/README.md | 2 + build-conf/datasets.properties | 9 +++ languages/Assembler.groovy | 62 +++++++++++----- languages/Cobol.groovy | 74 +++++++++---------- languages/LinkEdit.groovy | 13 +++- languages/PLI.groovy | 68 ++++++++++------- .../application-conf/Cobol.properties | 12 ++- samples/application-conf/Assembler.properties | 4 + samples/application-conf/Cobol.properties | 12 ++- samples/application-conf/PLI.properties | 9 +++ samples/application-conf/README.md | 6 +- utilities/BuildUtilities.groovy | 38 ++++++++++ 12 files changed, 211 insertions(+), 98 deletions(-) diff --git a/build-conf/README.md b/build-conf/README.md index 96ce5b6b..e48c97e0 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -23,7 +23,9 @@ IBMZPLI_V51 | PLI Compiler Data Set for version 5.1. Example: PLI.V5R1M0.SIBMZCM SDFHMAC | CICS Macro Library. Example: CICSTS.V3R2M0.CICS.SDFHMAC SDFHLOAD | CICS Load Library. Example: CICSTS.V3R2M0.CICS.SDFHLOAD SDFHCOB | CICS COBOL Library. Example: CICSTS.V3R2M0.CICS.SDFHCOB +SDFHPL1 | CICS PL1 Library. Example: CICSTS.V3R2M0.CICS.SDFHPL1 SCSQCOBC | MQ COBOL Library. Example: CSQ.V9R1M0.SCSQCOBC +SCSQPLIC | MQ PLI Library. Example: CSQ.V9R1M0.SCSQPLIC SCSQLOAD | MQ Load Library. Example: CSQ.V9R1M0.SCSQLOAD SDSNLOAD | DB2 Load Library. Example: DB2.V9R1M0.SDSNLOAD SFELLOAD | Optional IDz Load Library. Example: FEL.V14R0M0.SFELLOAD diff --git a/build-conf/datasets.properties b/build-conf/datasets.properties index 835e6897..48f5f582 100644 --- a/build-conf/datasets.properties +++ b/build-conf/datasets.properties @@ -33,9 +33,18 @@ SDFHLOAD= # CICS COBOL Library. Example: CICSTS.V3R2M0.CICS.SDFHCOB SDFHCOB= +# CICS PLI Library. Example: CICSTS.V3R2M0.CICS.SDFHPL1 +SDFHPL1= + # MQ COBOL Library. Example: CSQ.V9R1M0.SCSQCOBC SCSQCOBC= +# MQ PLI Library. Example: CSQ.V9R1M0.SCSQPLIC +SCSQPLIC= + +# MQ Assembler Library. Example: CSQ.V9R1M0.SCSQMACS +SCSQMACS= + # MQ Load Library. Example: CSQ.V9R1M0.SCSQLOAD SCSQLOAD= diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index b47bb896..fa8fbb9f 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -250,25 +250,20 @@ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String mem // add DD statements to the compile command String assembler_srcPDS = props.getFileProperty('assembler_srcPDS', buildFile) - // Pass different allocations - // Case: BATCH - allocation SYSIN - if (!buildUtils.isCICS(logicalFile) && !buildUtils.isSQL(logicalFile)) assembler.dd(new DDStatement().name("SYSIN").dsn("${assembler_srcPDS}($member)").options('shr')) - // else assembler.dd(new DDStatement().name("SYSCIN").ddref("SYSIN")) - - // Case: CICS - translator overwrite Ddnames - if (buildUtils.isCICS(logicalFile)) assembler.setDdnames("SYSLIN,,,SYSLIB,SYSPUNCH,,,,,,,,,,,,,,") - else if (buildUtils.isSQL(logicalFile)) assembler.setDdnames("SYSLIN,,,SYSLIB,SYSCIN,,,,,,,,,,,,,,") + // Pass different input allocations + if (buildUtils.isCICS(logicalFile)) { // Case: CICS - translator overwrite Ddnames + assembler.setDdnames("SYSLIN,,,SYSLIB,SYSPUNCH,,,,,,,,,,,,,,") + } else if (buildUtils.isSQL(logicalFile)) { // Case: Db2 - translator overwrite Ddnames + assembler.setDdnames("SYSLIN,,,SYSLIB,SYSCIN,,,,,,,,,,,,,,") + } else { // Case: Plain batch + assembler.dd(new DDStatement().name("SYSIN").dsn("${assembler_srcPDS}($member)").options('shr')) + } assembler.dd(new DDStatement().name("SYSPRINT").options(props.assembler_tempOptions)) assembler.dd(new DDStatement().name("SYSUT1").options(props.assembler_tempOptions)) - - // Write SYSLIN to temporary dataset if performing link edit - String doLinkEdit = props.getFileProperty('assembler_linkEdit', buildFile) - if (doLinkEdit && doLinkEdit.toBoolean()) - assembler.dd(new DDStatement().name("SYSLIN").dsn("&&TEMPOBJ").options(props.assembler_tempOptions).pass(true)) - else - assembler.dd(new DDStatement().name("SYSLIN").dsn("${props.assembler_objPDS}($member)").options('shr').output(true)) + // define object dataset allocation + assembler.dd(new DDStatement().name("SYSLIN").dsn("${props.assembler_objPDS}($member)").options('shr').output(true)) // create a SYSLIB concatenation with optional MACLIB and MODGEN assembler.dd(new DDStatement().name("SYSLIB").dsn(props.assembler_macroPDS).options("shr")) @@ -297,6 +292,8 @@ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String mem if (buildUtils.isCICS(logicalFile)) assembler.dd(new DDStatement().dsn(props.SDFHMAC).options("shr")) //if (buildUtils.isSQL(logicalFile)) + if (buildUtils.isMQ(logicalFile)) + assembler.dd(new DDStatement().dsn(props.SCSQMACS).options("shr")) if (props.SDFSMAC) assembler.dd(new DDStatement().dsn(props.SDFSMAC).options("shr")) @@ -318,7 +315,9 @@ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String mem */ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String member, File logFile) { String parameters = props.getFileProperty('assembler_linkEditParms', buildFile) - + String linkEditStream = props.getFileProperty('assembler_linkEditStream', buildFile) + + // obtain githash for buildfile String assembler_storeSSI = props.getFileProperty('assembler_storeSSI', buildFile) if (assembler_storeSSI && assembler_storeSSI.toBoolean() && (props.mergeBuild || props.impactBuild || props.fullBuild)) { @@ -328,14 +327,40 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb // define the MVSExec command to link edit the program MVSExec linkedit = new MVSExec().file(buildFile).pgm(props.assembler_linkEditor).parm(parameters) - + // add DD statements to the linkedit command String assembler_loadPDS = props.getFileProperty('assembler_loadPDS', buildFile) String deployType = buildUtils.getDeployType("assembler", buildFile, logicalFile) linkedit.dd(new DDStatement().name("SYSLMOD").dsn("${assembler_loadPDS}($member)").options('shr').output(true).deployType(deployType)) + linkedit.dd(new DDStatement().name("SYSPRINT").options(props.assembler_tempOptions)) linkedit.dd(new DDStatement().name("SYSUT1").options(props.assembler_tempOptions)) + // Create linkEditInstream + String sysin_linkEditInstream = '' + // linkEdit stream specified + if (linkEditStream) { + sysin_linkEditInstream += " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) + } + + // appending mq stub according to file flags + if(buildUtils.isMQ(logicalFile)) { + // include mq stub program + // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs + sysin_linkEditInstream += buildUtils.getMqStubInstruction(logicalFile) + } + + // Define SYSIN dd as instream data + if (sysin_linkEditInstream) { + if (props.verbose) println("** Generated linkcard input stream: \n $sysin_linkEditInstream") + linkedit.dd(new DDStatement().name("SYSIN").instreamData(sysin_linkEditInstream)) + } + + // add SYSLIN along the reference to SYSIN if configured through sysin_linkEditInstream + linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.assembler_objPDS}($member)").options('shr')) + if (sysin_linkEditInstream) linkedit.dd(new DDStatement().ddref("SYSIN")) + + // add a syslib to the linkedit command linkedit.dd(new DDStatement().name("SYSLIB").dsn(props.assembler_objPDS).options("shr")) // add custom concatenation @@ -353,6 +378,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) + if (buildUtils.isMQ(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SCSQLOAD).options("shr")) + // add a copy command to the linkedit command to append the SYSPRINT from the temporary dataset to the HFS log file linkedit.copy(new CopyToHFS().ddName("SYSPRINT").file(logFile).hfsEncoding(props.logEncoding).append(true)) return linkedit diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 09104726..d502d176 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -195,18 +195,8 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().name("SYSUT$num").options(props.cobol_tempOptions)) } - // Write SYSLIN to temporary dataset if performing link edit or to physical dataset - String doLinkEdit = props.getFileProperty('cobol_linkEdit', buildFile) - String linkEditStream = props.getFileProperty('cobol_linkEditStream', buildFile) - String linkDebugExit = props.getFileProperty('cobol_linkDebugExit', buildFile) - - if (props.debug && linkDebugExit && doLinkEdit.toBoolean()){ - compile.dd(new DDStatement().name("SYSLIN").dsn("${props.cobol_objPDS}($member)").options('shr').output(true)) - } else if (doLinkEdit && doLinkEdit.toBoolean() && ( !linkEditStream || linkEditStream.isEmpty())) { - compile.dd(new DDStatement().name("SYSLIN").dsn("&&TEMPOBJ").options(props.cobol_tempOptions).pass(true)) - } else { - compile.dd(new DDStatement().name("SYSLIN").dsn("${props.cobol_objPDS}($member)").options('shr').output(true)) - } + // define object dataset allocation + compile.dd(new DDStatement().name("SYSLIN").dsn("${props.cobol_objPDS}($member)").options('shr').output(true)) // add a syslib to the compile command with optional bms output copybook and CICS concatenation compile.dd(new DDStatement().name("SYSLIB").dsn(props.cobol_cpyPDS).options("shr")) @@ -231,10 +221,12 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe for (String syslibDataset : syslibDatasets ) compile.dd(new DDStatement().dsn(syslibDataset).options("shr")) } + + // add subsystem libraries if (buildUtils.isCICS(logicalFile)) compile.dd(new DDStatement().dsn(props.SDFHCOB).options("shr")) - String isMQ = props.getFileProperty('cobol_isMQ', buildFile) - if (isMQ && isMQ.toBoolean()) + + if (buildUtils.isMQ(logicalFile)) compile.dd(new DDStatement().dsn(props.SCSQCOBC).options("shr")) // add additional zunit libraries @@ -306,31 +298,36 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb // define the MVSExec command to link edit the program MVSExec linkedit = new MVSExec().file(buildFile).pgm(linker).parm(parms) - // Create a physical link card - if ( (linkEditStream) || (props.debug && linkDebugExit!= null)) { - def langQualifier = "linkedit" - buildUtils.createLanguageDatasets(langQualifier) - def lnkFile = new File("${props.buildOutDir}/linkCard.lnk") - if (lnkFile.exists()) - lnkFile.delete() - - if (linkEditStream) - lnkFile << " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) - else - lnkFile << " " + linkDebugExit.replace("\\n","\n").replace('@{member}',member) - - if (props.verbose) - println("Copying ${props.buildOutDir}/linkCard.lnk to ${props.linkedit_srcPDS}($member)") - new CopyToPDS().file(lnkFile).dataset(props.linkedit_srcPDS).member(member).execute() - // Alloc SYSLIN - linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.linkedit_srcPDS}($member)").options("shr")) - // add the obj DD - linkedit.dd(new DDStatement().name("OBJECT").dsn("${props.cobol_objPDS}($member)").options('shr')) - - } else { // no debug && no link card - // Use &&TEMP from Compile + // Assemble linkEditInstream to define SYSIN as instreamData + String sysin_linkEditInstream = '' + + // appending configured linkEdit stream if specified + if (linkEditStream) { + sysin_linkEditInstream += " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) + } + + // appending mq stub according to file flags + if(buildUtils.isMQ(logicalFile)) { + // include mq stub program + // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs + sysin_linkEditInstream += buildUtils.getMqStubInstruction(logicalFile) + } + + // appending debug exit to link instructions + if (props.debug && linkDebugExit!= null) { + sysin_linkEditInstream += " " + linkDebugExit.replace("\\n","\n").replace('@{member}',member) + } + + // Define SYSIN dd as instream data + if (sysin_linkEditInstream) { + if (props.verbose) println("** Generated linkcard input stream: \n $sysin_linkEditInstream") + linkedit.dd(new DDStatement().name("SYSIN").instreamData(sysin_linkEditInstream)) } + // add SYSLIN along the reference to SYSIN if configured through sysin_linkEditInstream + linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.cobol_objPDS}($member)").options('shr')) + if (sysin_linkEditInstream) linkedit.dd(new DDStatement().ddref("SYSIN")) + // add DD statements to the linkedit command String deployType = buildUtils.getDeployType("cobol", buildFile, logicalFile) if(isZUnitTestCase){ @@ -369,8 +366,7 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) - String isMQ = props.getFileProperty('cobol_isMQ', buildFile) - if (isMQ && isMQ.toBoolean()) + if (buildUtils.isMQ(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SCSQLOAD).options("shr")) // add a copy command to the linkedit command to append the SYSPRINT from the temporary dataset to the HFS log file diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 5d048f14..990b9c3b 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -105,8 +105,19 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb linkedit.dd(new DDStatement().dsn(syslibDataset).options("shr")) } linkedit.dd(new DDStatement().dsn(props.SCEELKED).options("shr")) - linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (props.debug && props.SEQAMOD) + linkedit.dd(new DDStatement().dsn(props.SEQAMOD).options("shr")) + + if (props.SDFHLOAD) + linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + + if (props.SDSNLOAD) + linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) + + if (props.SCSQLOAD) + linkedit.dd(new DDStatement().dsn(props.SCSQLOAD).options("shr")) + // add a copy command to the linkedit command to append the SYSPRINT from the temporary dataset to the HFS log file linkedit.copy(new CopyToHFS().ddName("SYSPRINT").file(logFile).hfsEncoding(props.logEncoding)) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 720842a8..d77bc45c 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -173,13 +173,8 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().name("SYSUT$num").options(props.pli_tempOptions)) } - // Write SYSLIN to temporary dataset if performing link edit - String doLinkEdit = props.getFileProperty('pli_linkEdit', buildFile) - String linkEditStream = props.getFileProperty('pli_linkEditStream', buildFile) - if (linkEditStream == null && doLinkEdit && doLinkEdit.toBoolean()) - compile.dd(new DDStatement().name("SYSLIN").dsn("&&TEMPOBJ").options(props.pli_tempOptions).pass(true)) - else - compile.dd(new DDStatement().name("SYSLIN").dsn("${props.pli_objPDS}($member)").options('shr').output(true)) + // define object dataset allocation + compile.dd(new DDStatement().name("SYSLIN").dsn("${props.pli_objPDS}($member)").options('shr').output(true)) // add a syslib to the compile command with optional bms output copybook and CICS concatenation compile.dd(new DDStatement().name("SYSLIB").dsn(props.pli_incPDS).options("shr")) @@ -205,9 +200,13 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().dsn(syslibDataset).options("shr")) } + // add subsystem libraries if (buildUtils.isCICS(logicalFile)) - compile.dd(new DDStatement().dsn(props.SDFHCOB).options("shr")) + compile.dd(new DDStatement().dsn(props.SDFHPL1).options("shr")) + if (buildUtils.isMQ(logicalFile)) + compile.dd(new DDStatement().dsn(props.SCSQPLIC).options("shr")) + // add additional zunit libraries if (isZUnitTestCase) compile.dd(new DDStatement().dsn(props.SBZUSAMP).options("shr")) @@ -264,6 +263,8 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb String parms = props.getFileProperty('pli_linkEditParms', buildFile) String linker = props.getFileProperty('pli_linkEditor', buildFile) String linkEditStream = props.getFileProperty('pli_linkEditStream', buildFile) + String linkDebugExit = props.getFileProperty('pli_linkDebugExit', buildFile) + // obtain githash for buildfile String pli_storeSSI = props.getFileProperty('pli_storeSSI', buildFile) @@ -272,21 +273,7 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (ssi != null) parms = parms + ",SSI=$ssi" } - // Create the link stream if needed - if ( linkEditStream != null ) { - def langQualifier = "linkedit" - buildUtils.createLanguageDatasets(langQualifier) - def lnkFile = new File("${props.buildOutDir}/linkCard.lnk") - if (lnkFile.exists()) - lnkFile.delete() - - lnkFile << " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) - if (props.verbose) - println("Copying ${props.buildOutDir}/linkCard.lnk to ${props.linkedit_srcPDS}($member)") - new CopyToPDS().file(lnkFile).dataset(props.linkedit_srcPDS).member(member).execute() - - } - + // define the MVSExec command to link edit the program MVSExec linkedit = new MVSExec().file(buildFile).pgm(linker).parm(parms) // add DD statements to the linkedit command @@ -300,11 +287,36 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb linkedit.dd(new DDStatement().name("SYSPRINT").options(props.pli_tempOptions)) linkedit.dd(new DDStatement().name("SYSUT1").options(props.pli_tempOptions)) - // add the link source code - if ( linkEditStream != null ) { - linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.linkedit_srcPDS}($member)").options("shr")) + // Assemble linkEditInstream to define SYSIN as instreamData + String sysin_linkEditInstream = '' + + // appending configured linkEdit stream if specified + if (linkEditStream) { + sysin_linkEditInstream += " " + linkEditStream.replace("\\n","\n").replace('@{member}',member) + } + + // appending mq stub according to file flags + if(buildUtils.isMQ(logicalFile)) { + // include mq stub program + // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs + sysin_linkEditInstream += buildUtils.getMqStubInstruction(logicalFile) } + // appending debug exit to link instructions + if (props.debug && linkDebugExit!= null) { + sysin_linkEditInstream += " " + linkDebugExit.replace("\\n","\n").replace('@{member}',member) + } + + // Define SYSIN dd + if (sysin_linkEditInstream) { + if (props.verbose) println("** Generated linkcard input stream: \n $sysin_linkEditInstream") + linkedit.dd(new DDStatement().name("SYSIN").instreamData(sysin_linkEditInstream)) + } + + // add SYSLIN along the reference to SYSIN if configured through sysin_linkEditInstream + linkedit.dd(new DDStatement().name("SYSLIN").dsn("${props.pli_objPDS}($member)").options('shr')) + if (sysin_linkEditInstream) linkedit.dd(new DDStatement().ddref("SYSIN")) + // add RESLIB if ( props.RESLIB ) linkedit.dd(new DDStatement().name("RESLIB").dsn(props.RESLIB).options("shr")) @@ -319,12 +331,16 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb linkedit.dd(new DDStatement().dsn(syslibDataset).options("shr")) } linkedit.dd(new DDStatement().dsn(props.SCEELKED).options("shr")) + if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) + if (buildUtils.isMQ(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SCSQLOAD).options("shr")) + // add dummy SYSDEFSD to avoid IEW2689W 4C40 DEFINITION SIDE FILE IS NOT DEFINED message from program binder if (isZUnitTestCase) linkedit.dd(new DDStatement().name("SYSDEFSD").options("DUMMY")) diff --git a/samples/MortgageApplication/application-conf/Cobol.properties b/samples/MortgageApplication/application-conf/Cobol.properties index d918b3cb..199d4a00 100644 --- a/samples/MortgageApplication/application-conf/Cobol.properties +++ b/samples/MortgageApplication/application-conf/Cobol.properties @@ -49,15 +49,13 @@ cobol_compileDebugParms=TEST # can be overridden by file properties cobol_linkEditParms=MAP,RENT,COMPAT(PM5) -# If you would like to have a physical link card, we generated it for you given the below pattern -# This property has priority over cobol_linkDebugExit -# cobol_linkEditStream= INCLUDE OBJECT(@{member}) +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n cobol_linkEditStream= -# If using a debug exit, provide the SYSLIN instream DD -# Samp: cobol_linkDebugExit= INCLUDE OBJECT(@{member}) \n INCLUDE SYSLIB(EQAD3CXT) -cobol_linkDebugExit= INCLUDE OBJECT(@{member}) \n INCLUDE SYSLIB(EQAD3CXT) - +# If using a debug exit for IBM Debug tool, provide the SYSIN instream DD which is appended to SYSIN +# Samp: cobol_linkDebugExit= INCLUDE SYSLIB(EQAD3CXT) \n +cobol_linkDebugExit= # # execute link edit step diff --git a/samples/application-conf/Assembler.properties b/samples/application-conf/Assembler.properties index 921d7826..b12d1991 100644 --- a/samples/application-conf/Assembler.properties +++ b/samples/application-conf/Assembler.properties @@ -14,6 +14,10 @@ assembler_compileErrorPrefixParms=ADATA,EX(ADX(ELAXHASM)) assembler_db2precompilerParms=HOST(ASM) assembler_cicsprecompilerParms= +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n +assembler_linkEditStream= + # # execute link edit step # can be overridden by file properties diff --git a/samples/application-conf/Cobol.properties b/samples/application-conf/Cobol.properties index 6ae40306..8a04555e 100644 --- a/samples/application-conf/Cobol.properties +++ b/samples/application-conf/Cobol.properties @@ -50,15 +50,13 @@ cobol_compileDebugParms=TEST # can be overridden by file properties cobol_linkEditParms=MAP,RENT,COMPAT(PM5) -# If you would like to have a physical link card, we generated it for you given the below pattern -# This property has priority over cobol_linkDebugExit -# cobol_linkEditStream= INCLUDE OBJECT(@{member}) +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n cobol_linkEditStream= -# If using a debug exit, provide the SYSLIN instream DD -# Samp: cobol_linkDebugExit= INCLUDE OBJECT(@{member}) \n INCLUDE SYSLIB(EQAD3CXT) -cobol_linkDebugExit= INCLUDE OBJECT(@{member}) \n INCLUDE SYSLIB(EQAD3CXT) - +# If using a debug exit for IBM Debug tool, provide the SYSIN instream DD which is appended to SYSIN +# Samp: cobol_linkDebugExit= INCLUDE SYSLIB(EQAD3CXT) \n +cobol_linkDebugExit= # # execute link edit step diff --git a/samples/application-conf/PLI.properties b/samples/application-conf/PLI.properties index dba32378..3ec84979 100644 --- a/samples/application-conf/PLI.properties +++ b/samples/application-conf/PLI.properties @@ -47,6 +47,15 @@ pli_compileDebugParms=TEST # can be overridden by file properties pli_linkEditParms=MAP,RENT,COMPAT(PM5) pli_linkEditParms=DYNAM(DLL) :: **/testcase/*.pli + +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n +pli_linkEditStream= + +# If using a debug exit for IBM Debug tool, provide the SYSIN instream DD which is appended to SYSIN +# Samp: cobol_linkDebugExit= INCLUDE SYSLIB(EQAD3CXT) \n +pli_linkDebugExit= + # # execute link edit step # can be overridden by file properties diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index 67ea16cd..1c68d792 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -68,6 +68,7 @@ assembler_pgmParms | Default Assembler parameters. | true assembler_linkEditParms | Default parameters for the link edit step. | true assembler_compileErrorPrefixParms | Default parameters to support remote error feedback in user build scenarios | true assembler_linkEdit | Flag indicating to execute the link edit step to produce a load module for the source file. If false then a object deck will be created instead for later linking. | true +assembler_linkEditStream | Optional linkEditStream defining additional link instructions via SYSIN dd | true assembler_maxRC | Default Assembler maximum RC allowed. | true assembler_linkEditMaxRC | Default link edit maximum RC allowed. | true assembler_impactPropertyList | List of build properties causing programs to rebuild when changed | false @@ -121,7 +122,8 @@ cobol_impactPropertyList | List of build properties causing programs to rebuild cobol_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false cobol_impactPropertyListSQL | List of SQL build properties causing programs to rebuild when changed | false cobol_linkEdit | Flag indicating to execute the link edit step to produce a load module for the source file. If false then a object deck will be created instead for later linking. | true -cobol_isMQ | Flag indicating that the program contains MQ calls | true +cobol_linkEditStream | Optional linkEditStream defining additional link instructions via SYSIN dd | true +cobol_linkDebugExit | linkEditStream to append a debug exit via SYSIN dd | true cobol_deployType | default deployType for build output | true cobol_deployTypeCICS | deployType for build output for build files where isCICS=true | true cobol_deployTypeDLI | deployType for build output for build files with isDLI=true | true @@ -165,6 +167,8 @@ pli_impactPropertyList | List of build properties causing programs to rebuild wh pli_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false pli_impactPropertyListSQL | List of SQL build properties causing programs to rebuild when changed | false pli_linkEditParms | Default link edit parameters. | true +pli_linkEditStream | Optional linkEditStream defining additional link instructions via SYSIN dd | true +pli_linkDebugExit | linkEditStream to append a debug exit via SYSIN dd | true pli_storeSSI | Flag to store abbrev git hash in ssi field in link step | true pli_impactPropertyList | List of build properties causing programs to rebuild when changed | false pli_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index bdd90443..796781ea 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -399,6 +399,44 @@ def isDLI(LogicalFile logicalFile) { return isDLI } +/* + * isMQ - tests to see if the program uses MQ. If the logical file is false, then + * check to see if there is a file property. + */ +def isMQ(LogicalFile logicalFile) { + boolean isMQ = logicalFile.isMQ() + if (!isMQ) { + String isMQFlag = props.getFileProperty('isMQ', logicalFile.getFile()) + if (isMQFlag) + isMQ = isMQFlag.toBoolean() + } + + return isMQ +} + +/* + * getMqStubInstruction - + * returns include defintion for mq sub program for link edit + */ +def getMqStubInstruction(LogicalFile logicalFile) { + String mqStubInstruction + + if (isMQ(logicalFile)) { + // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs + if (isCICS(logicalFile)) { + mqStubInstruction = " INCLUDE SYSLIB(CSQCSTUB)\n" + } else if (isDLI(logicalFile)) { + mqStubInstruction = " INCLUDE SYSLIB(CSQQSTUB)\n" + } else { + mqStubInstruction = " INCLUDE SYSLIB(CSQBSTUB)\n" + } + } else { + println("*! (BuildUtilities.getMqStubInstruction) MQ file attribute for ${logicalFile.getFile()} is false.") + } + + return mqStubInstruction +} + /* * getAbsolutePath - returns the absolute path of a relative (to workspace) file or directory */ From c325e44ce06d5f5eb1d4c82e031b350e13ab739e Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 10 Nov 2022 17:00:45 +0100 Subject: [PATCH 2/9] Search path configuration updates (#267) * Search path configuration updates * Move methods from resolverUtils to build and impact utilities * Adding dependencySearch configurations to list of required build props * Removal of references of old DependencyResolver configuration in application-conf * Removal of references of old DependencyResolver configuration in MortgageApp Signed-off-by: Dennis Behm --- build-conf/Assembler.properties | 3 +- build-conf/Cobol.properties | 3 +- build-conf/PLI.properties | 5 +- build-conf/REXX.properties | 3 +- build-conf/ZunitConfig.properties | 3 +- languages/Assembler.groovy | 5 +- languages/Cobol.groovy | 5 +- languages/PLI.groovy | 5 +- languages/REXX.groovy | 5 +- languages/ZunitConfig.groovy | 3 +- .../application-conf/Assembler.properties | 68 --------- .../application-conf/Cobol.properties | 5 - .../application-conf/README.md | 3 - .../application-conf/application.properties | 43 +----- samples/application-conf/Assembler.properties | 6 - samples/application-conf/Cobol.properties | 6 - samples/application-conf/PLI.properties | 6 - samples/application-conf/README.md | 5 - samples/application-conf/REXX.properties | 6 - .../application-conf/ZunitConfig.properties | 6 - .../application-conf/application.properties | 133 ++---------------- utilities/BuildUtilities.groovy | 81 +++++------ utilities/ImpactUtilities.groovy | 26 +++- utilities/ResolverUtilities.groovy | 66 --------- 24 files changed, 98 insertions(+), 402 deletions(-) delete mode 100644 samples/MortgageApplication/application-conf/Assembler.properties delete mode 100644 utilities/ResolverUtilities.groovy diff --git a/build-conf/Assembler.properties b/build-conf/Assembler.properties index 9590d1d1..587cce71 100644 --- a/build-conf/Assembler.properties +++ b/build-conf/Assembler.properties @@ -4,7 +4,8 @@ # Comma separated list of required build properties for language/Assembler.groovy assembler_requiredBuildProperties=assembler_srcPDS,assembler_macroPDS,assembler_objPDS,assembler_loadPDS, \ assembler_pgm,assembler_linkEditor,assembler_tempOptions,assembler_maxRC, \ - SASMMOD1,SDFHLOAD,SDFHMAC,MACLIB,SCEELKED,SCEEMAC + SASMMOD1,SDFHLOAD,SDFHMAC,MACLIB,SCEELKED,SCEEMAC, \ + assembler_dependencySearch # # assembler source data sets diff --git a/build-conf/Cobol.properties b/build-conf/Cobol.properties index b063330f..e96a5bda 100644 --- a/build-conf/Cobol.properties +++ b/build-conf/Cobol.properties @@ -4,7 +4,8 @@ # Comma separated list of required build properties for Cobol.groovy cobol_requiredBuildProperties=cobol_srcPDS,cobol_cpyPDS,cobol_objPDS,cobol_loadPDS,\ cobol_compiler,cobol_linkEditor,cobol_tempOptions,applicationOutputsCollectionName,\ - SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED + SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED, \ + cobol_dependencySearch # # COBOL compiler name diff --git a/build-conf/PLI.properties b/build-conf/PLI.properties index 60c3764d..1bec3161 100644 --- a/build-conf/PLI.properties +++ b/build-conf/PLI.properties @@ -3,8 +3,9 @@ # # Comma separated list of required build properties for PLI.groovy pli_requiredBuildPropeties=pli_srcPDS,pli_incPDS,pli_objPDS,pli_loadPDS,\ -pli_compiler,pli_linkEditor,pli_tempOptions,applicationOutputsCollectionName,\ -SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED + pli_compiler,pli_linkEditor,pli_tempOptions,applicationOutputsCollectionName,\ + SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED, \ + pli_dependencySearch # # PL/I compiler name diff --git a/build-conf/REXX.properties b/build-conf/REXX.properties index 05c9ee01..0e42a81a 100644 --- a/build-conf/REXX.properties +++ b/build-conf/REXX.properties @@ -4,7 +4,8 @@ # Comma separated list of required build properties for REXX.groovy rexx_requiredBuildProperties=rexx_srcPDS,rexx_objPDS,rexx_loadPDS,\ rexx_cexecPDS, rexx_compiler,rexx_linkEditor,rexx_tempOptions, \ - SFANLMD + SFANLMD, \ + rexx_dependencySearch # # rexx compiler name diff --git a/build-conf/ZunitConfig.properties b/build-conf/ZunitConfig.properties index cd0fb218..813cbd4f 100644 --- a/build-conf/ZunitConfig.properties +++ b/build-conf/ZunitConfig.properties @@ -3,7 +3,8 @@ zunit_requiredBuildProperties=zunit_srcDatasets,zunit_loadDatasets,zunit_reportDatasets,zunit_bzucfgPDS,\ zunit_bzureportPDS,zunit_bzuplayPDS,zunit_srcOptions,zunit_loadOptions,zunit_reportOptions,\ jobCard,zunit_maxPassRC,zunit_maxWarnRC,zunit_playbackFileExtension,zunit_resolutionRules,\ - zunit_bzuplayParms,zunit_userDebugSessionTestParm + zunit_bzuplayParms,zunit_userDebugSessionTestParm, \ + zunit_dependencySearch zunit_bzucfgPDS=${hlq}.BZU.BZUCFG zunit_bzureportPDS=${hlq}.BZU.BZURPT diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index fa8fbb9f..dc329ffe 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -11,7 +11,6 @@ import com.ibm.dbb.build.report.records.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -@Field def resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -30,13 +29,13 @@ sortedList.each { buildFile -> // Configure dependency resolution String dependencySearch = props.getFileProperty('assembler_dependencySearch', buildFile) - def dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // Copy build file and dependency files to data sets buildUtils.copySourceFiles(buildFile, props.assembler_srcPDS, 'assembler_dependenciesDatasetMapping', null ,dependencyResolver) // Create logical file - LogicalFile logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) // create mvs commands String member = CopyToPDS.createMemberName(buildFile) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index d502d176..f0b9a470 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -13,7 +13,6 @@ import com.ibm.dbb.build.report.records.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) -@Field def resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -41,7 +40,7 @@ sortedList.each { buildFile -> // configure dependency resolution and create logical file String dependencySearch = props.getFileProperty('cobol_dependencySearch', buildFile) - def dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // copy build file and dependency files to data sets if(isZUnitTestCase){ @@ -51,7 +50,7 @@ sortedList.each { buildFile -> } // Get logical file - LogicalFile logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) // create mvs commands String member = CopyToPDS.createMemberName(buildFile) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index d77bc45c..96dfc0ab 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -12,7 +12,6 @@ import com.ibm.dbb.build.report.records.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -@Field def resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -39,7 +38,7 @@ sortedList.each { buildFile -> // configure SearchPathDependencyResolver String dependencySearch = props.getFileProperty('pli_dependencySearch', buildFile) - def dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // copy build file and dependency files to data sets @@ -50,7 +49,7 @@ sortedList.each { buildFile -> } // Get logical file - LogicalFile logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) // create mvs commands String member = CopyToPDS.createMemberName(buildFile) diff --git a/languages/REXX.groovy b/languages/REXX.groovy index 719bb10c..7a9c7bf6 100644 --- a/languages/REXX.groovy +++ b/languages/REXX.groovy @@ -7,7 +7,6 @@ import com.ibm.dbb.metadata.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) -@Field def resolverUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -29,13 +28,13 @@ sortedList.each { buildFile -> // configure dependency resolution and create logical file String dependencySearch = props.getFileProperty('rexx_dependencySearch', buildFile) - def dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // copy build file and dependency files to data sets buildUtils.copySourceFiles(buildFile, props.rexx_srcPDS, 'rexx_dependenciesDatasetMapping', null, dependencyResolver) // Get logical file - LogicalFile logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) // create mvs commands String member = CopyToPDS.createMemberName(buildFile) diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index f7671502..36e1c20b 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -11,7 +11,6 @@ import groovy.xml.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) -@Field def resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -34,7 +33,7 @@ buildUtils.createLanguageDatasets(langQualifier) String dependencySearch = props.getFileProperty('zunit_dependencySearch', buildFile) - def dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + SearchPathDependency dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // copy build file and dependency files to data sets buildUtils.copySourceFiles(buildUtils.getAbsolutePath(buildFile), props.zunit_bzucfgPDS, 'zunit_dependenciesDatasetMapping', null, dependencyResolver) diff --git a/samples/MortgageApplication/application-conf/Assembler.properties b/samples/MortgageApplication/application-conf/Assembler.properties deleted file mode 100644 index 4f0b5f87..00000000 --- a/samples/MortgageApplication/application-conf/Assembler.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Application properties used by zAppBuild/language/Assembler.groovy - -# -# default Assemble program build rank - used to sort language build file list -# leave empty - overridden by file properties if sorting needed -assembler_fileBuildRank= - -# -# default Assembler parameters -# can be overridden by file properties -assembler_pgmParms=LIST -assembler_linkEditParms=MAP,RENT,COMPAT(PM5) -assembler_compileErrorPrefixParms=ADATA,EX(ADX(ELAXHASM)) -assembler_db2precompilerParms=HOST(ASM) -assembler_cicsprecompilerParms= - -# -# execute link edit step -# can be overridden by file properties -assembler_linkEdit=true - -# -# default Assembler maximum RCs allowed -# can be overridden by file properties -assembler_maxRC=4 -assembler_linkEditMaxRC=0 - -# -# lists of properties which should cause a rebuild after being changed -assembler_impactPropertyList=assembler_pgmParms -assembler_impactPropertyListCICS=assembler_db2precompilerParms -assembler_impactPropertyListSQL=assembler_cicsprecompilerParms - -# -# ASM Dependency resolution rules -# Rules defined in app-properties file -assembler_resolutionRules=[${maclibRule},${asmCopyRule}] - -# -# store abbrev git hash in ssi field -# available for buildTypes impactBuild, mergeBuild and fullBuild -# can be overridden by file properties -assembler_storeSSI=true - -# -# default deployType -assembler_deployType=LOAD - -# -# deployType for build files with isCICS=true -assembler_deployTypeCICS=CICSLOAD - -# -# deployType for build files with isDLI=true -assembler_deployTypeDLI=IMSLOAD - -# -# scan link edit load module for link dependencies -# can be overridden by file properties -assembler_scanLoadModule=true - -# -# additional libraries for assembler SYSLIB concatenation, comma-separated -assembler_assemblySyslibConcatenation= - -# -# additional libraries for linkEdit SYSLIB concatenation, comma-separated -assembler_linkEditSyslibConcatenation= diff --git a/samples/MortgageApplication/application-conf/Cobol.properties b/samples/MortgageApplication/application-conf/Cobol.properties index 199d4a00..49b6f692 100644 --- a/samples/MortgageApplication/application-conf/Cobol.properties +++ b/samples/MortgageApplication/application-conf/Cobol.properties @@ -5,11 +5,6 @@ # leave empty - overridden by file properties if sorting needed cobol_fileBuildRank= -# -# COBOL dependency resolution rules -# Rules defined in application.properties -cobol_resolutionRules=[${copybookRule}] - # # COBOL dependencySearch configuration # searchPath defined in application.properties diff --git a/samples/MortgageApplication/application-conf/README.md b/samples/MortgageApplication/application-conf/README.md index 0b5f441b..f60e8962 100644 --- a/samples/MortgageApplication/application-conf/README.md +++ b/samples/MortgageApplication/application-conf/README.md @@ -21,11 +21,9 @@ excludeFileList | Files to exclude when scanning or running full build. skipImpactCalculationList | Files for which the impact analysis should be skipped in impact build jobCard | JOBCARD for JCL execs resolveSubsystems | boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile -impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. ** deprecated ** Please consider moving to new SearchPathDepedencyAPI leveraging `impactSearch` configuration. impactSearch | Impact finder resolution search configuration leveraging the SearchPathImpactFinder API. Sample configurations are inlcuded below, next to the previous rule definitions. - ### file.properties Location of file properties, script mappings and file level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for column 'Overridable' in the tables below for build properties that can have file level property overrides. @@ -55,7 +53,6 @@ Application properties used by zAppBuild/language/Cobol.groovy Property | Description | Overridable --- | --- | --- cobol_fileBuildRank | Default Cobol program build rank. Used to sort Cobol build file sub-list. Leave empty. | true -cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true cobol_dependencySearch | Cobol dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true cobol_compilerVersion | Default Cobol compiler version. | true cobol_compileMaxRC | Default compile maximum RC allowed. | true diff --git a/samples/MortgageApplication/application-conf/application.properties b/samples/MortgageApplication/application-conf/application.properties index 88806c93..69fd9203 100644 --- a/samples/MortgageApplication/application-conf/application.properties +++ b/samples/MortgageApplication/application-conf/application.properties @@ -105,25 +105,13 @@ propertyFileExtension=properties resolveSubsystems=false # -# Impact analysis resolution rules (JSON format) -# leverages the ImpactResolver API -# ** deprecated ** - see zAppBuild build property >impactSearch< -impactResolutionRules=[${copybookRule},${bmsRule},${linkRule},${propertyRule}] - +# SearchPathImpactFinder resolution searchPath configuration +# list of multiple search path configurations which are defined below # -# Impact finder resolution search configuration -# leverages the SearchPathImpactFinder API +# this configuration is used when running zAppBuild with the --impactBuild option +# to calculate impacted files based on the identified changed files impactSearch=${copybookSearch}${bmsSearch}${linkSearch} -# -# Rule to locate Cobol copy books -# ** deprecated ** -copybookRule = {"library": "SYSLIB", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/copybook"} \ - ] \ - } - # # copybookSearch # searchPath to locate Cobol copybooks @@ -143,33 +131,16 @@ copybookRule = {"library": "SYSLIB", \ # dependency resolution in the application directory and a shared common copybook location: # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy # -# More samples can be found at: https://www.ibm.com/docs/en/adfz/dbb/1.1.0?topic=scripts-how-manage-build-dependencies +# More samples can be found at: https://www.ibm.com/docs/en/dbb/2.0.0?topic=apis-dependency-impact-resolution # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy -# -# Rule to locate BMS maps -# ** deprecated ** -bmsRule = {"library": "SYSLIB", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/bms"} \ - ] \ - } - # # bmsSearch # searchPath to locate impacted bms maps # used only in impact analysis bmsSearch = search:${workspace}/?path=${application}/bms/*.bms -# Rule to locate COBOL programs and Link files for rebuilding statically linked load modules -# ** deprecated ** -linkRule = {"category": "LINK", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ - ] \ - } - # # linkSearch # @@ -191,7 +162,3 @@ linkRule = {"category": "LINK", \ # linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl -# Rule to detect DBB Build property changes -# ** deprecated ** -propertyRule = {"category": "PROPERTY"} - diff --git a/samples/application-conf/Assembler.properties b/samples/application-conf/Assembler.properties index b12d1991..7f8fde85 100644 --- a/samples/application-conf/Assembler.properties +++ b/samples/application-conf/Assembler.properties @@ -35,12 +35,6 @@ assembler_impactPropertyList=assembler_pgmParms assembler_impactPropertyListCICS=assembler_db2precompilerParms assembler_impactPropertyListSQL=assembler_cicsprecompilerParms -# -# ASM Dependency resolution rules -# Rules defined in app-properties file -# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch -assembler_resolutionRules=[${maclibRule},${asmCopyRule}] - # # ASM dependencySearch configuration # searchPaths defined in app-properties file diff --git a/samples/application-conf/Cobol.properties b/samples/application-conf/Cobol.properties index 8a04555e..48b9f8e4 100644 --- a/samples/application-conf/Cobol.properties +++ b/samples/application-conf/Cobol.properties @@ -5,12 +5,6 @@ # leave empty - overridden by file properties if sorting needed cobol_fileBuildRank= -# -# COBOL dependency resolution rules -# Rules defined in application.properties -# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch -cobol_resolutionRules=[${copybookRule}] - # # COBOL dependencySearch configuration # searchPath defined in application.properties diff --git a/samples/application-conf/PLI.properties b/samples/application-conf/PLI.properties index 3ec84979..c391ea4d 100644 --- a/samples/application-conf/PLI.properties +++ b/samples/application-conf/PLI.properties @@ -5,12 +5,6 @@ # leave empty - overridden by file properties if sorting needed pli_fileBuildRank= -# -# PLI dependency resolution rules -# Rules defined in rules.properties -# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch -pli_resolutionRules=[${plincRule}] - # # PLI dependencySearch configuration # searchPath defined in application.properties diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index 1c68d792..f50963ab 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -74,7 +74,6 @@ assembler_linkEditMaxRC | Default link edit maximum RC allowed. | true assembler_impactPropertyList | List of build properties causing programs to rebuild when changed | false assembler_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false assembler_impactPropertyListSQL | List of SQL build properties causing programs to rebuild when changed | false -assembler_resolutionRules | Assembler dependency resolution rules used to create a Assmebler dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true assembler_dependencySearch | Assembler dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true assembler_storeSSI | Flag to store abbrev git hash in ssi field in link step | true assembler_deployType | default deployType for build output | true @@ -106,7 +105,6 @@ Application properties used by zAppBuild/language/Cobol.groovy Property | Description | Overridable --- | --- | --- cobol_fileBuildRank | Default Cobol program build rank. Used to sort Cobol build file sub-list. Leave empty. | true -cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true cobol_dependencySearch | Cobol dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true cobol_compilerVersion | Default Cobol compiler version. | true cobol_compileMaxRC | Default compile maximum RC allowed. | true @@ -153,7 +151,6 @@ Application properties used by zAppBuild/language/LinkEdit.groovy Property | Description | Overridable --- | --- | --- pli_fileBuildRank | Default PLI program build rank. Used to sort PLI program sub-list. Leave empty. | true -pli_resolutionRules | PLI dependency resolution rules used to create a PLI dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true pli_dependencySearch | PLI dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true pli_compilerVersion | Default PLI compiler version. | true pli_compileMaxRC | Default compile maximum RC allowed. | true @@ -254,7 +251,6 @@ Property | Description | Overridable zunit_maxPassRC | Default zUnit maximum RC allowed for a Pass. | true zunit_maxWarnRC | Default zUnit maximum RC allowed for a Warninig (everything beyond this value will Fail). | true zunit_playbackFileExtension | Default zUnit Playback File Extension. | true -zunit_resolutionRules | Default resolution rules for zUnit. ** deprecated ** | true zunit_dependencySearch | Default zUnit dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true zunit_bzuplayParms | Default options passed to the zUnit runner BZUPLAY | true zunit_userDebugSessionTestParm | Debug Tool Test parameter to initiate the debug session | true @@ -269,7 +265,6 @@ Property | Description | Overridable --- | --- | --- rexx_compileMaxRC | Default compile maximum RC allowed. | true rexx_linkEditMaxRC | Default link edit maximum RC allowed. | true -rexx_resolutionRules | Default resolution rules for zUnit. ** deprecated ** | true rexx_dependencySearch | Default REXX dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true rexx_compileParms | Default base compile parameters. | true rexx_compiler | Default REXX compiler | true diff --git a/samples/application-conf/REXX.properties b/samples/application-conf/REXX.properties index 62e8db43..99a8f252 100644 --- a/samples/application-conf/REXX.properties +++ b/samples/application-conf/REXX.properties @@ -5,12 +5,6 @@ # leave empty - overridden by file properties if sorting needed rexx_fileBuildRank= -# -# REXX dependency resolution rules -# Rules defined in application.properties -# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch -rexx_resolutionRules=[${rexxRule}] - # # REXX dependencySearch configuration # searchPath defined in application.properties diff --git a/samples/application-conf/ZunitConfig.properties b/samples/application-conf/ZunitConfig.properties index 0c1bb6a6..d3f12908 100755 --- a/samples/application-conf/ZunitConfig.properties +++ b/samples/application-conf/ZunitConfig.properties @@ -9,12 +9,6 @@ zunit_maxWarnRC=8 # file extension of zunit playback files zunit_playbackFileExtension=plbck -# -# zUnit dependency resolution rules -# Rules defined in application.properties -# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch -zunit_resolutionRules=[${testcaseRule}] - # # zUnit dependencySearch configuration # searchPath defined in application.properties diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index f1282d31..fa21c760 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -116,6 +116,7 @@ propertyFileExtension=properties ############################################################### # Dependency Analysis and Impact Analysis configuration ############################################################### + # # boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact # the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile @@ -124,25 +125,12 @@ propertyFileExtension=properties resolveSubsystems=false # -# Impact analysis resolution rules (JSON format). -# Defaults to just looking for local application dependency folders -# ** deprecated ** - see zAppBuild build property >impactSearch< -impactResolutionRules=[${copybookRule},${plincRule},${maclibRule},${asmCopyRule},${rexxRule},${linkRule},${testcaseRule},${testconfigRule},${testcasePgmRule},${propertyRule}] - +# SearchPathImpactFinder resolution searchPath configuration +# list of multiple search path configurations which are defined below # -# Impact finder resolution search configuration -# leverages the SearchPathImpactFinder API -impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitApplicationPgmSearch} - -# Rule to locate Cobol copy books. This rule defaults to the local copybook folder -# in the main application folder. -# used in dependency resolution and impact analysis -# ** deprecated ** -copybookRule = {"library": "SYSLIB", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/copybook"} \ - ] \ - } +# this configuration is used when running zAppBuild with the --impactBuild option +# to calculate impacted files based on the identified changed files +impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitTestcasePgmSearch} # # copybookSearch @@ -164,98 +152,39 @@ copybookRule = {"library": "SYSLIB", \ # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy # # More samples can be found along with the syntax for the search path configurations at: -# https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-manage-build-dependencies#6-resolving-logical-build-dependencies-to-local-physical-files +# https://www.ibm.com/docs/en/dbb/2.0.0?topic=apis-dependency-impact-resolution#6-resolving-logical-build-dependencies-to-local-physical-files # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy - -# Rule to locate PLI include files. This rule defaults to the local plinc folder -# in the main application folder. -# used in dependency resolution and impact analysis -plincRule = {"library": "SYSLIB", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/plinc"} \ - ] \ - } # # pliincludeSearch # searchPath to locate PLI include files # used in dependency resolution and impact analysis pliincludeSearch = search:${workspace}/?path=${application}/plinc/*.cpy -# Rule to locate ASM macros. This rule defaults to the local maclib folder -# in the main application folder. -# Category filters on what you want to include during the scanner (i.e. excludes macro-def keyword) -# used in dependency resolution and impact analysis -# ** deprecated ** -maclibRule = {"library": "SYSLIB", "category": "MACRO", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/maclib"} \ - ] \ - } - # asmMacroSearch # searchPath to locate Assembler macro files +# use category filters on what you want to include during the scan (i.e. excludes macro-def keyword) # used in dependency resolution and impact analysis asmMacroSearch = search:[SYSLIB:MACRO]${workspace}/?path=${application}/maclib/*.mac -# Rule to locate ASM copybooks. This rule defaults to the local maclib folder -# in the main application folder. -# used in dependency resolution and impact analysis -# ** deprecated ** -asmCopyRule = {"library": "SYSLIB", "category": "COPY", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/maclib"} \ - ] \ - } - # asmCopySearch # searchPath to locate Assembler copy files # used in dependency resolution and impact analysis asmCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/maclib/*.mac -# -# Rule to locate BMS maps -# used only in impact analysis -# ** deprecated ** -bmsRule = {"library": "SYSLIB", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/bms"} \ - ] \ - } - # # bmsSearch # searchPath to locate impacted bms maps # used only in impact analysis bmsSearch = search:${workspace}/?path=${application}/bms/*.bms -# Rule to locate REXX includes. This rule defaults to the local rexx folder -# in the main application folder. -# used in dependency resolution and impact analysis -# ** deprecated ** -rexxRule = {"library": "SYSLIB", "category": "COPY", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/rexx"} \ - ] \ - } - # # rexxCopySearch -# searchPath to locate rexx copy +# searchPath to locate rexx copy - defaults to the local rexx folder in the main application folder # used in dependency resolution and impact analysis rexxCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/rexx/*.rexx -# Rule to locate COBOL programs and Link files for rebuilding statically linked load modules -# searchPath configuration for dependencies to a COBOL program located in the cobol subfolder -# used only in impact analysis -# ** deprecated ** -linkRule = {"category": "LINK", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ - ] \ - } - # # linkSearch # @@ -272,7 +201,7 @@ linkRule = {"category": "LINK", \ # staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl,**/*.pli # # More samples can be found along with the syntax for the search path configurations at: -# https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-manage-build-dependencies#6-resolving-logical-build-dependencies-to-local-physical-files +# https://www.ibm.com/docs/en/dbb/2.0.0?topic=apis-dependency-impact-resolution#6-resolving-logical-build-dependencies-to-local-physical-files # # Special case with Dependency Scanner Transfer Control Statement capturing turned on (default is off) # the scanners detect a static call to the literal, which would need to turn into a new rule for CALL: @@ -280,28 +209,10 @@ linkRule = {"category": "LINK", \ # linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl -# Rule to locate the zUnit test configuration file -# used in dependency resolution and impact analysis -# ** deprecated ** -> zunitTestConfigSearch -testconfigRule = {"library": "SYSPROG", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/testcfg"} \ - ] \ - } - # zunitTestConfigSearch # searchPath to locate zunit config files -# used in dependency resolution and impact analysis -zunitTestConfigSearch = search:[SYSPROG:]${workspace}/?path=${application}/testcfg/*.bzucfg - -# Rule to locate the zUnit playback file -# used in dependency resolution and impact analysis -# ** deprecated ** -> zunitPlayfileSearch -testcaseRule = {"library": "SYSPLAY", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/testplayfiles"} \ - ] \ - } +# used in impact analysis +zunitTestConfigSearch = search:[:ZUNITINC]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli;${application}/testcase/*.cbl;${application}/testcase/*.pli # # zunitPlayfileSearch @@ -309,23 +220,9 @@ testcaseRule = {"library": "SYSPLAY", \ # used in dependency resolution zunitPlayfileSearch = search:[SYSPLAY:]${workspace}/?path=${application}/testplayfiles/*.bzuplay - -# Rule to locate the zUnit test case program when the program or the bzucfg file changes -# used in impact analysis -# ** deprecated ** -> zunitApplicationPgmSearch -testcasePgmRule = {"category": "PROGRAMDEPENDENCY", \ - "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ - ] \ - } - # -# zunitApplicationPgmSearch -# searchPath to locate impacted application programs +# zunitTestcasePgmSearch +# searchPath to locate impacted test case programs # see also build-conf/build.properties -> createTestcaseDependency # used in impact analysis -zunitApplicationPgmSearch = search:[:PROGRAMDEPENDENCY]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli - -# Rule to detect DBB Build property changes -# ** deprecated ** -propertyRule = {"category": "PROPERTY"} +zunitTestcasePgmSearch = search:[SYSPROG:PROGRAMDEPENDENCY]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 796781ea..0f1b2b03 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -91,7 +91,7 @@ def getFileSet(String dir, boolean relativePaths, String includeFileList, String * - DependencyResolver to resolve dependencies */ -def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMapping, String dependenciesAlternativeLibraryNameMapping, Object dependencyResolver) { +def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMapping, String dependenciesAlternativeLibraryNameMapping, SearchPathDependencyResolver dependencyResolver) { // only copy the build file once if (!copiedFileCache.contains(buildFile)) { copiedFileCache.add(buildFile) @@ -115,8 +115,6 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap String lname = CopyToPDS.createMemberName(buildFile) String language = props.getFileProperty('dbb.DependencyScanner.languageHint', buildFile) ?: 'UNKN' LogicalFile lfile = new LogicalFile(lname, buildFile, language, depFileData.isCICS, depFileData.isSQL, depFileData.isDLI) - // set logical file in the dependency resolver if using deprecated API - //dependencyResolver.setLogicalFile(lfile) // get list of dependencies from userBuildDependencyFile List dependencyPaths = depFileData.dependencies @@ -158,9 +156,7 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap else if (dependencyDatasetMapping && dependencyResolver) { // resolve the logical dependencies to physical files to copy to data sets - resolverUtils = loadScript(new File("ResolverUtilities.groovy")) - List physicalDependencies = resolverUtils.resolveDependencies(dependencyResolver, buildFile) - + List physicalDependencies = resolveDependencies(dependencyResolver, buildFile) if (props.verbose) println "*** Physical dependencies for $buildFile:" @@ -305,54 +301,45 @@ def updateBuildResult(Map args) { } } -/* - * createDependencyResolver - Creates a dependency resolver using resolution rules declared - * in a build or file property (json format). +/** + * Method to create the logical file using SearchPathDependencyResolver + * + * evaluates if it should resolve file flags for resolved dependencies + * + * @param spDependencyResolver + * @param buildFile + * @return logicalFile */ -// def createDependencyResolver(String buildFile, String rules) { -// if (props.verbose) println "*** Creating dependency resolver for $buildFile with $rules rules" -// // create a dependency resolver for the build file -// DependencyResolver resolver = new DependencyResolver().file(buildFile) -// .sourceDir(props.workspace) +def createLogicalFile(SearchPathDependencyResolver spDependencyResolver, String buildFile) { -// // add scanner if userBuild Dep File not provided, or not a user build -// if (!props.userBuildDependencyFile || !props.userBuild) -// resolver.setScanner(getScanner(buildFile)) - -// // add resolution rules -// if (rules) -// resolver.setResolutionRules(parseResolutionRules(rules)) + LogicalFile logicalFile + + if (props.resolveSubsystems && props.resolveSubsystems.toBoolean()) { + // include resolved dependencies to define file flags of logicalFile + logicalFile = spDependencyResolver.resolveSubsystems(buildFile,props.workspace) + } + else { + logicalFile = SearchPathDependencyResolver.getLogicalFile(buildFile,props.workspace) + } -// return resolver -// } + return logicalFile -// def parseResolutionRules(String json) { -// List rules = new ArrayList() -// JsonSlurper slurper = new groovy.json.JsonSlurper() -// List jsonRules = slurper.parseText(json) -// if (jsonRules) { -// jsonRules.each { jsonRule -> -// ResolutionRule resolutionRule = new ResolutionRule() -// resolutionRule.library(jsonRule.library) -// resolutionRule.lname(jsonRule.lname) -// resolutionRule.category(jsonRule.category) -// if (jsonRule.searchPath) { -// jsonRule.searchPath.each { jsonPath -> -// DependencyPath dependencyPath = new DependencyPath() -// dependencyPath.collection(jsonPath.collection) -// dependencyPath.sourceDir(jsonPath.sourceDir) -// dependencyPath.directory(jsonPath.directory) -// resolutionRule.path(dependencyPath) -// } -// } -// rules << resolutionRule -// } -// } -// return rules -// } +} +/** + * Method to execute dependency resolution based on configured SearchPathDependencyResolver + * + * @return resolved list of physical dependencies + */ +def resolveDependencies(SearchPathDependencyResolver dependencyResolver, String buildFile) { + if (props.verbose) { + println "*** Resolution rules for $buildFile:" + println dependencyResolver.getSearchPath() + } + return dependencyResolver.resolveDependencies(buildFile, props.workspace) +} /* * isCICS - tests to see if the program is a CICS program. If the logical file is false, then diff --git a/utilities/ImpactUtilities.groovy b/utilities/ImpactUtilities.groovy index 58fb06e4..6ba499ba 100644 --- a/utilities/ImpactUtilities.groovy +++ b/utilities/ImpactUtilities.groovy @@ -19,7 +19,6 @@ import java.util.regex.* def createImpactBuildList() { MetadataStore metadataStore = MetadataStoreFactory.getMetadataStore() - resolverUtils = loadScript(new File("ResolverUtilities.groovy")) // local variables Set changedFiles = new HashSet() @@ -77,7 +76,7 @@ def createImpactBuildList() { // list of impacts String impactSearch = props.getFileProperty('impactSearch', changedFile) - def impacts = resolverUtils.findImpactedFiles(impactSearch, changedFile) + def impacts = findImpactedFiles(impactSearch, changedFile) impacts.each { impact -> @@ -197,6 +196,29 @@ def createMergeBuildList(){ return [buildSet, changedFiles, deletedFiles, renamedFiles, changedBuildProperties] } +/* + * findImpactedFiles - + * method to configure and invoke SearchPathImpactFinder + * + * @return list of impacted files + * + */ +def findImpactedFiles(String impactSearch, String changedFile) { + + List collections = new ArrayList() + collections.add(props.applicationCollectionName) + collections.add(props.applicationOutputsCollectionName) + + if (props.verbose) + println ("*** Creating SearchPathImpactFinder with collections " + collections + " and impactSearch configuration " + impactSearch) + + def finder = new SearchPathImpactFinder(impactSearch, collections) + + // Find all files impacted by the changed file + impacts = finder.findImpactedFiles(changedFile, props.workspace) + return impacts +} + /* * calculateChangedFiles - method to caluclate the the changed files diff --git a/utilities/ResolverUtilities.groovy b/utilities/ResolverUtilities.groovy deleted file mode 100644 index 9fe55991..00000000 --- a/utilities/ResolverUtilities.groovy +++ /dev/null @@ -1,66 +0,0 @@ -@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript -import com.ibm.dbb.dependency.* -import com.ibm.dbb.metadata.* -import com.ibm.dbb.build.* -import groovy.transform.* - -@Field BuildProperties props = BuildProperties.getInstance() - -// Externalized Method to preserve backward compatibility with older DBB toolkit versions. -// TODO: Refactoring as soon as the deprecated API is dropped. - -/** - * Method to create the logical file using SearchPathDependencyResolver - * - * evaluates if it should resolve file flags for resolved dependencies - * - * @param spDependencyResolver - * @param buildFile - * @return - */ - -def createLogicalFile(SearchPathDependencyResolver spDependencyResolver, String buildFile) { - - LogicalFile logicalFile - - if (props.resolveSubsystems && props.resolveSubsystems.toBoolean()) // include resolved dependencies to define file flags of logicalFile - logicalFile = spDependencyResolver.resolveSubsystems(buildFile,props.workspace) - else - logicalFile = SearchPathDependencyResolver.getLogicalFile(buildFile,props.workspace) - - return logicalFile - -} - -/** - * - * @param dependencySearch - * @return SearchPathDependencyResolver - */ -def createSearchPathDependencyResolver(String dependencySearch) { - return new SearchPathDependencyResolver(dependencySearch) -} - -def findImpactedFiles(String impactSearch, String changedFile) { - - List collections = new ArrayList() - collections.add(props.applicationCollectionName) - collections.add(props.applicationOutputsCollectionName) - - if (props.verbose) - println ("*** Creating SearchPathImpactFinder with collections " + collections + " and impactSearch configuration " + impactSearch) - - def finder = new SearchPathImpactFinder(impactSearch, collections) - - // Find all files impacted by the changed file - impacts = finder.findImpactedFiles(changedFile, props.workspace) - return impacts -} - -def resolveDependencies(SearchPathDependencyResolver dependencyResolver, String buildFile) { - if (props.verbose) { - println "*** Resolution rules for $buildFile:" - println dependencyResolver.getSearchPath() - } - return dependencyResolver.resolveDependencies(buildFile, props.workspace) -} \ No newline at end of file From af198710b77dffee604ccff336a6616b10ec0256 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 15 Dec 2022 21:35:37 +0100 Subject: [PATCH 3/9] ZunitConfig.groovy updates (#299) * ZunitConfig groovy refresh to use logicalFile Signed-off-by: Dennis Behm --- languages/ZunitConfig.groovy | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index 36e1c20b..43c51e93 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -10,7 +10,6 @@ import groovy.xml.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -@Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) println("** Building files mapped to ${this.class.getName()}.groovy script") @@ -33,15 +32,18 @@ buildUtils.createLanguageDatasets(langQualifier) String dependencySearch = props.getFileProperty('zunit_dependencySearch', buildFile) - SearchPathDependency dependencyResolver = new SearchPathDependencyResolver(dependencySearch) + SearchPathDependencyResolver dependencyResolver = new SearchPathDependencyResolver(dependencySearch) // copy build file and dependency files to data sets - buildUtils.copySourceFiles(buildUtils.getAbsolutePath(buildFile), props.zunit_bzucfgPDS, 'zunit_dependenciesDatasetMapping', null, dependencyResolver) - - // Parse the playback from the bzucfg file - Boolean hasPlayback = false - String playback - (hasPlayback, playback) = getPlaybackFile(buildFile); + buildUtils.copySourceFiles(buildFile, props.zunit_bzucfgPDS, 'zunit_dependenciesDatasetMapping', null, dependencyResolver) + + // get logical file + LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) + + // get playback dependency for bzucfg file from logicalFile + boolean hasPlayback = false + LogicalDependency playbackFile + (hasPlayback, playbackFile) = getPlaybackFile(logicalFile); // Create JCLExec String String jobcard = props.jobCard.replace("\\n", "\n") @@ -67,7 +69,7 @@ jcl += """\ if (hasPlayback) { // bzucfg contains reference to a playback file jcl += "//REPLAY.BZUPLAY DD DISP=SHR, \n" + - "// DSN=${props.zunit_bzuplayPDS}(${playback}) \n" + "// DSN=${props.zunit_bzuplayPDS}(${playbackFile.getLname()}) \n" } else { // no playbackfile referenced jcl += "//REPLAY.BZUPLAY DD DUMMY \n" @@ -216,17 +218,17 @@ zunitDebugParm = props.getFileProperty('zunit_userDebugSessionTestParm', buildFi */ /* - * returns containsPlayback, + * returns the LogicalDependency of the playbackfile */ -def getPlaybackFile(String xmlFile) { - String xml = new File(buildUtils.getAbsolutePath(xmlFile)).getText("IBM-1047") - def parser = new XmlParser().parseText(xml) - if (parser.'runner:playback'.playbackFile.size()==0) return [false, null] - else { - String playbackFileName = parser.'runner:playback'.@moduleName[0] - return [true, playbackFileName] - } -} +def getPlaybackFile(LogicalFile logicalFile) { + // find playback file dependency + LogicalDependency playbackDependency = logicalFile.getLogicalDependencies().find { + it.getLibrary() == "SYSPLAY" + } + if (playbackDependency) { + return [true, playbackDependency] + } + } /** * Parsing the result file and prints summary of the result From 0be0854eca7c8eb91d36cb79347b6f291a12ee61 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 11 Jan 2023 13:06:46 +0100 Subject: [PATCH 4/9] Remove zunit_resolutionRules from required build properties (#307) Signed-off-by: Dennis Behm --- build-conf/ZunitConfig.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-conf/ZunitConfig.properties b/build-conf/ZunitConfig.properties index 813cbd4f..c4c28fc9 100644 --- a/build-conf/ZunitConfig.properties +++ b/build-conf/ZunitConfig.properties @@ -2,7 +2,7 @@ zunit_requiredBuildProperties=zunit_srcDatasets,zunit_loadDatasets,zunit_reportDatasets,zunit_bzucfgPDS,\ zunit_bzureportPDS,zunit_bzuplayPDS,zunit_srcOptions,zunit_loadOptions,zunit_reportOptions,\ - jobCard,zunit_maxPassRC,zunit_maxWarnRC,zunit_playbackFileExtension,zunit_resolutionRules,\ + jobCard,zunit_maxPassRC,zunit_maxWarnRC,zunit_playbackFileExtension,\ zunit_bzuplayParms,zunit_userDebugSessionTestParm, \ zunit_dependencySearch From 21ed56e6512a62ef1ba6cd1e6677ae2669c02e83 Mon Sep 17 00:00:00 2001 From: TL Donnelly <43755127+donnellt@users.noreply.github.com> Date: Fri, 13 Jan 2023 08:30:09 -0800 Subject: [PATCH 5/9] Externalize the Timestamp Format for Build Folder and Build Label (#314) Externalize the Timestamp Format for Build Folder and Build Label Signed-off-by: TL Donnelly <43755127+donnellt@users.noreply.github.com> --- build-conf/build.properties | 18 ++++++++++++------ build.groovy | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/build-conf/build.properties b/build-conf/build.properties index e0f1656b..7633c347 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -10,7 +10,7 @@ buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.propert buildListFileExt=txt # -# Service URL for the Git provider to have a visual comparision of two hashes +# Service URL for the Git provider to have a visual comparision of two hashes # Leveraged as a build result property /compare/ # samples: GitHub : /compare/ ; GitLab : /-/compare/ gitRepositoryCompareService=/compare/ @@ -44,7 +44,7 @@ applicationConfRootDir= # # Minimum required DBB ToolkitVersion to run this version of zAppBuild -# Build initialization process validates the DBB Toolkit Version in use and matches that against this setting +# Build initialization process validates the DBB Toolkit Version in use and matches that against this setting requiredDBBToolkitVersion=2.0.0 # @@ -116,15 +116,21 @@ continueOnScanFailure=true # Default: true createBuildOutputSubfolder=true +# +# Build Timestamp Format +# Applies to all build types except userBuild +# Default: yyyyMMdd.HHmmss.mmm - See Date format method pattern strings +buildOutputTSformat=yyyyMMdd.HHmmss.mmm + # # Flag to determine if the build framework should document deletions of outputs in DBB Build Report # for build files being mapped to language scripts # -# Requires the DBB toolkit 1.1.3 or higher. Backward compatibility of zAppBuild is preserved, +# Requires the DBB toolkit 1.1.3 or higher. Backward compatibility of zAppBuild is preserved, # when feature is turned off -# +# # Default : false -documentDeleteRecords=false +documentDeleteRecords=false # MetadataStore configuration properties: @@ -139,7 +145,7 @@ metadataStoreType=file #metadataStoreDb2Url=jdbc:db2: # Db2 connection configuration property file -# Sample is povided at $DBB_HOME/conf/db2Connection.conf +# Sample is povided at $DBB_HOME/conf/db2Connection.conf #metadataStoreDb2ConnectionConf= diff --git a/build.groovy b/build.groovy index 74a705a7..c2d079da 100644 --- a/build.groovy +++ b/build.groovy @@ -21,9 +21,9 @@ import groovy.cli.commons.* @Field String giturlPrefix = ':giturl:' @Field String gitchangedfilesPrefix = ':gitchangedfiles:' @Field MetadataStore metadataStore +@Field startTime = new Date() // start time message -def startTime = new Date() props.startTime = startTime.format("yyyyMMdd.hhmmss.mmm") println("\n** Build start at $props.startTime") @@ -446,7 +446,7 @@ def populateBuildProperties(def opts) { props.topicBranchBuild = (props.applicationCurrentBranch.equals(props.mainBuildBranch)) ? null : 'true' props.applicationBuildGroup = ((props.applicationCurrentBranch) ? "${props.application}-${props.applicationCurrentBranch}" : "${props.application}") as String - props.applicationBuildLabel = "build.${props.startTime}" as String + props.applicationBuildLabel = ("build." + ( (props.buildOutputTSformat) ? startTime.format("${props.buildOutputTSformat}") : "${props.startTime}" ) ) as String props.applicationCollectionName = ((props.applicationCurrentBranch) ? "${props.application}-${props.applicationCurrentBranch}" : "${props.application}") as String props.applicationOutputsCollectionName = "${props.applicationCollectionName}-outputs" as String From 3d0ae0652c7e8564faba14357335359160b7ea56 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 24 Feb 2023 08:29:01 +0100 Subject: [PATCH 6/9] Updating zappbuild test framework (#325) * Updating zappbuild test framework Signed-off-by: Dennis Behm --- test/testScripts/fullBuild.groovy | 2 +- test/testScripts/impactBuild.groovy | 2 +- test/testScripts/impactBuild_deletion.groovy | 5 ++-- .../testScripts/impactBuild_properties.groovy | 6 ++--- test/testScripts/impactBuild_renaming.groovy | 2 +- test/testScripts/mergeBuild.groovy | 2 +- test/testScripts/resetBuild.groovy | 26 ++++++++++++------- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/test/testScripts/fullBuild.groovy b/test/testScripts/fullBuild.groovy index d6c33a02..b706b371 100644 --- a/test/testScripts/fullBuild.groovy +++ b/test/testScripts/fullBuild.groovy @@ -57,7 +57,7 @@ try { catch(AssertionError e) { def result = e.getMessage() assertionList << result; - props.testsSucceeded = false + props.testsSucceeded = 'false' } finally { cleanUpDatasets() diff --git a/test/testScripts/impactBuild.groovy b/test/testScripts/impactBuild.groovy index 413a819e..cdd35749 100644 --- a/test/testScripts/impactBuild.groovy +++ b/test/testScripts/impactBuild.groovy @@ -102,7 +102,7 @@ def validateImpactBuild(String changedFile, PropertyMappings filesBuiltMappings, catch(AssertionError e) { def result = e.getMessage() assertionList << result; - props.testsSucceeded = false + props.testsSucceeded = 'false' } } def cleanUpDatasets() { diff --git a/test/testScripts/impactBuild_deletion.groovy b/test/testScripts/impactBuild_deletion.groovy index 988662c8..34c10a56 100644 --- a/test/testScripts/impactBuild_deletion.groovy +++ b/test/testScripts/impactBuild_deletion.groovy @@ -25,7 +25,7 @@ fullBuildCommand << "--url ${props.url}" fullBuildCommand << "--id ${props.id}" fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") fullBuildCommand << (props.verbose ? "--verbose" : "") -fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}" : "--propFiles ${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}") fullBuildCommand << "--fullBuild" // create impact build command @@ -41,7 +41,7 @@ impactBuildCommand << "--url ${props.url}" impactBuildCommand << "--id ${props.id}" impactBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") impactBuildCommand << "--verbose" -impactBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}" : "") +impactBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}" : "--propFiles ${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_deletion_buildPropSetting}") impactBuildCommand << "--impactBuild" // iterate through change files to test impact build @@ -147,6 +147,7 @@ def validateImpactBuild(String deleteFile, PropertyMappings outputsDeletedMappin catch(AssertionError e) { def result = e.getMessage() assertionList << result; + props.testsSucceeded = 'false' } } def cleanUpDatasets() { diff --git a/test/testScripts/impactBuild_properties.groovy b/test/testScripts/impactBuild_properties.groovy index 727cd82f..6bbea355 100644 --- a/test/testScripts/impactBuild_properties.groovy +++ b/test/testScripts/impactBuild_properties.groovy @@ -25,7 +25,7 @@ fullBuildCommand << "--url ${props.url}" fullBuildCommand << "--id ${props.id}" fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") fullBuildCommand << (props.verbose ? "--verbose" : "") -fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}" : "--propFiles ${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}") fullBuildCommand << "--fullBuild" // create impact build command @@ -41,7 +41,7 @@ impactBuildCommand << "--url ${props.url}" impactBuildCommand << "--id ${props.id}" impactBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") impactBuildCommand << (props.verbose ? "--verbose" : "") -impactBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}" : "") +impactBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles},${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}" : "--propFiles ${props.zAppBuildDir}/test/applications/${props.app}/${props.impactBuild_properties_buildPropSetting}") impactBuildCommand << "--impactBuild" // iterate through change files to test impact build @@ -126,7 +126,7 @@ def validateImpactBuild(String changedFile, PropertyMappings filesBuiltMappings, catch(AssertionError e) { def result = e.getMessage() assertionList << result; - props.testsSucceeded = false + props.testsSucceeded = 'false' } } def cleanUpDatasets() { diff --git a/test/testScripts/impactBuild_renaming.groovy b/test/testScripts/impactBuild_renaming.groovy index de52df40..8e7ed295 100644 --- a/test/testScripts/impactBuild_renaming.groovy +++ b/test/testScripts/impactBuild_renaming.groovy @@ -113,7 +113,7 @@ def validateImpactBuild(String renameFile, PropertyMappings filesBuiltMappings, catch(AssertionError e) { def result = e.getMessage() assertionList << result; - props.testsSucceeded = false + props.testsSucceeded = 'false' } } def cleanUpDatasets() { diff --git a/test/testScripts/mergeBuild.groovy b/test/testScripts/mergeBuild.groovy index 1567647c..b384b19e 100644 --- a/test/testScripts/mergeBuild.groovy +++ b/test/testScripts/mergeBuild.groovy @@ -117,7 +117,7 @@ def validateMergeBuild(String changedFile, PropertyMappings filesBuiltMappings, catch(AssertionError e) { def result = e.getMessage() assertionList << result; - props.testsSucceeded = false + props.testsSucceeded = 'false' } } def cleanUpDatasets() { diff --git a/test/testScripts/resetBuild.groovy b/test/testScripts/resetBuild.groovy index a66d5b57..f549afa5 100644 --- a/test/testScripts/resetBuild.groovy +++ b/test/testScripts/resetBuild.groovy @@ -33,12 +33,20 @@ def process = ['bash', '-c', resetBuildCommand.join(" ")].execute() def outputStream = new StringBuffer(); process.waitForProcessOutput(outputStream, System.err) -// Validate reset build -println "** Validating reset build" - -// Validate clean reset build -assert outputStream.contains("Deleting collection") && ("Deleting build result group") && ("Build finished") : "*! RESET OF THE BUILD FAILED\nOUTPUT STREAM:\n$outputStream\n" - -println "**" -println "** RESET OF THE BUILD : PASSED **" -println "**" + +try { + // Validate reset build + println "** Validating reset build" + + // Validate clean reset build + assert (outputStream.contains("Build finished")) && (process.exitValue() == 0) : "*! RESET OF THE BUILD FAILED\nOUTPUT STREAM:\n$outputStream\n" + + println "**" + println "** RESET OF THE BUILD : PASSED **" + println "**" + +}catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' +} From 01fc2bc1c763406ee42bc5576e459bff814371c0 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 24 Feb 2023 08:31:50 +0100 Subject: [PATCH 7/9] Simplify obtaining the zunit playbackfile dependency (#311) * Simplify obtaining the zunit playbackfile dependency Signed-off-by: Dennis Behm --- languages/ZunitConfig.groovy | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index 43c51e93..7dcd2e5f 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -41,9 +41,7 @@ buildUtils.createLanguageDatasets(langQualifier) LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) // get playback dependency for bzucfg file from logicalFile - boolean hasPlayback = false - LogicalDependency playbackFile - (hasPlayback, playbackFile) = getPlaybackFile(logicalFile); + LogicalDependency playbackFile = getPlaybackFile(logicalFile); // Create JCLExec String String jobcard = props.jobCard.replace("\\n", "\n") @@ -66,7 +64,7 @@ zunitParms = props.getFileProperty('zunit_bzuplayParms', buildFile) jcl += """\ // PARM=('$zunitParms') """ - if (hasPlayback) { // bzucfg contains reference to a playback file + if (playbackFile != null) { // bzucfg contains reference to a playback file jcl += "//REPLAY.BZUPLAY DD DISP=SHR, \n" + "// DSN=${props.zunit_bzuplayPDS}(${playbackFile.getLname()}) \n" @@ -225,9 +223,7 @@ def getPlaybackFile(LogicalFile logicalFile) { LogicalDependency playbackDependency = logicalFile.getLogicalDependencies().find { it.getLibrary() == "SYSPLAY" } - if (playbackDependency) { - return [true, playbackDependency] - } + return playbackDependency } /** From a65089c67c345cb897e7f1ed07e94d59682d178f Mon Sep 17 00:00:00 2001 From: Anuprakash Moothedath Date: Mon, 6 Mar 2023 18:45:06 +0530 Subject: [PATCH 8/9] Implement language configuration strategy to provide alternative approach to define file property overrides (#295) Implement language configuration strategy to provide alternative approach to define file property overrides --------- Signed-off-by: Anuprakash Moothedath Signed-off-by: Dennis Behm Signed-off-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> Co-authored-by: Dennis Behm Co-authored-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> --- README.md | 1 + build-conf/README.md | 7 + .../languageConfigProps01.properties | 11 + .../languageConfigProps02.properties | 15 ++ build.groovy | 10 +- docs/FilePropertyManagement.md | 174 +++++++++++++ docs/README.md | 7 + languages/Cobol.groovy | 2 + languages/LinkEdit.groovy | 3 + .../application-conf/README.md | 8 + .../application-conf/application.properties | 61 ++--- .../application-conf/file.properties | 21 +- .../languageConfigurationMapping.properties | 11 + .../properties/epsmlist.cbl.properties | 8 +- samples/application-conf/README.md | 18 +- .../application-conf/application.properties | 61 ++--- .../languageConfigurationMapping.properties | 11 + .../file_languageConfig_TC1.properties | 43 ++++ .../file_languageConfig_TC2.properties | 43 ++++ .../languageConfigProps01.properties | 18 ++ .../languageConfigProps02.properties | 15 ++ .../MortgageApplication/test.properties | 63 ++++- test/testScripts/README.md | 23 ++ .../fullBuild_languageConfigurations.groovy | 228 ++++++++++++++++++ test/testScripts/impactBuild.groovy | 33 ++- utilities/BuildUtilities.groovy | 37 +-- utilities/FilePropUtilities.groovy | 216 +++++++++++++++++ utilities/README.md | 3 +- 28 files changed, 1007 insertions(+), 144 deletions(-) create mode 100644 build-conf/language-conf/languageConfigProps01.properties create mode 100644 build-conf/language-conf/languageConfigProps02.properties create mode 100644 docs/FilePropertyManagement.md create mode 100644 docs/README.md create mode 100644 samples/MortgageApplication/application-conf/languageConfigurationMapping.properties create mode 100644 samples/application-conf/languageConfigurationMapping.properties create mode 100644 test/applications/MortgageApplication/application-conf/file_languageConfig_TC1.properties create mode 100644 test/applications/MortgageApplication/application-conf/file_languageConfig_TC2.properties create mode 100644 test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps01.properties create mode 100644 test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps02.properties create mode 100644 test/testScripts/fullBuild_languageConfigurations.groovy create mode 100644 utilities/FilePropUtilities.groovy diff --git a/README.md b/README.md index 685abaf2..390a248e 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ zAppBuild comes with a set of reporting features. It helps development teams to Folder/File | Description | Documentation Link --- | --- | --- build-conf | This folder contains global configuration properties used by build.groovy and language build scripts. | [build-conf/README.md](build-conf/README.md) +docs | This folder contains supplemental documentation to help explain the implementation and usage of zAppBuild features. | [docs/README.md](docs/README.md) languages | This folder contains the language specific build scripts that are associated to build files via script mappings (see `samples/application-conf/files.properties`) and called by build.groovy. | [languages/README.md](languages/README.md) samples/application-conf | The `application-conf` folder contains application specific configuration properties used by build.groovy and language build scripts. It is intended to be copied as a high level folder to the application repository and configured to meet the build requirments of the application. Ex. `myAppRepository/application-conf` | [samples/application-conf/README.md](samples/application-conf/README.md) samples/MortgageApplication | This is an updated version of the original [MortgageApplication](https://github.com/IBM/dbb/tree/master/Build/MortgageApplication) sample designed to be built by zAppBuild. | [samples/MortgageApplication/README.md](samples/MortgageApplication/README.md) diff --git a/build-conf/README.md b/build-conf/README.md index e48c97e0..aa82a0e8 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -288,3 +288,10 @@ transfer_jclPDS | Sample dataset for JCL members transfer_xmlPDS | Sample dataset for xml members transfer_srcOptions | BPXWDYN creation options for creating 'source' type data sets transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets + +### language-conf/languageConfigProps01.properties +Sample language configuration properties file used by dbb-zappbuild/utilities/BuildUtilities.groovy. + +This is a custom properties file to override file properties for a group of files, based on mapping defined in `zAppBuild/samples/application-conf/languageConfigurationMapping.properties`. Multiple language configuration properties files can be defined and mapped against different file groups in `zAppBuild/samples/application-conf/languageConfigurationMapping.properties`. + +Note: The name of this property file need not be `languageConfigProps01.properties`. Any name can be given to this property file and can use the same name while mapping source file in `zAppBuild/samples/application-conf/languageConfigurationMapping.properties`. diff --git a/build-conf/language-conf/languageConfigProps01.properties b/build-conf/language-conf/languageConfigProps01.properties new file mode 100644 index 00000000..d979505f --- /dev/null +++ b/build-conf/language-conf/languageConfigProps01.properties @@ -0,0 +1,11 @@ +# Sample of a language configuration properties file override for group languageConfigProps01 + +# For details, please see the file property management documentation at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md + +# +# Sample to merge properties of this file level overwrite with the default setting for cobol_linkEditParms +# A more exclusive overwrite would be cobol_linkEditParms=MAP +cobol_linkEditParms=${cobol_linkEditParms},MAP + +# Add custom language configuration properties below if any diff --git a/build-conf/language-conf/languageConfigProps02.properties b/build-conf/language-conf/languageConfigProps02.properties new file mode 100644 index 00000000..6315d436 --- /dev/null +++ b/build-conf/language-conf/languageConfigProps02.properties @@ -0,0 +1,15 @@ +# Sample of a language configuration properties file override for group languageConfigProps02 + +# For details, please see the file property management documentation at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md + +# +# Sample to merge properties of this file level overwrite with the default setting for cobol_compileParms +# A more exclusive overwrite would be cobol_compileParms=SOURCE +cobol_compileParms=${cobol_compileParms},SOURCE + +# +# Defines exclusively the compile options for this file +#cobol_compilerVersion=V4 + +# Add custom language configuration properties below if any \ No newline at end of file diff --git a/build.groovy b/build.groovy index c2d079da..91fea08e 100644 --- a/build.groovy +++ b/build.groovy @@ -17,6 +17,7 @@ import groovy.cli.commons.* @Field def gitUtils= loadScript(new File("utilities/GitUtilities.groovy")) @Field def buildUtils= loadScript(new File("utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("utilities/ImpactUtilities.groovy")) +@Field def filePropUtils= loadScript(new File("utilities/FilePropUtilities.groovy")) @Field String hashPrefix = ':githash:' @Field String giturlPrefix = ':giturl:' @Field String gitchangedfilesPrefix = ':gitchangedfiles:' @@ -583,9 +584,12 @@ def createBuildList() { } // Loading file/member level properties from member specific properties files - if (props.filePropertyValueKeySet().getAt("loadFileLevelProperties") || props.loadFileLevelProperties) { - println "** Populating file level properties from individual property files." - buildUtils.loadFileLevelPropertiesFromFile(buildList) + if (props.filePropertyValueKeySet().getAt("loadFileLevelProperties") + || props.filePropertyValueKeySet().getAt("loadLanguageConfigurationProperties") + || (props.loadFileLevelProperties && props.loadFileLevelProperties.toBoolean()) + || (props.loadLanguageConfigurationProperties && props.loadLanguageConfigurationProperties.toBoolean())) { + println "** Populating file level properties from individual artifact properties files." + filePropUtils.loadFileLevelPropertiesFromFile(buildList) } // Perform analysis and build report of external impacts diff --git a/docs/FilePropertyManagement.md b/docs/FilePropertyManagement.md new file mode 100644 index 00000000..1d788855 --- /dev/null +++ b/docs/FilePropertyManagement.md @@ -0,0 +1,174 @@ +# File Property Management + +## Table of contents + +- [Introduction](#introduction) +- [Overriding build properties with DBB and zAppBuild](#overriding-build-properties-with-dbb-and-zappbuild) +- [Default properties](#default-properties) +- [Overriding properties](#overriding-properties) + - [DBB file properties](#dbb-file-properties) + - [Individual artifact properties file](#individual-artifact-properties-file) + - [Language configuration mapping](#language-configuration-mapping) + +## Introduction + +This document explains how to define compiler and other options when building a program or subset of programs with zAppBuild. Building mainframe application programs requires configuring various parameters and options for the different build steps, such as the pre-compile, compile, or link-edit step. For example, an application can contain COBOL programs that need to be link edited with the option `NCAL` or without the option `NCAL` for different purposes. An override may be required for any build parameter for the various build steps like compile parameters, bind parameters, link edit parameters, and so on. + +In existing mainframe toolchains, this customization is performed by assigning a type to the application artifact[^1]. This *type* is often used to specify the build parameters and options for an entire **subgroup** of application artifacts. Additionally, it allows that an application program might have some individual overrides as well. + +Generally, each build parameter for an application artifact will either have a default value or an override of the default value. + +## Overriding build properties with DBB and zAppBuild + +Dependency Based Build comes with its own [APIs](https://www.ibm.com/docs/api/v1/content/SS6T76_2.0.0/javadoc/index.html) to manage build properties, which extend the standard key-value pair strategy of *java.util.Properties*. DBB refers to the term *File properties* to allow overriding the corresponding default build properties using the DBB file property path syntax. See [IBM DBB Docs - Build properties](https://www.ibm.com/docs/en/dbb/latest?topic=apis-build-properties#file-properties) for more details about this syntax. + +zAppBuild supports overriding the majority of build properties defined within its framework. The full list can be viewed at [application-conf/README.md](../samples/application-conf/README.md). + +zAppBuild leverages DBB's API and allows you to define build parameters on three different levels for each language script: + +1. General defaults in the corresponding language properties files: For example, you can define the compile options for building COBOL programs in [application-conf/Cobol.properties](../samples/application-conf/Cobol.properties). Property keys make use of a language prefix; for instance, COBOL property keys are prefixed with `cobol_`. +2. A group definition that overrides the general default in one of two ways: + - By using DBB's file property syntax in [application-conf/file.properties](../samples/application-conf/file.properties), and specifying the application artifact group via a pattern filter on the path name(s) + - By mapping to a language configuration file to override the defaults, such as in [application-conf/languageConfigurationMapping.properties](../samples/MortgageApplication/application-conf/languageConfigurationMapping.properties) +3. An individual file-level definition that overwrites the general default in one of two ways: + - By using DBB's file properties syntax in [application-conf/file.properties](../samples/application-conf/file.properties), and specifying the application artifact's path as the file path pattern + - By specifying multiple parameters for a specific file using the individual artifact properties file. For example: [epsmlist.cbl.properties](../samples/MortgageApplication/properties/epsmlist.cbl.properties). + +zAppBuild comes with various build property strategies that can be combined via an order of precedence. The following table summarizes the strategies for overriding file properties from highest to lowest precedence: + +|Precedence|Strategy|Use case|Implementation| +|-|-|-|-| +|1.|Individual artifact properties file|Override one or multiple build parameters for individual files|DBB file property defining an override for the specific file[^2]| +|2.|Language configuration mapping|Override and define one or multiple build parameters for a group of mapped application artifacts|DBB file property defining an override for the specific file(s)[^2]| +|3.|DBB file properties|Override a single build parameter for individual files or a grouped list of application artifacts|DBB file property defining an override for the specific[^2] or grouped list[^3] of application artifacts| +|4.|Default properties|General build properties used when no overrides are defined|Build property defining the default value for all files| + +To understand the order of precedence, think of this as a merge of the property configurations. For example, if both an individual artifact properties file and a language configuration mapping are configured for a file, then the properties defined through the individual artifact properties file take precedence, but are also merged with other properties defined by the language configuration mapping and the default properties. + +The following sections explain these build property strategies in more detail. + +## Default properties + +Default properties can be set in the corresponding language properties file. For example, the COBOL file properties can be set in [application-conf/Cobol.properties](../samples/application-conf/Cobol.properties), while the Assembler file properties can be set in [application-conf/Assembler.properties](../samples/application-conf/Assembler.properties), and so on. + +By default, zAppBuild applies the properties stored in the [application-conf](../samples/application-conf) folder of the application repository, which allows the application team to have control over these files. These property definitions in [application-conf](../samples/application-conf) take precedence over corresponding properties defined in the [build-conf](../build-conf/) folder. If you are looking for a more centralized way to manage the default options for all applications, you can move the relevant property definitions into the zAppBuild build framework itself by taking them out of the [application-conf](../samples/application-conf) folder, and then only storing them in one of the following locations: + +- In the appropriate language properties file(s) under [build-conf](../build-conf/) +- In a separate directory within zAppBuild itself, while using the applicationConfRootDir property in [build-conf/build.properties](../build-conf/build.properties) + +## Overriding properties + +The following section describes the various strategies to override default build property values. The DBB file property syntax is the most commonly used approach within the zAppBuild samples. Two alternate approaches to override build properties are implemented in zAppBuild to serve the different use cases, and can be used to simplify the adoption of zAppBuild by either leveraging an individual artifact properties file per application artifact, or by defining a language configuration mapping. + +### DBB file properties + +The most common way to override a single build parameter makes use of the [DBB file property syntax](https://www.ibm.com/docs/en/dbb/latest?topic=apis-build-properties#file-properties). This strategy can be applied to individual artifacts or groups of artifacts. + +For example, to use a DBB file property to override the default COBOL compile parameters for build artifact `MortgageApplication/cobol/epsnbrvl.cbl`, follow the DBB file property syntax in `file.properties`, which is the default property file for configuring file property overrides: + +```properties +cobol_compileParms=LIB,SOURCE :: **/cobol/epsnbrvl.cbl +``` + +For merging the property values of this file-level override with the default COBOL compile parameters (rather than just overriding them), you can specify the following syntax: + +```properties +cobol_compileParms=${cobol_compileParms},SOURCE :: **/cobol/epsnbrvl.cbl +``` + +In order to override the build parameter for a group of files, you can use wildcards when specifying the file path pattern in DBB's file property path syntax. For example, let's assume that you are storing all CICS modules in the `cobol_cics` subfolder. Using the following sample will ensure that the file flag `isCICS` is set to `true` for all COBOL files in this subfolder with the file extension `*.cbl`. + +```properties +isCICS = true :: **/cobol_cics/*.cbl +``` + +- **Note:** We do not recommend organizing the layout of repository folders/files based on build property management, because future updates to the build information of a file could require it to be moved into different folders. Instead, we recommend that the repository's folder layout represent the functional context of the files. + +The MortgageApplication sample contains a good example of how the DBB file property can be used. Typically, these overrides are defined in [application-conf/file.properties](../samples/MortgageApplication/application-conf/file.properties). + +### Individual artifact properties file + +The approach of using the DBB file property syntax might become cumbersome if you want to manage multiple property overrides for a given application artifact. + +The "individual artifact properties file" strategy changes the way of specifying the properties by allowing you to manage multiple property overrides together within an individual properties file for a given build artifact. It centers around the build artifact rather than the build property. + +The functionality to load properties from an individual artifact properties file can be activated by setting the configuration property `loadFileLevelProperties` in the `application-conf/application.properties` file to `true`. To enable this feature for a specific artifact or a subset of application artifacts, use the DBB file property syntax in `application-conf/file.properties` to set `loadFileLevelProperties` to `true`. The following snippet from a sample `application-conf/file.properties` file configures zAppBuild to look for an individual artifact properties file for all the programs starting with `eps` and `lga`: + +```properties +loadFileLevelProperties = true :: **/cobol/eps*.cbl, **/cobol/lga*.cbl` +``` + +Individual artifact properties files are resolved using the pattern `/.`. The `propertyFilePath` and `propertyFileExtension` can be customized in [application-conf/application.properties](../samples/MortgageApplication/application-conf/application.properties). For example, for the source file `epsmlist.cbl`, the process searches for an individual artifact properties file in the defined `propertyFilePath` directory. If no corresponding properties file is found, the build will use the default build values or, if any file properties were defined using the DBB file property path syntax or an alternate approach, then the build will use those. + +Once the `loadFileLevelProperties` property functionality is enabled, create a properties file for each application artifact for which individual artifact properties need to be defined. For example, to override build parameters for the file `epsmlist.cbl`, create the properties file `epsmlist.cbl.properties` in the defined `propertyFilePath` folder. The name of the properties file needs to have the entire source file name including the extension; hence, the properties file for `epsmlist.cbl` needs to be named `epsmlist.cbl.properties`. + +The individual artifact properties file allows you to define multiple build properties using the standard property syntax. For instance, in `epsmlist.cbl.properties`, you can define the following properties: + +```properties +cobol_compileParms=LIB,SOURCE +isCICS = true +``` + +With the above configuration, zAppBuild will import these properties and set them as DBB file properties. + +You can view a sample individual artifact properties file, [epsmlist.cbl.properties](../samples/MortgageApplication/properties/epsmlist.cbl.properties), within the MortgageApplication sample. + +**Note:** Overrides for a given build property should be managed either via the DBB file property path syntax or in the individual artifact properties files, but not both at the same time (as this can cause unpredictable behavior). The following example shows how both approaches for defining file properties can be combined to specify a set of build properties for the same source file: + +- Example using the DBB file property path syntax and an individual artifact properties file to define build properties for a source file named `app/cobol/AB123456.cbl`: + - You can use the DBB file property path syntax to define a file property for a group of files. The below defines the `deployType` for all source files in the folder cobol beginning with `AB*` to be `BATCHLOAD`: + + ```properties + cobol_deployType = BATCHLOAD :: **/cobol/AB*.cbl + ``` + + - At the same time, you can define an individual artifact properties file for `app/cobol/AB123456.cbl` with the following *different* build property: + + ```properties + cobol_compileParms = LIB,SOURCE + ``` + + - During the build, the file `app/cobol/AB123456.cbl` will have the `deployType` `BATCHLOAD` and the COBOL compile parameters `LIB` and `SOURCE`. + +### Language configuration mapping + +An alternative way to define build properties for a **subgroup of files** is by leveraging a mapping approach. Rather than specifying individual parameters or properties for an individual application artifact, the application artifacts are mapped to a language configuration, which can then define multiple build parameters in a central language configuration properties file. All mapped application artifacts will inherit those defined build parameters. + +This approach consists of: + +- Language configuration mapping file: A mapping of the application artifact(s) to a language configuration file +- Language configuration properties file(s): For each language configuration, a properties file defining that language's build parameters + +The "language configuration mapping" approach can be enabled by setting the property `loadLanguageConfigurationProperties` in the `application-conf/application.properties` file to `true`. To enable this option for a specific file or a set of files, use the DBB file property syntax and set `loadLanguageConfigurationProperties` to `true` in the `application-conf/file.properties` file. Below is a sample to enable language configuration mapping for all programs starting with `eps` and `lga` via the `application-conf/file.properties` file: + +```properties +loadLanguageConfigurationProperties = true :: **/cobol/eps*.cbl, **/cobol/lga*.cbl +``` + +You can specify build properties for a language in a language configuration properties file, which should be created in the `build-conf/language-conf` folder. zAppBuild will import these properties from the language configuration properties file and set them as DBB file properties for the mapped artifacts. You can implement multiple language configurations to serve different variations or types by creating multiple language configuration properties files under the `build-conf/language-conf` folder. A sample language configuration properties file can be found at [languageConfigProps01.properties](../build-conf/language-conf/languageConfigProps01.properties). + +A language configuration properties file allows you to centrally specify build properties for the group of mapped application artifacts. All mapped files will inherit those build properties. However, in the case of combining the language configuration mapping with an individual artifact properties file override, for any build property that is defined in both places, the property definition in the individual artifact properties file will take precedence and be applied. Properties that are not specified in the individual artifact properties file will be defined by lower precedence strategies - that is, from the language configuration mapping if defined there, or if not, then from the default properties. + +In the following sample language configuration properties file, the properties defined in this snippet are overriding the default COBOL compile parameters (`cobol_compileParms`), the file flag `isCICS`, and the linkEdit statement (`cobol_linkEditStream`): + +```properties +cobol_compileParms=LIB,SOURCE +isCICS = true +cobol_linkEditStream= INCLUDE OBJECT(@{member})\n INCLUDE SYSLIB(CUSTOBJ) +``` + +To map files to a language configuration, create a `languageConfigurationMapping.properties` file in the `application-conf` folder of your application repository. Then, within this new language configuration mapping file, map each artifact to its corresponding language configuration using the syntax `=`. + +- For example, the following snippet in [application-conf/languageConfigurationMapping.properties](../samples/MortgageApplication/application-conf/languageConfigurationMapping.properties) maps both source files `epsnbrvl.cbl` and `epsmlist.cbl` to use the properties defined in `build-conf/language-conf/languageConfigProps01.properties`, while the source file `epscmort.cbl` is mapped to use the properties defined in `build-conf/language-conf/languageConfigProps02.properties` for language configuration mapping overrides: + + ```properties + epsnbrvl.cbl=languageConfigProps01 + epsmlist.cbl=languageConfigProps01 + epscmort.cbl=languageConfigProps02 + ``` + +See [languageConfigurationMapping.properties](../samples/MortgageApplication/application-conf/languageConfigurationMapping.properties) for a sample language configuration mapping file. + +[^1]: The term "artifact" and "file" in this document refer to program source code that will built (as opposed to JCL or other non-buildable items), for example by DBB. +[^2]: DBB is managing the DBB file properties in its separate internal table compared to the default properties. This table leverages the combination of [property name + file pattern] as the key of the internal table. When the same key is declared a second time, it overrides the first one. +[^3]: Because of managing DBB file properties is done in a single table, you can experience unpredictable behaviour when mixing qualified file path pattern definitions and file path patterns containing wildcards for the same property name. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..8ef04565 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,7 @@ +# zAppBuild Docs + +This folder contains supplemental documentation to help explain the implementation and usage of zAppBuild features. + +|File|Description| +|-|-| +|FilePropertyManagement|Documentation on managing default and overriding build properties for specific application files| diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index f0b9a470..5d7cea99 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -294,6 +294,8 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (ssi != null) parms = parms + ",SSI=$ssi" } + if (props.verbose) println "Link-Edit parms for $buildFile = $parms" + // define the MVSExec command to link edit the program MVSExec linkedit = new MVSExec().file(buildFile).pgm(linker).parm(parms) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 990b9c3b..25a04301 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -84,6 +84,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb String ssi = buildUtils.getShortGitHash(buildFile) if (ssi != null) parms = parms + ",SSI=$ssi" } + + if (props.verbose) println "Link-Edit parms for $buildFile = $parms" + // define the MVSExec command to link edit the program MVSExec linkedit = new MVSExec().file(buildFile).pgm(linker).parm(parms) diff --git a/samples/MortgageApplication/application-conf/README.md b/samples/MortgageApplication/application-conf/README.md index f60e8962..9475d1f1 100644 --- a/samples/MortgageApplication/application-conf/README.md +++ b/samples/MortgageApplication/application-conf/README.md @@ -91,3 +91,11 @@ linkedit_deployTypeDLI | deployType for build output for build files with isDLI= linkedit_scanLoadModule | Flag indicating to scan the load module for link dependencies and store in the application's outputs collection. | true linkEdit_linkEditSyslibConcatenation | A comma-separated list of libraries to be concatenated in syslib during linkEdit step | true +### languageConfigurationMapping.properties +Sample Language Configuration mapping properties used by dbb-zappbuild/utilities/BuildUtilities.groovy. + +This contains the mapping of the files and their corresponding Language Configuration properties files residing in `zAppBuild/build-conf/language-conf` to override the default file properties. + +Example: The entry - `epsnbrvl.cbl=languageConfigProps01`, means the file properties of file `epsnbrvl.cbl` will be overridden by the properties mentioned in `dbb-zappbuild/build-conf/language-conf/languageConfigProps01.properties` + +See the [language configuration mapping documentation](../../../docs/FilePropertyManagement.md#language-configuration-mapping) for more details on how to enable and use language configuration mapping. diff --git a/samples/MortgageApplication/application-conf/application.properties b/samples/MortgageApplication/application-conf/application.properties index 69fd9203..bd2500ee 100644 --- a/samples/MortgageApplication/application-conf/application.properties +++ b/samples/MortgageApplication/application-conf/application.properties @@ -3,7 +3,7 @@ # # Comma separated list of additional application property files to load # Supports both relative path (to ${application}/application-conf/) and absolute path -applicationPropFiles=file.properties,BMS.properties,Cobol.properties,LinkEdit.properties +applicationPropFiles=file.properties,BMS.properties,Cobol.properties,LinkEdit.properties,languageConfigurationMapping.properties # # Comma separated list all source directories included in application build. Supports both absolute @@ -39,58 +39,31 @@ skipImpactCalculationList= ############################################################### # Build Property management ############################################################### -# # zAppBuild allows you to manage default properties and file properties: -# -# - Default build properties are defined in the /build-conf and /application-conf property files (e.g. Cobol.properties) -# -# - File properties override corresponding default build properties, and are defined through one of two methods: -# - Overwrites for groups or individual files -# - Typically defined in file.properties using the DBB file property path syntax -# - See: https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-organize-build-script#file-properties -# - Overwrites for an individual file -# - Defined in an individual property file located in a configurable subfolder (e.g. properties/epsmlist.cbl.properties) -# -# A typical scenario for using zAppBuild's capability to set build properties for an individual source file via a corresponding -# individual property file is to overwrite compile or link options. This approach might help ease the migration of properties -# from the previous build system. -# -# Individual property files are resolved using the pattern /.. For example, -# for the source file epsmlist.cbl, the process searches for a file in the propertyFilePath directory -# with the name epsmlist.cbl.properties. -# If no corresponding property file is found, the build will use the default build values. (Or, if any file properties were defined -# using the DBB file property path syntax, then the build will use those.) -# -# Note: Overwrites for a specific build property should be managed either via the file property path syntax or -# in the individual property files, but not both. The following example shows how both approaches for defining -# file properties can be combined to specify a set of build properties for the same source file: -# -# ### Example: Using the file property path syntax and individual property files to define build properties for a source file named app/cobol/AB123456.cbl -# - You can use the file property path syntax to define a file property for a group of files. The below defines the deployType for -# all source files in the folder cobol beginning with AB* to be BATCHLOAD: -# -# cobol_deployType = BATCHLOAD :: **/cobol/AB*.cbl -# -# - At the same time, you can define an individual file property file for app/cobol/AB123456.cbl with the following build property: -# -# cobol_compileParms = LIB,SOURCE -# -# - During the build, the file app/cobol/AB123456.cbl will have the deployType BATCHLOAD and the COBOL compile parameters LIB and SOURCE. -# ### End example ### +# - Documentation on how to override corresponding default build properties can be found at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md -# ### Properties to enable and configure build property overwrites using individual property files +# ### Properties to enable and configure build property overrides using individual artifact properties files -# flag to enable the zAppBuild capability to load individual property files for individual source files, -# Note: To activate the loading of property files for a group of files, it is recommended to use the file property path -# syntax instead. (i.e. loadFileLevelProperties=true :: **/cobol/*.cbl to enable it for all cobol files) +# flag to enable the zAppBuild capability to load individual artifact properties files for all individual source files. +# Note: To only activate loadFileLevelProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. # default: false loadFileLevelProperties=false -# relative path to folder containing individual property files +# Property to enable/disable and configure build property overrides using language configuration mapping +# file - languageConfigurationMapping.properties +# If loadFileLevelProperties is set as true above, the properties from the individual artifact properties files will override the +# properties from language configuration properties file. +# Note: To only activate loadLanguageConfigurationProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. +loadLanguageConfigurationProperties=false + +# relative path to folder containing individual artifact properties files # assumed to be relative to ${workspace}/${application} propertyFilePath=properties -# file extension for individual property files +# file extension for individual artifact properties files # default: properties propertyFileExtension=properties diff --git a/samples/MortgageApplication/application-conf/file.properties b/samples/MortgageApplication/application-conf/file.properties index cc65f78d..a282b358 100644 --- a/samples/MortgageApplication/application-conf/file.properties +++ b/samples/MortgageApplication/application-conf/file.properties @@ -24,7 +24,20 @@ isCICS = true :: **/cobol/epsmlist.cbl, **/link/epsmlist.lnk, **/cobol/epscsmrt. # # Property File Management # -# Flag to enable the zAppBuild capability to load individual property files for source files, -# This file path pattern looks for an individual property file for epsmlist.cbl. -# Please see detailed description in application.properties -loadFileLevelProperties = true :: **/cobol/epsmlist.cbl \ No newline at end of file +# Flag to enable the zAppBuild capability to load individual artifact properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable individual artifact properties file for all COBOL files: loadFileLevelProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for an individual artifact properties file for epsmlist.cbl: +loadFileLevelProperties = true :: **/cobol/epsmlist.cbl + +# +# Language Configuration properties Management +# +# Flag to enable the zAppBuild capability to load language configuration properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable language configuration mapping for all COBOL files: loadLanguageConfigurationProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for a language configuration properties file mapping for epsnbrvl.cbl and epsmpmt.cbl +# in the application-conf/languageconfigurationMapping.properties file: +loadLanguageConfigurationProperties = true :: **/cobol/epsnbrvl.cbl, **/cobol/epsmpmt.cbl \ No newline at end of file diff --git a/samples/MortgageApplication/application-conf/languageConfigurationMapping.properties b/samples/MortgageApplication/application-conf/languageConfigurationMapping.properties new file mode 100644 index 00000000..602b348c --- /dev/null +++ b/samples/MortgageApplication/application-conf/languageConfigurationMapping.properties @@ -0,0 +1,11 @@ +# This file maps the program file with the corresponding language configuration properties file in the build-conf/language-conf folder. + +# For example: As shown below, the programs `epsnbrvl.cbl` and `epsmpmt.cbl` will map to the language configuration file +# build-conf/language-conf/languageConfigProps01.properties. +# The program `epscmort.cbl` will map to the language configuration file build-conf/language-conf/languageConfigProps02.properties. + +epsnbrvl.cbl=languageConfigProps01 +epsmpmt.cbl=languageConfigProps01 +epscmort.cbl=languageConfigProps02 + +# Insert all the file ==> language configuration properties mapping here! \ No newline at end of file diff --git a/samples/MortgageApplication/properties/epsmlist.cbl.properties b/samples/MortgageApplication/properties/epsmlist.cbl.properties index a2809069..69f84b7a 100644 --- a/samples/MortgageApplication/properties/epsmlist.cbl.properties +++ b/samples/MortgageApplication/properties/epsmlist.cbl.properties @@ -1,6 +1,7 @@ -# Sample of an individual property files overwrite for file cobol/epsmlist.cbl +# Sample of an individual artifact properties file override for source file cobol/epsmlist.cbl -# For details, check out section in application.properties +# For details, please see the file property management documentation at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md # # Sample to merge properties of this file level overwrite with the default setting for cobol_compileParms @@ -9,4 +10,5 @@ cobol_compileParms=${cobol_compileParms},SOURCE # # Defines exclusively the compile options for this file -#cobol_compilerVersion=V4 \ No newline at end of file +#cobol_compilerVersion=V4 + diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index f50963ab..5232fcfb 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -24,16 +24,17 @@ excludeFileList | Files to exclude when scanning or running full build. | false skipImpactCalculationList | Files for which the impact analysis should be skipped in impact build | false jobCard | JOBCARD for JCL execs | false **Build Property management** | | -loadFileLevelProperties | Flag to enable the zAppBuild capability to load individual property files for a build file | true -propertyFilePath | relative path to folder containing individual property files | true -propertyFileExtension | file extension for individual property files | true +loadFileLevelProperties | Flag to enable the zAppBuild capability to load individual artifact properties files for a build file | true +loadLanguageConfigurationProperties | Flag to enable the zAppBuild capability to load language configuration properties for build files mapped in languageConfigurationMapping.properties | true +propertyFilePath | relative path to folder containing individual artifact properties files | true +propertyFileExtension | file extension for individual artifact properties files | true **Dependency and Impact resolution configuration** || resolveSubsystems | boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile | false impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. ** deprecated ** Please consider moving to new SearchPathDepedencyAPI leveraging `impactSearch` configuration. | true, recommended in file.properties impactSearch | Impact finder resolution search configuration leveraging the SearchPathImpactFinder API. Sample configurations are inlcuded below, next to the previous rule definitions. | true ### file.properties -Location of file properties, script mappings and file-level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for the column 'Overridable' in the tables below for build properties that can have file-level property overrides. Additional file-level properties can be defined through individual property files in a separate directory of the repository. For more details, see the section _Build Property Management_ in `application.properties` +Location of file properties, script mappings and file-level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for the column 'Overridable' in the tables below for build properties that can have file-level property overrides. Additional file-level properties can be defined through individual artifact properties files in a separate directory of the repository. For more details, see [File Property Management](https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md). Property | Description --- | --- @@ -281,3 +282,12 @@ Application properties used by zAppBuild/language/Transfer.groovy Property | Description | Overridable --- | --- | --- transfer_deployType | deployType | true + +### languageConfigurationMapping.properties +Sample language configuration mapping properties used by dbb-zappbuild/utilities/BuildUtilities.groovy. + +This contains the mapping of the files and their corresponding Language Configuration properties files residing in `zAppBuild/build-conf/language-conf` to override the default file properties. + +Example: The entry - `epsnbrvl.cbl=languageConfigProps01`, means the file properties of file `epsnbrvl.cbl` will be overridden by the properties mentioned in `zAppBuild/build-conf/language-conf/languageConfigProps01.properties` + +See the [language configuration mapping documentation](../../docs/FilePropertyManagement.md#language-configuration-mapping) for more details on how to enable and use language configuration mapping. diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index fa21c760..cd1e0e31 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -8,7 +8,7 @@ # # Comma separated list of additional application property files to load # Supports both relative path (to ${application}/application-conf/) and absolute path -applicationPropFiles=file.properties,bind.properties,Assembler.properties,BMS.properties,Cobol.properties,LinkEdit.properties,bind.properties,PLI.properties,MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,reports.properties +applicationPropFiles=file.properties,bind.properties,Assembler.properties,BMS.properties,Cobol.properties,LinkEdit.properties,bind.properties,PLI.properties,MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,reports.properties,languageConfigurationMapping.properties # # Comma separated list all source directories included in application build. Supports both absolute @@ -58,58 +58,31 @@ jobCard= ############################################################### # Build Property management ############################################################### -# # zAppBuild allows you to manage default properties and file properties: -# -# - Default build properties are defined in the /build-conf and /application-conf property files (e.g. Cobol.properties) -# -# - File properties override corresponding default build properties, and are defined through one of two methods: -# - Overwrites for groups or individual files -# - Typically defined in file.properties using the DBB file property path syntax -# - See: https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-organize-build-script#file-properties -# - Overwrites for an individual file -# - Defined in an individual property file located in a configurable subfolder (e.g. properties/epsmlist.cbl.properties) -# -# A typical scenario for using zAppBuild's capability to set build properties for an individual source file via a corresponding -# individual property file is to overwrite compile or link options. This approach might help ease the migration of properties -# from the previous build system. -# -# Individual property files are resolved using the pattern /.. For example, -# for the source file epsmlist.cbl, the process searches for a file in the propertyFilePath directory -# with the name epsmlist.cbl.properties. -# If no corresponding property file is found, the build will use the default build values. (Or, if any file properties were defined -# using the DBB file property path syntax, then the build will use those.) -# -# Note: Overwrites for a specific build property should be managed either via the file property path syntax or -# in the individual property files, but not both. The following example shows how both approaches for defining -# file properties can be combined to specify a set of build properties for the same source file: -# -# ### Example: Using the file property path syntax and individual property files to define build properties for a source file named app/cobol/AB123456.cbl -# - You can use the file property path syntax to define a file property for a group of files. The below defines the deployType for -# all source files in the folder cobol beginning with AB* to be BATCHLOAD: -# -# cobol_deployType = BATCHLOAD :: **/cobol/AB*.cbl -# -# - At the same time, you can define an individual file property file for app/cobol/AB123456.cbl with the following build property: -# -# cobol_compileParms = LIB,SOURCE -# -# - During the build, the file app/cobol/AB123456.cbl will have the deployType BATCHLOAD and the COBOL compile parameters LIB and SOURCE. -# ### End example ### +# - Documentation on how to override corresponding default build properties can be found at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md -# ### Properties to enable and configure build property overwrites using individual property files +# ### Properties to enable and configure build property overrides using individual artifact properties files -# flag to enable the zAppBuild capability to load individual property files for individual source files, -# Note: To activate the loading of property files for a group of files, it is recommended to use the file property path -# syntax instead. (i.e. loadFileLevelProperties=true :: **/cobol/*.cbl to enable it for all cobol files) +# flag to enable the zAppBuild capability to load individual artifact properties files for all individual source files. +# Note: To only activate loadFileLevelProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. # default: false loadFileLevelProperties=false -# relative path to folder containing individual property files +# Property to enable/disable and configure build property overrides using language configuration mapping +# file - languageConfigurationMapping.properties +# If loadFileLevelProperties is set as true above, the properties from the individual artifact properties files will override the +# properties from language configuration properties file. +# Note: To only activate loadLanguageConfigurationProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. +loadLanguageConfigurationProperties=false + +# relative path to folder containing individual artifact properties files # assumed to be relative to ${workspace}/${application} propertyFilePath=properties -# file extension for individual property files +# file extension for individual artifact properties files # default: properties propertyFileExtension=properties diff --git a/samples/application-conf/languageConfigurationMapping.properties b/samples/application-conf/languageConfigurationMapping.properties new file mode 100644 index 00000000..602b348c --- /dev/null +++ b/samples/application-conf/languageConfigurationMapping.properties @@ -0,0 +1,11 @@ +# This file maps the program file with the corresponding language configuration properties file in the build-conf/language-conf folder. + +# For example: As shown below, the programs `epsnbrvl.cbl` and `epsmpmt.cbl` will map to the language configuration file +# build-conf/language-conf/languageConfigProps01.properties. +# The program `epscmort.cbl` will map to the language configuration file build-conf/language-conf/languageConfigProps02.properties. + +epsnbrvl.cbl=languageConfigProps01 +epsmpmt.cbl=languageConfigProps01 +epscmort.cbl=languageConfigProps02 + +# Insert all the file ==> language configuration properties mapping here! \ No newline at end of file diff --git a/test/applications/MortgageApplication/application-conf/file_languageConfig_TC1.properties b/test/applications/MortgageApplication/application-conf/file_languageConfig_TC1.properties new file mode 100644 index 00000000..a282b358 --- /dev/null +++ b/test/applications/MortgageApplication/application-conf/file_languageConfig_TC1.properties @@ -0,0 +1,43 @@ +# Application script mappings and file property overrides + +# +# Script mappings for all application programs +dbb.scriptMapping = Assembler.groovy :: **/*.asm +dbb.scriptMapping = BMS.groovy :: **/*.bms +dbb.scriptMapping = Cobol.groovy :: **/*.cbl +dbb.scriptMapping = LinkEdit.groovy :: **/*.lnk +dbb.scriptMapping = PLI.groovy :: **/*.pli + +# +# Need to build epsnbrvl.cbl first during cobol builds +cobol_fileBuildRank = 1 :: **/cobol/epsnbrvl.cbl + +# +# Skip creating a load module for these programs as they will be statically linked to other programs +cobol_linkEdit = false :: **/cobol/epsnbrvl.cbl, **/cobol/epsmlist.cbl + +# +# epsmlist needs to compile as CICS but does not have EXEC CICS statements +# so is not automatically flagged as CICS by dependency scanner +isCICS = true :: **/cobol/epsmlist.cbl, **/link/epsmlist.lnk, **/cobol/epscsmrt.cbl + +# +# Property File Management +# +# Flag to enable the zAppBuild capability to load individual artifact properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable individual artifact properties file for all COBOL files: loadFileLevelProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for an individual artifact properties file for epsmlist.cbl: +loadFileLevelProperties = true :: **/cobol/epsmlist.cbl + +# +# Language Configuration properties Management +# +# Flag to enable the zAppBuild capability to load language configuration properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable language configuration mapping for all COBOL files: loadLanguageConfigurationProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for a language configuration properties file mapping for epsnbrvl.cbl and epsmpmt.cbl +# in the application-conf/languageconfigurationMapping.properties file: +loadLanguageConfigurationProperties = true :: **/cobol/epsnbrvl.cbl, **/cobol/epsmpmt.cbl \ No newline at end of file diff --git a/test/applications/MortgageApplication/application-conf/file_languageConfig_TC2.properties b/test/applications/MortgageApplication/application-conf/file_languageConfig_TC2.properties new file mode 100644 index 00000000..5d8f834b --- /dev/null +++ b/test/applications/MortgageApplication/application-conf/file_languageConfig_TC2.properties @@ -0,0 +1,43 @@ +# Application script mappings and file property overrides + +# +# Script mappings for all application programs +dbb.scriptMapping = Assembler.groovy :: **/*.asm +dbb.scriptMapping = BMS.groovy :: **/*.bms +dbb.scriptMapping = Cobol.groovy :: **/*.cbl +dbb.scriptMapping = LinkEdit.groovy :: **/*.lnk +dbb.scriptMapping = PLI.groovy :: **/*.pli + +# +# Need to build epsnbrvl.cbl first during cobol builds +cobol_fileBuildRank = 1 :: **/cobol/epsnbrv*.cbl + +# +# Skip creating a load module for these programs as they will be statically linked to other programs +cobol_linkEdit = false :: **/cobol/epsnbrvl.cbl, **/cobol/epsmlist.cbl + +# +# epsmlist needs to compile as CICS but does not have EXEC CICS statements +# so is not automatically flagged as CICS by dependency scanner +isCICS = true :: **/cobol/epsmlist.cbl, **/link/epsmlist.lnk, **/cobol/epscsmrt.cbl + +# +# Property File Management +# +# Flag to enable the zAppBuild capability to load individual artifact properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable individual artifact properties file for all COBOL files: loadFileLevelProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for an individual artifact properties file for epsmlist.cbl: +loadFileLevelProperties = true :: **/cobol/epsmlist.cbl + +# +# Language Configuration properties Management +# +# Flag to enable the zAppBuild capability to load language configuration properties files for source files. +# Please see the detailed documentation on file property management at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md +# Example 1: Enable language configuration mapping for all COBOL files: loadLanguageConfigurationProperties=true :: **/cobol/*.cbl +# Example 2: The following file path pattern looks for a language configuration properties file mapping for epsnbrvl.cbl and epsmpmt.cbl +# in the application-conf/languageconfigurationMapping.properties file: +loadLanguageConfigurationProperties = true :: **/cobol/epsnbrvl.cbl, **/cobol/epsmpmt.cbl \ No newline at end of file diff --git a/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps01.properties b/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps01.properties new file mode 100644 index 00000000..e6ead582 --- /dev/null +++ b/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps01.properties @@ -0,0 +1,18 @@ +# Sample of a language configuration properties file override for group languageConfigProps01 + +# For details, please see the file property management documentation at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md + +# +# Sample to merge properties of this file level overwrite with the default setting for cobol_linkEditParms +# A more exclusive overwrite would be cobol_linkEditParms=MAP,LIST +cobol_linkEditParms=${cobol_linkEditParms},MAP + +# +# Update cobol_fileRank +# see dbb-zappbuild/test/testScripts/fullBuild_languageConfigurations.groovy +cobol_fileBuildRank = 2 + +# This is the test case being added for test script: +# dbb-zappbuild/test/testScripts/fullBuild_languageConfigurations.groovy +cobol_compileParms=${cobol_compileParms},SOURCE,MAP,LIST \ No newline at end of file diff --git a/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps02.properties b/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps02.properties new file mode 100644 index 00000000..6315d436 --- /dev/null +++ b/test/applications/MortgageApplication/build-conf/language-conf/languageConfigProps02.properties @@ -0,0 +1,15 @@ +# Sample of a language configuration properties file override for group languageConfigProps02 + +# For details, please see the file property management documentation at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md + +# +# Sample to merge properties of this file level overwrite with the default setting for cobol_compileParms +# A more exclusive overwrite would be cobol_compileParms=SOURCE +cobol_compileParms=${cobol_compileParms},SOURCE + +# +# Defines exclusively the compile options for this file +#cobol_compilerVersion=V4 + +# Add custom language configuration properties below if any \ No newline at end of file diff --git a/test/applications/MortgageApplication/test.properties b/test/applications/MortgageApplication/test.properties index 65b459c9..759e7d90 100644 --- a/test/applications/MortgageApplication/test.properties +++ b/test/applications/MortgageApplication/test.properties @@ -4,8 +4,17 @@ # # list of test scripts to run for this application # the order of the list matters -test_testOrder=resetBuild.groovy,mergeBuild.groovy,fullBuild.groovy,impactBuild.groovy,impactBuild_properties.groovy,impactBuild_renaming.groovy,impactBuild_deletion.groovy,resetBuild.groovy - +# impactBuild_renaming + impactBuild_deletion impact name and the number of files +# impactBuild_properties changes the build property definitions +test_testOrder=resetBuild.groovy,\ +mergeBuild.groovy,\ +fullBuild.groovy,\ +fullBuild_languageConfigurations.groovy,\ +impactBuild.groovy,\ +impactBuild_properties.groovy,\ +impactBuild_renaming.groovy,\ +impactBuild_deletion.groovy,\ +resetBuild.groovy ############################### # mergeBuild.groovy properties @@ -88,4 +97,52 @@ impactBuild_deletion_deleteFiles = cobol/epscmort.cbl # expected files deleted impactBuild_deletion_deletedOutputs = LOAD(EPSCMORT) :: cobol/epscmort.cbl # list of source datasets (LLQ) that should be deleted during impactBuild.groovy cleanUp -impactBuild_deletion_datasetsToCleanUp = BMS,COBOL,LINK,COPY,BMS.COPY,DBRM,LOAD,MFS,OBJ,TFORMAT \ No newline at end of file +impactBuild_deletion_datasetsToCleanUp = BMS,COBOL,LINK,COPY,BMS.COPY,DBRM,LOAD,MFS,OBJ,TFORMAT + +############################# +# fullBuild_languageConfigurations.groovy properties +# this test script is validating the language configuration capability of dbb-zappbuild +# it contains of +# - fullBuild - validate the Cobol compiler parms +# - userBuild TC1 - with a successful overwrite +# - userBuild TC2 - with a failing overwrite which is reported in the output +############################# +# +# fullBuild +# +# list of programs should be built for a full build for this application +fullBuild_languageConfigurations_expectedFilesBuilt_fullBuild = epsnbrvl.cbl,epscsmrt.cbl,epsmort.bms,epsmlist.lnk,epsmlis.bms,epsmlist.cbl,epsmpmt.cbl,epscmort.cbl,epscsmrd.cbl + +# updated configuration +fullBuild_languageConfigurations_updatedLanguageConfigs_fullBuild = build-conf/language-conf/languageConfigProps01.properties, build-conf/language-conf/languageConfigProps02.properties + +# +# Expected Compile options for COBOL source files printed to the console output +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epsnbrvl.cbl = LIB,SOURCE,MAP,LIST :: epsnbrvl.cbl +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epscsmrt.cbl = LIB,CICS :: epscsmrt.cbl +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epsmlist.cbl = LIB,SOURCE,CICS :: epsmlist.cbl +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epsmpmt.cbl = LIB,SOURCE,MAP,LIST :: epsmpmt.cbl +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epscsmrd.cbl = LIB,CICS :: epscsmrd.cbl +fullBuild_languageConfigurations_compileParms_fullBuild = Cobol compiler parms for MortgageApplication/cobol/epscmort.cbl = LIB,CICS,SQL :: epscmort.cbl + +# +# User Build sample file +userBuild_languageConfigurations_buildFile = MortgageApplication/cobol/epsnbrvl.cbl + +# +# TC1 - userBuild with file property override +# cobol_fileBuildRank +userBuild_languageConfigurations_fileProperties_TC1 = application-conf/file_languageConfig_TC1.properties +userBuild_languageConfigurations_expected_message01_TC1= *! An existing file property was detected for MortgageApplication/cobol/epsnbrvl.cbl +userBuild_languageConfigurations_expected_message02_TC1 = Updating file property override cobol_fileBuildRank = 2 for MortgageApplication/cobol/epsnbrvl.cbl + +# +# TC2 - userBuild with an property override which cannot be resolved +# +userBuild_languageConfigurations_fileProperties_TC2 = application-conf/file_languageConfig_TC2.properties +userBuild_languageConfigurations_expected_message01_TC2= *! An existing file property was detected for MortgageApplication/cobol/epsnbrvl.cbl +userBuild_languageConfigurations_expected_message02_TC2 = *! Warning: MortgageApplication/cobol/epsnbrvl.cbl is already mapped as a file pattern as part of a file group wildcard possibly in file.properties. + +# +# list of source datasets (LLQ) that should be deleted during fullBuild.groovy cleanUp +fullBuild_languageConfigurations_datasetsToCleanUp = BMS,COBOL,LINK,LOAD,OBJ \ No newline at end of file diff --git a/test/testScripts/README.md b/test/testScripts/README.md index 121f5d9e..034fac5a 100644 --- a/test/testScripts/README.md +++ b/test/testScripts/README.md @@ -4,11 +4,34 @@ This script is called by test.groovy to run a full build by creating a new “au - Number of expected build files equal the number of files build during the full build in the console. - Build files expected is the same as build files during the full build in the console. +## fullBuild_languageConfigurations.groovy +This script is called by test.groovy to run a full build and multiple user builds specifically to validate the the language configuration override capability. It verifies on +- fullBuild to see the expected overwrite for cobol_compileParms +- userBuild TC1 to have a successful override of a previously defined file level property +- userBuild TC2 to validate a failing override and check on an expected warning message + ## impactBuild.groovy This script that is called by test.groovy to run an impact build against all the program file specified in the [test.properties](/test/applications/MortgageApplication/test.properties). It verifies the below requirements - Impact build ran clean - Number of expected build files equal the number of files build during the impact build in the console. - Build files expected is the same as build files during the impact build in the console. +## impactBuild_properties.groovy +This script is called by test.groovy to run an impact build on an update of the Cobol.properties. It verifies +- that all expected files got build + +## imimpactBuild_renaming.groovy +This script is called by test.groovy to run an impact build on a renamed source file. It verifies +- clean build +- that the file with the new name is built +- that the logical file with the original name is deleted from the metadata store +Please note : This script is committing a potentially breaking change to any potential following test scripts. + +## impactBuild_deletion.groovy +This script is called by test.groovy. It runs a fullbuild first to set the baseline and then to execute an impact build after a source file was deleted. It verifies +- that the deleted file is correctly detected +- that the deleted output file is removed from the build library +Please note : This script is committing disrupting changes to the test branch and need to run at the end. + ## resetBuild.groovy This is a maintenance script that runs at the end of this test pipeline to delete collections and build result groups. diff --git a/test/testScripts/fullBuild_languageConfigurations.groovy b/test/testScripts/fullBuild_languageConfigurations.groovy new file mode 100644 index 00000000..f91a8755 --- /dev/null +++ b/test/testScripts/fullBuild_languageConfigurations.groovy @@ -0,0 +1,228 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import com.ibm.dbb.* +import com.ibm.dbb.build.* +import com.ibm.jzos.ZFile + +@Field BuildProperties props = BuildProperties.getInstance() +println "\n** Executing test script fullBuild_languageConfigurations.groovy" + +// Get the DBB_HOME location +def dbbHome = EnvVars.getHome() +if (props.verbose) println "** DBB_HOME = ${dbbHome}" + +// Create full build command +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << "--url ${props.url}" +fullBuildCommand << "--id ${props.id}" +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +fullBuildCommand << (props.verbose ? "--verbose" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" + +// Create full build command +def userBuildCommand = [] +userBuildCommand << "${dbbHome}/bin/groovyz" +userBuildCommand << "${props.zAppBuildDir}/build.groovy" +userBuildCommand << "--workspace ${props.workspace}" +userBuildCommand << "--application ${props.app}" +userBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +userBuildCommand << "--hlq ${props.hlq}" +userBuildCommand << "--logEncoding UTF-8" +userBuildCommand << "--url ${props.url}" +userBuildCommand << "--id ${props.id}" +userBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +userBuildCommand << (props.verbose ? "--verbose" : "") +userBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +userBuildCommand << "--userBuild ${props.userBuild_languageConfigurations_buildFile}" + +try { + + + /** Test FullBuild **/ + + // update language definitions files in workspace + copyLanguageConfigurations(props.fullBuild_languageConfigurations_updatedLanguageConfigs_fullBuild) + + // Run full build + println "** Executing ${fullBuildCommand.join(" ")}" + def process = [ + 'bash', + '-c', + fullBuildCommand.join(" ") + ].execute() + def outputStream = new StringBuffer(); + process.waitForProcessOutput(outputStream, System.err) + + //Validate build results + println "** Validating full build with language configuration results" + def expectedFilesBuiltList = props.fullBuild_languageConfigurations_expectedFilesBuilt_fullBuild.split(',') + + @Field def assertionList = [] + + // Validate clean build + assert outputStream.contains("Build State : CLEAN") : "*! FULL BUILD FAILED\nOUTPUT STREAM:\n$outputStream\n" + + // Validate expected number of files built + def numFullFiles = expectedFilesBuiltList.size() + assert outputStream.contains("Total files processed : ${numFullFiles}") : "*! TOTAL FILES PROCESSED ARE NOT EQUAL TO ${numFullFiles}\nOUTPUT STREAM:\n$outputStream\n" + + // Obtain property mapping of prints in the console for compile compileParms + PropertyMappings compileParmsofFiles = new PropertyMappings("fullBuild_languageConfigurations_compileParms_fullBuild") + // Iterate over build list + expectedFilesBuiltList.each { file -> + def expectedCompilerParms = compileParmsofFiles.getValue(file) + if (expectedCompilerParms) assert outputStream.contains("${expectedCompilerParms}") : "*! Expected Compiler Parms (${expectedCompilerParms}) do not match for $file\nOUTPUT STREAM:\n$outputStream\n" + } + + // Validate expected built files in output stream + assert expectedFilesBuiltList.count{ i-> outputStream.contains(i) } == expectedFilesBuiltList.size() : "*! FILES PROCESSED IN THE FULL BUILD DOES NOT CONTAIN THE LIST OF FILES PASSED ${expectedFilesBuiltList}\nOUTPUT STREAM:\n$outputStream\n" + + // reset language configuration changes in workspace to cleanup + resetLanguageConfigurationChanges() + + println "**" + println "** FULL BUILD TEST Language Configurations: PASSED **" + println "**" + + /** Test UserBuild overwride existing build property **/ + + // update language definitions files in workspace + copyLanguageConfigurations(props.fullBuild_languageConfigurations_updatedLanguageConfigs_fullBuild) + + // update file.propertes in workspace + copyFileProperties(props.userBuild_languageConfigurations_fileProperties_TC1) + + // Run user build + println "** Executing ${userBuildCommand.join(" ")}" + process = [ + 'bash', + '-c', + userBuildCommand.join(" ") + ].execute() + outputStream = new StringBuffer(); + process.waitForProcessOutput(outputStream, System.err) + + assert outputStream.contains(props.userBuild_languageConfigurations_expected_message01_TC1) : "*! Message (${props.userBuild_languageConfigurations_expected_message01_TC1}) could not be found\nOUTPUT STREAM:\n$outputStream\n" + assert outputStream.contains(props.userBuild_languageConfigurations_expected_message02_TC1) : "*! Message (${props.userBuild_languageConfigurations_expected_message02_TC1}) could not be found\nOUTPUT STREAM:\n$outputStream\n" + + // reset language configuration changes in workspace to cleanup + resetLanguageConfigurationChanges() + + println "**" + println "** USER BUILD TEST Language Configurations TEST 1 OVERRIDE FILE PROPERTY: PASSED **" + println "**" + + /** Test UserBuild unable to override existing build property **/ + + // update language definitions files in workspace + copyLanguageConfigurations(props.fullBuild_languageConfigurations_updatedLanguageConfigs_fullBuild) + + // update file.propertes in workspace + copyFileProperties(props.userBuild_languageConfigurations_fileProperties_TC2) + + // Run user build + println "** Executing ${userBuildCommand.join(" ")}" + process = [ + 'bash', + '-c', + userBuildCommand.join(" ") + ].execute() + outputStream = new StringBuffer(); + process.waitForProcessOutput(outputStream, System.err) + + assert outputStream.contains(props.userBuild_languageConfigurations_expected_message01_TC2) : "*! Message (${props.userBuild_languageConfigurations_expected_message01_TC2}) could not be found\nOUTPUT STREAM:\n$outputStream\n" + assert outputStream.contains(props.userBuild_languageConfigurations_expected_message02_TC2) : "*! Message (${props.userBuild_languageConfigurations_expected_message02_TC2}) could not be found\nOUTPUT STREAM:\n$outputStream\n" + + // reset language configuration changes in workspace to cleanup + resetLanguageConfigurationChanges() + + println "**" + println "** USER BUILD TEST Language Configurations TEST 2 with FAILING OVERRIDE FILE PROPERTY: PASSED **" + println "**" + + + +} +catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' +} +finally { + cleanUpDatasets() + // reset language configuration changes + resetLanguageConfigurationChanges() + + if (assertionList.size()>0) { + println "\n***" + println "**START OF FAILED TEST CASE for Language Configuration Overrides TEST RESULTS**\n" + println "*FAILED TEST CASE for Language Configurations RESULTS*\n" + assertionList + println "\n**END OF FAILED TEST CASE for Language Configurations **" + println "***" + } + +} + +// script end + +//************************************************************* +// Method Definitions +//************************************************************* + +def copyLanguageConfigurations(String languageConfigs) { + def langDefs = languageConfigs.split(',') + langDefs.each{ langDef -> + copyBuildConfiguration(langDef.trim()) + } +} + +def copyBuildConfiguration(String configFile) { + println "** Copying ${props.zAppBuildDir}/test/applications/${props.app}/$configFile to ${props.zAppBuildDir}/" + def commands = """ + cp ${props.zAppBuildDir}/test/applications/${props.app}/$configFile ${props.zAppBuildDir}/$configFile +""" + def task = ['bash', '-c', commands].execute() + def outputStream = new StringBuffer(); + task.waitForProcessOutput(outputStream, System.err) +} + +def copyFileProperties(String configFile) { + println "** Copying ${props.zAppBuildDir}/test/applications/${props.app}/$configFile to ${props.zAppBuildDir}/" + def commands = """ + cp ${props.zAppBuildDir}/test/applications/${props.app}/$configFile ${props.appLocation}/application-conf/file.properties +""" + def task = ['bash', '-c', commands].execute() + def outputStream = new StringBuffer(); + task.waitForProcessOutput(outputStream, System.err) +} + +def resetLanguageConfigurationChanges() { + println "** Resetting language configuration changes" + def commands = """ + git -C ${props.appLocation} reset --hard +""" + def task = ['bash', '-c', commands].execute() + def outputStream = new StringBuffer(); + task.waitForProcessOutput(outputStream, System.err) +} + +def cleanUpDatasets() { + def segments = props.fullBuild_languageConfigurations_datasetsToCleanUp.split(',') + + println "Deleting full build PDSEs ${segments}" + segments.each { segment -> + def pds = "'${props.hlq}.${segment}'" + if (ZFile.dsExists(pds)) { + if (props.verbose) println "** Deleting ${pds}" + ZFile.remove("//$pds") + } + } +} diff --git a/test/testScripts/impactBuild.groovy b/test/testScripts/impactBuild.groovy index cdd35749..1c6fde56 100644 --- a/test/testScripts/impactBuild.groovy +++ b/test/testScripts/impactBuild.groovy @@ -12,6 +12,22 @@ println "\n** Executing test script impactBuild.groovy" def dbbHome = EnvVars.getHome() if (props.verbose) println "** DBB_HOME = ${dbbHome}" +// Create full build command to set baseline +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << "--url ${props.url}" +fullBuildCommand << "--id ${props.id}" +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +fullBuildCommand << (props.verbose ? "--verbose" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" + // create impact build command def impactBuildCommand = [] impactBuildCommand << "${dbbHome}/bin/groovyz" @@ -34,6 +50,19 @@ PropertyMappings filesBuiltMappings = new PropertyMappings('impactBuild_expected def changedFiles = props.impactBuild_changedFiles.split(',') println("** Processing changed files from impactBuild_changedFiles property : ${props.impactBuild_changedFiles}") try { + + println "\n** Running full build to set baseline" + + // run impact build + println "** Executing ${fullBuildCommand.join(" ")}" + def outputStream = new StringBuffer() + def process = [ + 'bash', + '-c', + fullBuildCommand.join(" ") + ].execute() + process.waitForProcessOutput(outputStream, System.err) + changedFiles.each { changedFile -> println "\n** Running impact build test for changed file $changedFile" @@ -42,8 +71,8 @@ try { // run impact build println "** Executing ${impactBuildCommand.join(" ")}" - def outputStream = new StringBuffer() - def process = ['bash', '-c', impactBuildCommand.join(" ")].execute() + outputStream = new StringBuffer() + process = ['bash', '-c', impactBuildCommand.join(" ")].execute() process.waitForProcessOutput(outputStream, System.err) // validate build results diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 0f1b2b03..b26a5929 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -786,39 +786,4 @@ def getShortGitHash(String buildFile) { if (abbrevGitHash != null ) return abbrevGitHash if (props.verbose) println "*! Could not obtain abbreviated githash for buildFile $buildFile" return null -} - -/* - * Loading file level properties for all files on the buildList or list which is passed to this method. - */ -def loadFileLevelPropertiesFromFile(List buildList) { - - buildList.each { String buildFile -> - - // check for file level overwrite - loadFileLevelProperties = props.getFileProperty('loadFileLevelProperties', buildFile) - if (loadFileLevelProperties && loadFileLevelProperties.toBoolean()) { - - String member = new File(buildFile).getName() - String propertyFilePath = props.getFileProperty('propertyFilePath', buildFile) - String propertyExtention = props.getFileProperty('propertyFileExtension', buildFile) - String propertyFile = getAbsolutePath(props.application) + "/${propertyFilePath}/${member}.${propertyExtention}" - File fileLevelPropFile = new File(propertyFile) - - if (fileLevelPropFile.exists()) { - if (props.verbose) println "* Populating property file $propertyFile for $buildFile" - InputStream propertyFileIS = new FileInputStream(propertyFile) - Properties fileLevelProps = new Properties() - fileLevelProps.load(propertyFileIS) - - fileLevelProps.entrySet().each { entry -> - if (props.verbose) println "* Adding file level pattern $entry.key = $entry.value for $buildFile" - props.addFilePattern(entry.key, entry.value, buildFile) - } - } else { - if (props.verbose) println "* No property file found for $buildFile. Build will take the defaults or already defined file properties." - } - } - } -} - +} \ No newline at end of file diff --git a/utilities/FilePropUtilities.groovy b/utilities/FilePropUtilities.groovy new file mode 100644 index 00000000..1bfa715e --- /dev/null +++ b/utilities/FilePropUtilities.groovy @@ -0,0 +1,216 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.build.* +import groovy.transform.* +import com.ibm.dbb.build.report.records.* +import com.ibm.dbb.build.report.* + +// define script properties +@Field BuildProperties props = BuildProperties.getInstance() +@Field def buildUtils= loadScript(new File("BuildUtilities.groovy")) + +/* + * Loading file level properties for all files on the buildList or list which is passed to this method. + */ +def loadFileLevelPropertiesFromFile(List buildList) { + + if (props.verbose) println "* Populating file level properties overrides." + + buildList.each { String buildFile -> + if (props.verbose) println "** Checking file property overrides for $buildFile " + String propertyFilePath = props.getFileProperty('propertyFilePath', buildFile) + String propertyExtention = props.getFileProperty('propertyFileExtension', buildFile) + String member = new File(buildFile).getName() + def filePropMap = [:] + + // check for language configuration group level overwrite + loadLanguageConfigurationProperties = props.getFileProperty('loadLanguageConfigurationProperties', buildFile) + if (loadLanguageConfigurationProperties && loadLanguageConfigurationProperties.toBoolean()) { + String languageConfigurationPropertyFileName = props."$member" + if (languageConfigurationPropertyFileName != null) { + + // String languageConfigurationPropertyFilePath = buildUtils.getAbsolutePath(props.application) + "/${propertyFilePath}/${languageConfigurationPropertyFileName}.${propertyExtention}" + String languageConfigurationPropertyFilePath = "${props.zAppBuildDir}/build-conf/language-conf/${languageConfigurationPropertyFileName}.${propertyExtention}" + + File languageConfigurationPropertyFile = new File(languageConfigurationPropertyFilePath) + + if (languageConfigurationPropertyFile.exists()) { + filePropMap = loadProgramTypeProperties(languageConfigurationPropertyFileName, languageConfigurationPropertyFilePath, buildFile) + } else { + if (props.verbose) println "***! No language configuration properties file found for ${languageConfigurationPropertyFileName}.${propertyExtention}. Defaults or already defined file properties mapped to $buildFile." + } + + } else { + if (props.verbose) println "***! No language configuration properties file defined for $buildFile" + } + + } + + // check for file level overwrite + loadFileLevelProperties = props.getFileProperty('loadFileLevelProperties', buildFile) + if (loadFileLevelProperties && loadFileLevelProperties.toBoolean()) { + + String propertyFile = buildUtils.getAbsolutePath(props.application) + "/${propertyFilePath}/${member}.${propertyExtention}" + File fileLevelPropFile = new File(propertyFile) + + if (fileLevelPropFile.exists()) { + if (props.verbose) println "*** $buildFile has an individual artifact properties file defined in ${propertyFilePath}/${member}.${propertyExtention}" + InputStream propertyFileIS = new FileInputStream(propertyFile) + Properties fileLevelProps = new Properties() + fileLevelProps.load(propertyFileIS) + + fileLevelProps.entrySet().each { entry -> + if (props.verbose) println " Found file property ${entry.key} = ${entry.value}" + filePropMap[entry.key] = entry.value + } + } else { + if (props.verbose) println "***! No property file found for $buildFile. Build will take the defaults or already defined file properties." + } + } + + // Add the file patterns from file property map after checking the existence of the file patterns + if (props.verbose) println "*** Checking for existing file property overrides" + filePropMap.each { entry -> + // Check if the file property definition already exists + (filePatternIsMapped, filePatternIsMappedAtFileName, noChangeFilePattern, currValue) = checkExistingFilesPropertyDefinition(buildFile, member, entry.key) + + // If buildFile is already mapped to a value delete all the file patterns and add the backed-up file patterns + if (filePatternIsMapped ) { + if (filePatternIsMappedAtFileName) { + props.removeFileProperty(entry.key) + noChangeFilePattern.each { noChangeFile -> + if (props.verbose) println " Retaining file property override ${entry.key} = ${noChangeFile.value} for ${noChangeFile.key}" + props.addFilePattern(entry.key, noChangeFile.value, noChangeFile.key) + } + // Add the buildFile file pattern with new value + if (props.verbose) println " Updating file property override ${entry.key} = ${entry.value} for ${buildFile}" + props.addFilePattern(entry.key, entry.value, buildFile) + } else { + println(" *! Warning: $buildFile is already mapped as a file pattern as part of a file group wildcard possibly in file.properties.") + println(" *! Warning: Please check the existing file property list below and fix it to contain the correct file properties.") + println(" *! Existing File Property value and patterns :") + noChangeFilePattern.each { noChangeFile -> + println(" *! ${entry.key} = ${noChangeFile.value} for ${noChangeFile.key}") + } + println(" *! Warning: Override for $buildFile could not be applied. Existing file property value will be used: ${entry.key} = ${currValue}") + + // Add file property warning messages to BuildReport.json + addFilePropWarningRecord(buildFile, noChangeFilePattern, entry.key, entry.value, currValue) + } + } else { + // Add the buildFile file pattern with new value + if (props.verbose) println " Adding file property override ${entry.key} = ${entry.value} for ${buildFile}" + props.addFilePattern(entry.key, entry.value, buildFile) + } + } + } +} + +/* + * Add the language configuration properties to the file property map + */ +def loadProgramTypeProperties(String languageConfigurationPropertyFileName, String languageConfigurationPropertyFile, String buildFile) { + + def filePropMap = [:] + String propertyExtention = props.getFileProperty('propertyFileExtension', buildFile) + + if (props.verbose) println "*** $buildFile is mapped to ${languageConfigurationPropertyFileName}.${propertyExtention}" + + InputStream languageConfigurationPropertyFileIS = new FileInputStream(languageConfigurationPropertyFile) + Properties languageConfigProps = new Properties() + languageConfigProps.load(languageConfigurationPropertyFileIS) + + languageConfigProps.entrySet().each { entry -> + if (props.verbose) println " Found language configuration property ${entry.key} = ${entry.value}" + filePropMap[entry.key] = entry.value + } + return filePropMap +} + +/* + * Check if file is already mapped to the build property + * Return the file property map flag and backed-up file patterns + */ +def checkExistingFilesPropertyDefinition(String buildFile, String member, String entryKey){ + + if (props.verbose) println " Checking build property ${entryKey}" + PropertyMappings propertyMapping = new PropertyMappings(entryKey) + def propertyMappingValues = propertyMapping.getValues() + String expValue = "" + String currValue = "" + boolean filePatternIsMapped = false + boolean filePatternIsMappedAtFileName = false + def noChangeFilePattern = [:] + + propertyMappingValues.each { value -> + StringBuffer expandedValue = new StringBuffer() + + //Expand property references + value.split(",").each{ str -> + if (str.indexOf('${',0) == 0) { + str = str.substring(2, str.length()) + str = str.replaceAll("}","") + expandedValue.append(props.getProperty(str) + ",") + } else { + expandedValue.append(str + ",") + } + } + if (expandedValue.length() > 0) expandedValue.deleteCharAt(expandedValue.length()-1); + + expValue = expandedValue.toString() + + // If build file is already mapped to a value + if (propertyMapping.isMapped(expValue, buildFile)) { + if (props.verbose) println(" *! An existing file property was detected for $buildFile ") + if (props.verbose) println(" Existing value ${entryKey}=${value}") + filePatternIsMapped = true + currValue = value + } + + ArrayList filePropertyPatterns = props.getFilePropertyPatterns(entryKey,value) + + // Check and backup the mapped value for other file patterns + filePropertyPatterns.each{ filePattern -> + if (filePattern.toLowerCase().contains(member.toLowerCase())) { + filePatternIsMappedAtFileName = true + } else { + noChangeFilePattern[filePattern] = value + } + } + } + return [filePatternIsMapped, filePatternIsMappedAtFileName, noChangeFilePattern, currValue] +} + +/* + * Add file property warning messages to BuildReport.json + */ +def addFilePropWarningRecord(String buildFile, Map noChangeFilePattern, String entryKey, String entryValue, String currValue){ + + if (props.verbose) println(" *! Inserting warning message in BuildReport.json") + + // New Generic Property Record + def currPropList = [] + PropertiesRecord filePropWarningInfo = new PropertiesRecord() + + + // Add file property override warning info + def warningMsg1 = "$buildFile is already mapped as a file pattern as part of a file group wildcard possibly in file.properties." + def warningMsg2 = "Please check the existing file property list below and fix it to contain the correct file properties." + def warningMsg3 = "Override for $buildFile could not be applied. Existing file property value will be used: ${entryKey} = ${currValue}." + def warningMsg4 = "Existing File Property value and patterns in existingFilePropList:" + noChangeFilePattern.each { noChangeFile -> + currPropList.add("${entryKey} = ${noChangeFile.value} for ${noChangeFile.key}") + } + + // Add warning messages to BuildReport.json + filePropWarningInfo.addProperty("warningMsg1",warningMsg1) + filePropWarningInfo.addProperty("warningMsg2",warningMsg2) + filePropWarningInfo.addProperty("warningMsg3",warningMsg3) + filePropWarningInfo.addProperty("warningMsg4",warningMsg4) + filePropWarningInfo.addProperty("existingFilePropList",currPropList.toString()) + filePropWarningInfo.addProperty("skippedFileProperty","${entryKey} = ${entryValue}") + + def warning = warningMsg3 + " Check log or build report for more details." + + BuildReportFactory.getBuildReport().addRecord(filePropWarningInfo) + buildUtils.updateBuildResult(warningMsg:warning) +} diff --git a/utilities/README.md b/utilities/README.md index ca8234e9..8466e2f6 100644 --- a/utilities/README.md +++ b/utilities/README.md @@ -4,8 +4,9 @@ This folder contains common utilty files used by the zAppBuild `build.groovy` sc File | Description --- | --- ADMIN.pw | Encrypted password file for password "ADMIN" which is the default ID and password for the DBB Web Application Liberty server. Convenient for initial set-up and testing of DBB functionality. For more information on creating password files see the documentation for the [Repository Client](https://www.ibm.com/support/knowledgecenter/SS6T76_1.0.4/buildresult.html#repository-client) in the DBB Knowledge Center. +BindUtilities.groovy | Use for the DB2 binding. BuildUtilities.groovy | Common build utility methods. +FilePropUtilities.groovy | File property management utilities. GitUtilities.groovy | Git command methods. ImpactUtilities.groovy | Methods used for ImpactBuilds. -BindUtilities.groovy | Use for the DB2 binding. ScannerUtilities.groovy | Returns custom scanners. From 9b9449059ea4208a969529e796d757c4e50d328f Mon Sep 17 00:00:00 2001 From: M-DLB <47746258+M-DLB@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:43:37 +0100 Subject: [PATCH 9/9] Fixed lib imports and deployType in REXX.groovy (#320) Fixed lib imports and deployType in REXX.groovy Signed-off-by: M-DLB <47746258+M-DLB@users.noreply.github.com> --- languages/REXX.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/languages/REXX.groovy b/languages/REXX.groovy index 7a9c7bf6..55f0b2ec 100644 --- a/languages/REXX.groovy +++ b/languages/REXX.groovy @@ -1,6 +1,8 @@ @groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript import com.ibm.dbb.metadata.* - +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* // define script properties @Field BuildProperties props = BuildProperties.getInstance() @@ -122,7 +124,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe String linkDebugExit = props.getFileProperty('rexx_linkDebugExit', buildFile) compile.dd(new DDStatement().name("SYSPUNCH").dsn("${props.rexx_objPDS}($member)").options('shr').output(true)) - String deployType = buildUtils.getDeployType("rexx_exec", buildFile, null) + String deployType = buildUtils.getDeployType("rexx_cexec", buildFile, null) compile.dd(new DDStatement().name("SYSCEXEC").dsn("${props.rexx_cexecPDS}($member)").options('shr').output(true).deployType(deployType)) // add a syslib to the compile command with optional bms output copybook and CICS concatenation