From da2ffc2678a262503f210b64d9abda2f0649e41f Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Mon, 12 Aug 2024 08:51:36 -0600 Subject: [PATCH 1/6] feat: send zip file size in pre deploy event --- src/client/index.ts | 1 + src/client/metadataApiDeploy.ts | 3 ++- src/client/types.ts | 7 +++++++ src/convert/metadataConverter.ts | 1 + src/convert/streams.ts | 2 ++ src/index.ts | 1 + 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/client/index.ts b/src/client/index.ts index b98c69625f..6419c67b65 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -43,6 +43,7 @@ export { PackageOptions, RetrieveOptions, DeployVersionData, + DeployData, RetrieveVersionData, MetadataApiRetrieveOptions, } from './types'; diff --git a/src/client/metadataApiDeploy.ts b/src/client/metadataApiDeploy.ts index 51dc67563e..6a9b0dcba8 100644 --- a/src/client/metadataApiDeploy.ts +++ b/src/client/metadataApiDeploy.ts @@ -217,7 +217,8 @@ export class MetadataApiDeploy extends MetadataTransfer< const manifestMsg = manifestVersion ? ` in v${manifestVersion} shape` : ''; const debugMsg = format(`Deploying metadata source%s using ${webService} v${apiVersion}`, manifestMsg); this.logger.debug(debugMsg); - await LifecycleInstance.emit('apiVersionDeploy', { webService, manifestVersion, apiVersion }); + const zipSize = zipBuffer.byteLength; + await LifecycleInstance.emit('deployData', { webService, manifestVersion, apiVersion, zipSize }); return this.isRestDeploy ? connection.metadata.deployRest(zipBuffer, optionsWithoutRest) diff --git a/src/client/types.ts b/src/client/types.ts index e88bbb6f6f..883c789eb2 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -371,6 +371,13 @@ export type DeployVersionData = { webService: 'SOAP' | 'REST'; }; +/** + * Data about a deployment about to be sent to the Metadata API + */ +export type DeployData = DeployVersionData & { + zipSize: number; +}; + export type RetrieveVersionData = { apiVersion: string; manifestVersion: string; diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index b542e0da07..b9b36c2fd9 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -89,6 +89,7 @@ const getResult = async (writer: StandardWriter | ZipWriter): Promise => { // union type discrimination if ('addToZip' in writer) { + console.log('zip file count =', writer.fileCount); const buffer = writer.buffer; if (!packagePath) { return { packagePath, zipBuffer: buffer }; diff --git a/src/convert/streams.ts b/src/convert/streams.ts index 5fc18e1719..24b970a8e7 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -185,6 +185,7 @@ export class StandardWriter extends ComponentWriter { export class ZipWriter extends ComponentWriter { private zip = JSZip(); private zipBuffer?: Buffer; + public fileCount: number = 0; public constructor(rootDestination?: SourcePath) { super(rootDestination); @@ -236,6 +237,7 @@ export class ZipWriter extends ComponentWriter { // Ensure only posix paths are added to zip files const posixPath = path.replace(/\\/g, '/'); this.zip.file(posixPath, contents); + this.fileCount++; } } diff --git a/src/index.ts b/src/index.ts index 550b1ef50b..b8df13e3e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,6 +44,7 @@ export { PackageOptions, RetrieveOptions, DeployVersionData, + DeployData, RetrieveVersionData, } from './client'; export { From 79416cd7acdd09322b5009664ace792726532ae5 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Mon, 12 Aug 2024 08:52:00 -0600 Subject: [PATCH 2/6] feat: send zip file size in pre deploy event --- src/convert/metadataConverter.ts | 2 +- src/convert/streams.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index b9b36c2fd9..7c1cd64d99 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -89,7 +89,7 @@ const getResult = async (writer: StandardWriter | ZipWriter): Promise => { // union type discrimination if ('addToZip' in writer) { - console.log('zip file count =', writer.fileCount); + // console.log('zip file count =', writer.fileCount); const buffer = writer.buffer; if (!packagePath) { return { packagePath, zipBuffer: buffer }; diff --git a/src/convert/streams.ts b/src/convert/streams.ts index 24b970a8e7..a3572b3381 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -183,9 +183,9 @@ export class StandardWriter extends ComponentWriter { } export class ZipWriter extends ComponentWriter { + public fileCount: number = 0; private zip = JSZip(); private zipBuffer?: Buffer; - public fileCount: number = 0; public constructor(rootDestination?: SourcePath) { super(rootDestination); From 7ac8c829a6d91735c8cdd06adaabdfb97a880c13 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Wed, 14 Aug 2024 15:49:40 -0600 Subject: [PATCH 3/6] feat: fire event with zip file size and file count --- src/client/index.ts | 2 +- src/client/metadataApiDeploy.ts | 43 +++++++++++++++++++++++--------- src/client/types.ts | 5 ++-- src/convert/metadataConverter.ts | 3 +-- src/convert/streams.ts | 3 +++ src/convert/types.ts | 4 +++ src/index.ts | 2 +- 7 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/client/index.ts b/src/client/index.ts index 6419c67b65..345249dcf0 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -43,7 +43,7 @@ export { PackageOptions, RetrieveOptions, DeployVersionData, - DeployData, + DeployZipData, RetrieveVersionData, MetadataApiRetrieveOptions, } from './types'; diff --git a/src/client/metadataApiDeploy.ts b/src/client/metadataApiDeploy.ts index 6a9b0dcba8..beb8ff8628 100644 --- a/src/client/metadataApiDeploy.ts +++ b/src/client/metadataApiDeploy.ts @@ -207,7 +207,10 @@ export class MetadataApiDeploy extends MetadataTransfer< }) ); - const [zipBuffer] = await Promise.all([this.getZipBuffer(), this.maybeSaveTempDirectory('metadata')]); + const [{ zipBuffer, zipFileCount }] = await Promise.all([ + this.getZipBuffer(), + this.maybeSaveTempDirectory('metadata'), + ]); // SDR modifies what the mdapi expects by adding a rest param const { rest, ...optionsWithoutRest } = this.options.apiOptions ?? {}; @@ -217,8 +220,16 @@ export class MetadataApiDeploy extends MetadataTransfer< const manifestMsg = manifestVersion ? ` in v${manifestVersion} shape` : ''; const debugMsg = format(`Deploying metadata source%s using ${webService} v${apiVersion}`, manifestMsg); this.logger.debug(debugMsg); + + // Event and Debug output for the zip file used for deploy const zipSize = zipBuffer.byteLength; - await LifecycleInstance.emit('deployData', { webService, manifestVersion, apiVersion, zipSize }); + let zipMessage = `Deployment zip file size = ${zipSize} Bytes`; + if (zipFileCount) { + zipMessage += ` containing ${zipFileCount} files`; + } + this.logger.debug(zipMessage); + await LifecycleInstance.emit('apiVersionDeploy', { webService, manifestVersion, apiVersion }); + await LifecycleInstance.emit('deployZipData', { zipSize, zipFileCount }); return this.isRestDeploy ? connection.metadata.deployRest(zipBuffer, optionsWithoutRest) @@ -293,14 +304,17 @@ export class MetadataApiDeploy extends MetadataTransfer< return deployResult; } - private async getZipBuffer(): Promise { + private async getZipBuffer(): Promise<{ zipBuffer: Buffer; zipFileCount?: number }> { const mdapiPath = this.options.mdapiPath; + + // Zip a directory of metadata format source if (mdapiPath) { if (!fs.existsSync(mdapiPath) || !fs.lstatSync(mdapiPath).isDirectory()) { throw messages.createError('error_directory_not_found_or_not_directory', [mdapiPath]); } const zip = JSZip(); + let zipFileCount = 0; const zipDirRecursive = (dir: string): void => { const dirents = fs.readdirSync(dir, { withFileTypes: true }); @@ -314,33 +328,38 @@ export class MetadataApiDeploy extends MetadataTransfer< // Ensure only posix paths are added to zip files const relPosixPath = relPath.replace(/\\/g, '/'); zip.file(relPosixPath, fs.createReadStream(fullPath)); + zipFileCount++; } } }; this.logger.debug('Zipping directory for metadata deploy:', mdapiPath); zipDirRecursive(mdapiPath); - return zip.generateAsync({ - type: 'nodebuffer', - compression: 'DEFLATE', - compressionOptions: { level: 9 }, - }); + return { + zipBuffer: await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: { level: 9 }, + }), + zipFileCount, + }; } - // read the zip into a buffer + // Read a zip of metadata format source into a buffer if (this.options.zipPath) { if (!fs.existsSync(this.options.zipPath)) { throw new SfError(messages.getMessage('error_path_not_found', [this.options.zipPath])); } // does encoding matter for zip files? I don't know - return fs.promises.readFile(this.options.zipPath); + return { zipBuffer: await fs.promises.readFile(this.options.zipPath) }; } + // Convert a ComponentSet of metadata in source format and zip if (this.options.components && this.components) { const converter = new MetadataConverter(this.registry); - const { zipBuffer } = await converter.convert(this.components, 'metadata', { type: 'zip' }); + const { zipBuffer, zipFileCount } = await converter.convert(this.components, 'metadata', { type: 'zip' }); if (!zipBuffer) { throw new SfError(messages.getMessage('zipBufferError')); } - return zipBuffer; + return { zipBuffer, zipFileCount }; } throw new Error('Options should include components, zipPath, or mdapiPath'); } diff --git a/src/client/types.ts b/src/client/types.ts index 883c789eb2..035bbdb4bb 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -372,10 +372,11 @@ export type DeployVersionData = { }; /** - * Data about a deployment about to be sent to the Metadata API + * Data about a deployment zip file being sent to the Metadata API. */ -export type DeployData = DeployVersionData & { +export type DeployZipData = { zipSize: number; + zipFileCount: number; }; export type RetrieveVersionData = { diff --git a/src/convert/metadataConverter.ts b/src/convert/metadataConverter.ts index 7c1cd64d99..48b9c9286a 100644 --- a/src/convert/metadataConverter.ts +++ b/src/convert/metadataConverter.ts @@ -89,10 +89,9 @@ const getResult = async (writer: StandardWriter | ZipWriter): Promise => { // union type discrimination if ('addToZip' in writer) { - // console.log('zip file count =', writer.fileCount); const buffer = writer.buffer; if (!packagePath) { - return { packagePath, zipBuffer: buffer }; + return { packagePath, zipBuffer: buffer, zipFileCount: writer.fileCount }; } else if (buffer) { await promises.writeFile(packagePath, buffer); return { packagePath }; diff --git a/src/convert/streams.ts b/src/convert/streams.ts index a3572b3381..07b3667498 100644 --- a/src/convert/streams.ts +++ b/src/convert/streams.ts @@ -183,6 +183,9 @@ export class StandardWriter extends ComponentWriter { } export class ZipWriter extends ComponentWriter { + /** + * Count of files (not directories) added to the zip file. + */ public fileCount: number = 0; private zip = JSZip(); private zipBuffer?: Buffer; diff --git a/src/convert/types.ts b/src/convert/types.ts index e0e790a5bf..625196553a 100644 --- a/src/convert/types.ts +++ b/src/convert/types.ts @@ -107,6 +107,10 @@ export type ConvertResult = { * Buffer of converted package. `Undefined` if `outputDirectory` is omitted from zip output config. */ zipBuffer?: Buffer; + /** + * When a zip buffer is created, this is the number of files in the zip. + */ + zipFileCount?: number; /** * Converted source components. Not set if archiving the package. */ diff --git a/src/index.ts b/src/index.ts index b8df13e3e1..5c4804d412 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,7 +44,7 @@ export { PackageOptions, RetrieveOptions, DeployVersionData, - DeployData, + DeployZipData, RetrieveVersionData, } from './client'; export { From 47fff3abd0b4ff793b70ec40d6390cd60aaa65ac Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Mon, 19 Aug 2024 14:33:17 -0600 Subject: [PATCH 4/6] feat: zipSize and zipFileCount in DeployResult --- src/client/metadataApiDeploy.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/client/metadataApiDeploy.ts b/src/client/metadataApiDeploy.ts index beb8ff8628..284a65f887 100644 --- a/src/client/metadataApiDeploy.ts +++ b/src/client/metadataApiDeploy.ts @@ -43,7 +43,8 @@ export class DeployResult implements MetadataTransferResult { public constructor( public readonly response: MetadataApiDeployStatus, public readonly components?: ComponentSet, - public readonly replacements = new Map() + public readonly replacements = new Map(), + public readonly zipMeta?: { zipSize: number; zipFileCount?: number } ) {} public getFileResponses(): FileResponse[] { @@ -97,6 +98,8 @@ export class MetadataApiDeploy extends MetadataTransfer< // from the apiOptions and we need it for telemetry. private readonly isRestDeploy: boolean; private readonly registry: RegistryAccess; + private zipSize?: number; + private zipFileCount?: number; public constructor(options: MetadataApiDeployOptions) { super(options); @@ -222,14 +225,15 @@ export class MetadataApiDeploy extends MetadataTransfer< this.logger.debug(debugMsg); // Event and Debug output for the zip file used for deploy - const zipSize = zipBuffer.byteLength; - let zipMessage = `Deployment zip file size = ${zipSize} Bytes`; + this.zipSize = zipBuffer.byteLength; + let zipMessage = `Deployment zip file size = ${this.zipSize} Bytes`; if (zipFileCount) { + this.zipFileCount = zipFileCount; zipMessage += ` containing ${zipFileCount} files`; } this.logger.debug(zipMessage); await LifecycleInstance.emit('apiVersionDeploy', { webService, manifestVersion, apiVersion }); - await LifecycleInstance.emit('deployZipData', { zipSize, zipFileCount }); + await LifecycleInstance.emit('deployZipData', { zipSize: this.zipSize, zipFileCount }); return this.isRestDeploy ? connection.metadata.deployRest(zipBuffer, optionsWithoutRest) @@ -278,6 +282,8 @@ export class MetadataApiDeploy extends MetadataTransfer< numberTestsTotal: result.numberTestsTotal, testsTotalTime: result.details?.runTestResult?.totalTime, filesWithReplacementsQuantity: this.replacements.size ?? 0, + zipSize: this.zipSize ?? 0, + zipFileCount: this.zipFileCount ?? 0, }); } catch (err) { const error = err as Error; @@ -290,7 +296,8 @@ export class MetadataApiDeploy extends MetadataTransfer< const deployResult = new DeployResult( result, this.components, - new Map(Array.from(this.replacements).map(([k, v]) => [k, Array.from(v)])) + new Map(Array.from(this.replacements).map(([k, v]) => [k, Array.from(v)])), + { zipSize: this.zipSize ?? 0, zipFileCount: this.zipFileCount } ); // only do event hooks if source, (NOT a metadata format) deploy if (this.options.components) { From 07f332807a782ad8e9b1cd78fe068a59a496a3b9 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Mon, 19 Aug 2024 14:44:50 -0600 Subject: [PATCH 5/6] fix: unit test --- test/client/metadataApiDeploy.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/client/metadataApiDeploy.test.ts b/test/client/metadataApiDeploy.test.ts index 621babe757..48809027f7 100644 --- a/test/client/metadataApiDeploy.test.ts +++ b/test/client/metadataApiDeploy.test.ts @@ -156,7 +156,8 @@ describe('MetadataApiDeploy', () => { await operation.start(); const result = await operation.pollStatus(); - const expected = new DeployResult(response, deployedComponents); + const zipMeta = { zipSize: 4, zipFileCount: undefined }; + const expected = new DeployResult(response, deployedComponents, undefined, zipMeta); expect(result).to.deep.equal(expected); }); From dc2b29124ee3192a40309fb5b5ac28c0021d77a0 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Mon, 19 Aug 2024 15:21:22 -0600 Subject: [PATCH 6/6] test: unit test updates --- test/convert/metadataConverter.test.ts | 1 + test/convert/streams.test.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/test/convert/metadataConverter.test.ts b/test/convert/metadataConverter.test.ts index fb8fa833c1..df041ac205 100644 --- a/test/convert/metadataConverter.test.ts +++ b/test/convert/metadataConverter.test.ts @@ -322,6 +322,7 @@ describe('MetadataConverter', () => { const result = await converter.convert(components, 'metadata', { type: 'zip' }); expect(result.zipBuffer).to.deep.equal(testBuffer); + expect(result.zipFileCount).to.equal(1); }); it('should return packagePath in result', async () => { diff --git a/test/convert/streams.test.ts b/test/convert/streams.test.ts index dc2d07e876..191dfd5a9b 100644 --- a/test/convert/streams.test.ts +++ b/test/convert/streams.test.ts @@ -483,6 +483,7 @@ describe('Streams', () => { // NOTE: Zips must only contain files with posix paths expect(jsZipFileStub.firstCall.args[0]).to.equal('classes/myComponent.cls-meta.xml'); expect(jsZipFileStub.firstCall.args[1]).to.deep.equal(Buffer.from('hi')); + expect(writer.fileCount).to.equal(3); }); it('should add entries to zip based on given write infos when zip is in-memory only', async () => { @@ -495,6 +496,7 @@ describe('Streams', () => { }); expect(jsZipFileStub.firstCall.args[0]).to.equal('classes/myComponent.cls-meta.xml'); expect(jsZipFileStub.firstCall.args[1]).to.deep.equal(Buffer.from('hi')); + expect(writer.fileCount).to.equal(3); }); it('should generateAsync zip when stream is finished', async () => {