From 0c9d56883dd3f4c06c788242c3a9a27581ad5086 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Thu, 30 Nov 2023 11:12:04 -0700 Subject: [PATCH 1/3] feat: build ComponentSet from metadata and org connection --- src/collections/componentSet.ts | 2 +- src/collections/componentSetBuilder.ts | 50 ++++++++++++++++---- src/collections/types.ts | 4 ++ src/resolve/connectionResolver.ts | 18 +++++-- test/collections/componentSetBuilder.test.ts | 30 ++++++++++++ test/resolve/connectionResolver.test.ts | 33 +++++++++++++ 6 files changed, 124 insertions(+), 13 deletions(-) diff --git a/src/collections/componentSet.ts b/src/collections/componentSet.ts index 63ac8793d2..b32029b6f8 100644 --- a/src/collections/componentSet.ts +++ b/src/collections/componentSet.ts @@ -283,7 +283,7 @@ export class ComponentSet extends LazyCollection { } } - const connectionResolver = new ConnectionResolver(usernameOrConnection, options.registry); + const connectionResolver = new ConnectionResolver(usernameOrConnection, options.registry, options.metadataTypes); const manifest = await connectionResolver.resolve(options.componentFilter); const result = new ComponentSet([], options.registry); result.apiVersion = manifest.apiVersion; diff --git a/src/collections/componentSetBuilder.ts b/src/collections/componentSetBuilder.ts index aa2741e12f..cdff2ec4a7 100644 --- a/src/collections/componentSetBuilder.ts +++ b/src/collections/componentSetBuilder.ts @@ -12,6 +12,7 @@ import { StateAggregator, Logger, SfError, Messages } from '@salesforce/core'; import * as fs from 'graceful-fs'; import { ComponentSet } from '../collections'; import { RegistryAccess } from '../registry'; +import { FileProperties } from '../client'; Messages.importMessagesDirectory(__dirname); const messages = Messages.loadMessages('@salesforce/source-deploy-retrieve', 'sdr'); @@ -57,6 +58,11 @@ export class ComponentSetBuilder { const logger = Logger.childFromRoot('componentSetBuilder'); let componentSet: ComponentSet | undefined; + // A map used when building a ComponentSet from metadata type/name pairs + // key = a metadata type + // value = an array of metadata names + const mdMap = new Map(); + const { sourcepath, manifest, metadata, packagenames, apiversion, sourceapiversion, org } = options; try { if (sourcepath) { @@ -106,6 +112,15 @@ export class ComponentSetBuilder { const splitEntry = rawEntry.split(':').map((entry) => entry.trim()); // The registry will throw if it doesn't know what this type is. registry.getTypeByName(splitEntry[0]); + + // Add metadata entries to a map for possible use with an org connection below. + const mdMapEntry = mdMap.get(splitEntry[0]); + if (mdMapEntry) { + mdMapEntry.push(splitEntry[1]); + } else { + mdMap.set(splitEntry[0], [splitEntry[1]]); + } + // this '.*' is a surprisingly valid way to specify a metadata, especially a DEB :sigh: // https://github.com/salesforcecli/plugin-deploy-retrieve/blob/main/test/nuts/digitalExperienceBundle/constants.ts#L140 // because we're filtering from what we have locally, this won't allow you to retrieve new metadata (on the server only) using the partial wildcard @@ -119,7 +134,7 @@ export class ComponentSetBuilder { }) .getSourceComponents() .toArray() - .filter((cs) => Boolean(cs.fullName.match(new RegExp(splitEntry[1])))) + .filter((cs) => new RegExp(splitEntry[1]).test(cs.fullName)) .map((match) => { compSetFilter.add(match); componentSet?.add(match); @@ -147,15 +162,34 @@ export class ComponentSetBuilder { // Resolve metadata entries with an org connection if (org) { componentSet ??= new ComponentSet(); - logger.debug(`Building ComponentSet from targetUsername: ${org.username}`); + + let debugMsg = `Building ComponentSet from targetUsername: ${org.username}`; + + // *** Default Filter *** + // exclude components based on the results of componentFilter function + // components with namespacePrefix where org.exclude includes manageableState (to exclude managed packages) + // components with namespacePrefix where manageableState equals undefined (to exclude components e.g. InstalledPackage) + // components where org.exclude includes manageableState (to exclude packages without namespacePrefix e.g. unlocked packages) + let componentFilter = (component: Partial): boolean => + !component?.manageableState || !org.exclude?.includes(component.manageableState); + + if (metadata) { + debugMsg += ` filtered by metadata: ${metadata.metadataEntries.toString()}`; + + componentFilter = (component: Partial): boolean => { + if (component.type && component.fullName) { + const mdMapEntry = mdMap.get(component.type); + return !!mdMapEntry && mdMapEntry.some((mdName) => new RegExp(mdName).test(component.fullName as string)); + } + return false; + }; + } + + logger.debug(debugMsg); const fromConnection = await ComponentSet.fromConnection({ usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username, - // exclude components based on the results of componentFilter function - // components with namespacePrefix where org.exclude includes manageableState (to exclude managed packages) - // components with namespacePrefix where manageableState equals undefined (to exclude components e.g. InstalledPackage) - // components where org.exclude includes manageableState (to exclude packages without namespacePrefix e.g. unlocked packages) - componentFilter: (component): boolean => - !component?.manageableState || !org.exclude?.includes(component.manageableState), + componentFilter, + metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined, }); for (const comp of fromConnection) { diff --git a/src/collections/types.ts b/src/collections/types.ts index 0d71a2b45f..f685ece2fb 100644 --- a/src/collections/types.ts +++ b/src/collections/types.ts @@ -88,4 +88,8 @@ export interface FromConnectionOptions extends OptionalTreeRegistryOptions { * filter the result components to e.g. remove managed components */ componentFilter?: (component: Partial) => boolean; + /** + * array of metadata type names to use for `connection.metadata.list()` + */ + metadataTypes?: string[]; } diff --git a/src/resolve/connectionResolver.ts b/src/resolve/connectionResolver.ts index 2dfc332da5..a8707998b9 100644 --- a/src/resolve/connectionResolver.ts +++ b/src/resolve/connectionResolver.ts @@ -31,10 +31,20 @@ export class ConnectionResolver { private connection: Connection; private registry: RegistryAccess; - public constructor(connection: Connection, registry = new RegistryAccess()) { + // Array of metadata type names to use for listMembers. By default it includes + // all types defined in the registry. + private mdTypeNames: string[]; + + public constructor(connection: Connection, registry = new RegistryAccess(), mdTypes?: string[]) { this.connection = connection; this.registry = registry; this.logger = Logger.childFromRoot(this.constructor.name); + if (mdTypes?.length) { + // ensure the types passed in are valid per the registry + this.mdTypeNames = mdTypes.filter((t) => this.registry.getTypeByName(t)); + } else { + this.mdTypeNames = Object.values(defaultRegistry.types).map((t) => t.name); + } } public async resolve( @@ -45,9 +55,9 @@ export class ConnectionResolver { const componentTypes: Set = new Set(); const lifecycle = Lifecycle.getInstance(); const componentPromises: Array> = []; - for (const type of Object.values(defaultRegistry.types)) { - componentPromises.push(this.listMembers({ type: type.name })); - } + + this.mdTypeNames.forEach((type) => componentPromises.push(this.listMembers({ type }))); + (await Promise.all(componentPromises)).map(async (componentResult) => { for (const component of componentResult) { let componentType: MetadataType; diff --git a/test/collections/componentSetBuilder.test.ts b/test/collections/componentSetBuilder.test.ts index 8d13681b0c..ddfd1959c2 100644 --- a/test/collections/componentSetBuilder.test.ts +++ b/test/collections/componentSetBuilder.test.ts @@ -397,6 +397,36 @@ describe('ComponentSetBuilder', () => { expect(compSet.has(apexClassComponent)).to.equal(true); }); + it('should create ComponentSet from org connection and metadata', async () => { + const mdCompSet = new ComponentSet(); + mdCompSet.add(apexClassComponent); + + fromSourceStub.returns(mdCompSet); + const packageDir1 = path.resolve('force-app'); + + componentSet.add(apexClassWildcardMatch); + fromConnectionStub.resolves(componentSet); + const options = { + sourcepath: undefined, + metadata: { + metadataEntries: ['ApexClass:MyClas*'], + directoryPaths: [packageDir1], + }, + manifest: undefined, + org: { + username: 'manifest-test@org.com', + exclude: [], + }, + }; + + const compSet = await ComponentSetBuilder.build(options); + expect(fromSourceStub.calledTwice).to.equal(true); + expect(fromConnectionStub.calledOnce).to.equal(true); + expect(compSet.size).to.equal(2); + expect(compSet.has(apexClassComponent)).to.equal(true); + expect(compSet.has(apexClassWildcardMatch)).to.equal(true); + }); + it('should create ComponentSet from manifest and multiple package', async () => { fileExistsSyncStub.returns(true); diff --git a/test/resolve/connectionResolver.test.ts b/test/resolve/connectionResolver.test.ts index 98eab9f464..828f7aa77f 100644 --- a/test/resolve/connectionResolver.test.ts +++ b/test/resolve/connectionResolver.test.ts @@ -127,6 +127,39 @@ describe('ConnectionResolver', () => { ]; expect(result.components).to.deep.equal(expected); }); + it('should resolve components with specified types', async () => { + const metadataQueryStub = $$.SANDBOX.stub(connection.metadata, 'list'); + + metadataQueryStub.withArgs({ type: 'ApexClass' }).resolves([ + { + ...StdFileProperty, + fileName: 'classes/MyApexClass1.class', + fullName: 'MyApexClass1', + type: 'ApexClass', + }, + { + ...StdFileProperty, + fileName: 'classes/MyApexClass2.class', + fullName: 'MyApexClass2', + type: 'ApexClass', + }, + ]); + + const resolver = new ConnectionResolver(connection, undefined, ['ApexClass']); + const result = await resolver.resolve(); + const expected: MetadataComponent[] = [ + { + fullName: 'MyApexClass1', + type: registry.types.apexclass, + }, + { + fullName: 'MyApexClass2', + type: registry.types.apexclass, + }, + ]; + expect(result.components).to.deep.equal(expected); + expect(metadataQueryStub.calledOnce).to.be.true; + }); it('should resolve components with invalid type returned by metadata api', async () => { const metadataQueryStub = $$.SANDBOX.stub(connection.metadata, 'list'); metadataQueryStub.withArgs({ type: 'CustomLabels' }).resolves([ From e2618bf23b5361e7197d5ce0bae91a902df02239 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Thu, 30 Nov 2023 11:38:18 -0700 Subject: [PATCH 2/3] fix: use minimatch for more expected matching results --- src/collections/componentSetBuilder.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/collections/componentSetBuilder.ts b/src/collections/componentSetBuilder.ts index cdff2ec4a7..e1cd022897 100644 --- a/src/collections/componentSetBuilder.ts +++ b/src/collections/componentSetBuilder.ts @@ -10,6 +10,7 @@ import * as path from 'node:path'; import { StateAggregator, Logger, SfError, Messages } from '@salesforce/core'; import * as fs from 'graceful-fs'; +import * as minimatch from 'minimatch'; import { ComponentSet } from '../collections'; import { RegistryAccess } from '../registry'; import { FileProperties } from '../client'; @@ -134,7 +135,8 @@ export class ComponentSetBuilder { }) .getSourceComponents() .toArray() - .filter((cs) => new RegExp(splitEntry[1]).test(cs.fullName)) + // using minimatch versus RegExp provides better (more expected) matching results + .filter((cs) => minimatch(cs.fullName, splitEntry[1])) .map((match) => { compSetFilter.add(match); componentSet?.add(match); @@ -179,7 +181,8 @@ export class ComponentSetBuilder { componentFilter = (component: Partial): boolean => { if (component.type && component.fullName) { const mdMapEntry = mdMap.get(component.type); - return !!mdMapEntry && mdMapEntry.some((mdName) => new RegExp(mdName).test(component.fullName as string)); + // using minimatch versus RegExp provides better (more expected) matching results + return !!mdMapEntry && mdMapEntry.some((mdName) => minimatch(component.fullName as string, mdName)); } return false; }; From 4d94595f67678685ec41ff826115250a0081ac7c Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Tue, 5 Dec 2023 10:24:07 -0700 Subject: [PATCH 3/3] fix: address review comments --- src/collections/componentSetBuilder.ts | 28 ++++++++++++++------------ src/resolve/connectionResolver.ts | 13 +++++------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/collections/componentSetBuilder.ts b/src/collections/componentSetBuilder.ts index e1cd022897..c061e28364 100644 --- a/src/collections/componentSetBuilder.ts +++ b/src/collections/componentSetBuilder.ts @@ -59,9 +59,11 @@ export class ComponentSetBuilder { const logger = Logger.childFromRoot('componentSetBuilder'); let componentSet: ComponentSet | undefined; - // A map used when building a ComponentSet from metadata type/name pairs - // key = a metadata type - // value = an array of metadata names + /** + * A map used when building a ComponentSet from metadata type/name pairs + * key = a metadata type, e.g. `ApexClass` + * value = an array of metadata names, e.g. `['foo_*', 'BarClass']` + */ const mdMap = new Map(); const { sourcepath, manifest, metadata, packagenames, apiversion, sourceapiversion, org } = options; @@ -110,16 +112,16 @@ export class ComponentSetBuilder { // Build a Set of metadata entries metadata.metadataEntries.forEach((rawEntry) => { - const splitEntry = rawEntry.split(':').map((entry) => entry.trim()); + const [mdType, mdName] = rawEntry.split(':').map((entry) => entry.trim()); // The registry will throw if it doesn't know what this type is. - registry.getTypeByName(splitEntry[0]); + registry.getTypeByName(mdType); // Add metadata entries to a map for possible use with an org connection below. - const mdMapEntry = mdMap.get(splitEntry[0]); + const mdMapEntry = mdMap.get(mdType); if (mdMapEntry) { - mdMapEntry.push(splitEntry[1]); + mdMapEntry.push(mdName); } else { - mdMap.set(splitEntry[0], [splitEntry[1]]); + mdMap.set(mdType, [mdName]); } // this '.*' is a surprisingly valid way to specify a metadata, especially a DEB :sigh: @@ -127,24 +129,24 @@ export class ComponentSetBuilder { // because we're filtering from what we have locally, this won't allow you to retrieve new metadata (on the server only) using the partial wildcard // to do that, you'd need check the size of the CS created below, see if it's 0, and then query the org for the metadata that matches the regex // but building a CS from a metadata argument doesn't require an org, so we can't do that here - if (splitEntry[1]?.includes('*') && splitEntry[1]?.length > 1 && !splitEntry[1].includes('.*')) { + if (mdName?.includes('*') && mdName?.length > 1 && !mdName.includes('.*')) { // get all components of the type, and then filter by the regex of the fullName ComponentSet.fromSource({ fsPaths: directoryPaths, - include: new ComponentSet([{ type: splitEntry[0], fullName: ComponentSet.WILDCARD }]), + include: new ComponentSet([{ type: mdType, fullName: ComponentSet.WILDCARD }]), }) .getSourceComponents() .toArray() // using minimatch versus RegExp provides better (more expected) matching results - .filter((cs) => minimatch(cs.fullName, splitEntry[1])) + .filter((cs) => minimatch(cs.fullName, mdName)) .map((match) => { compSetFilter.add(match); componentSet?.add(match); }); } else { const entry = { - type: splitEntry[0], - fullName: splitEntry.length === 1 ? '*' : splitEntry[1], + type: mdType, + fullName: !mdName ? '*' : mdName, }; // Add to the filtered ComponentSet for resolved source paths, // and the unfiltered ComponentSet to build the correct manifest. diff --git a/src/resolve/connectionResolver.ts b/src/resolve/connectionResolver.ts index a8707998b9..fe8e2ffea5 100644 --- a/src/resolve/connectionResolver.ts +++ b/src/resolve/connectionResolver.ts @@ -39,12 +39,10 @@ export class ConnectionResolver { this.connection = connection; this.registry = registry; this.logger = Logger.childFromRoot(this.constructor.name); - if (mdTypes?.length) { - // ensure the types passed in are valid per the registry - this.mdTypeNames = mdTypes.filter((t) => this.registry.getTypeByName(t)); - } else { - this.mdTypeNames = Object.values(defaultRegistry.types).map((t) => t.name); - } + this.mdTypeNames = mdTypes?.length + ? // ensure the types passed in are valid per the registry + mdTypes.filter((t) => this.registry.getTypeByName(t)) + : Object.values(defaultRegistry.types).map((t) => t.name); } public async resolve( @@ -54,9 +52,8 @@ export class ConnectionResolver { const childrenPromises: Array> = []; const componentTypes: Set = new Set(); const lifecycle = Lifecycle.getInstance(); - const componentPromises: Array> = []; - this.mdTypeNames.forEach((type) => componentPromises.push(this.listMembers({ type }))); + const componentPromises = this.mdTypeNames.map((type) => this.listMembers({ type })); (await Promise.all(componentPromises)).map(async (componentResult) => { for (const component of componentResult) {