feat: warn about missing sourceMembers

package.json

@@ -45,20 +45,20 @@
   },
   "dependencies": {
     "@oclif/core": "^3.10.8",
-    "@salesforce/core": "^6.1.3",
+    "@salesforce/core": "^6.2.0",
     "@salesforce/kit": "^3.0.15",
-    "@salesforce/source-deploy-retrieve": "^10.0.0",
+    "@salesforce/source-deploy-retrieve": "^10.0.2",
     "@salesforce/ts-types": "^2.0.9",
     "fast-xml-parser": "^4.2.5",
     "graceful-fs": "^4.2.11",
     "isomorphic-git": "1.23.0",
     "ts-retry-promise": "^0.7.0"
   },
   "devDependencies": {
-    "@salesforce/cli-plugins-testkit": "^5.0.2",
-    "@salesforce/dev-scripts": "^6.0.4",
+    "@salesforce/cli-plugins-testkit": "^5.0.4",
+    "@salesforce/dev-scripts": "^7.1.1",
     "@types/graceful-fs": "^4.1.8",
-    "eslint-plugin-sf-plugin": "^1.16.14",
+    "eslint-plugin-sf-plugin": "^1.16.15",
     "shx": "^0.3.4",
     "ts-node": "^10.9.1",
     "ts-patch": "^3.0.2",

src/index.ts

@@ -15,5 +15,6 @@ export {
   StatusOutputRow,
   ConflictResponse,
   SourceConflictError,
+  SourceMemberPollingEvent,
 } from './shared/types';
 export { getKeyFromObject, deleteCustomLabels } from './shared/functions';

src/shared/conflicts.ts

@@ -12,9 +12,7 @@ import { populateTypesAndNames } from './populateTypesAndNames';
 
 export const throwIfConflicts = (conflicts: ConflictResponse[]): void => {
   if (conflicts.length > 0) {
-    const conflictError = new SourceConflictError(`${conflicts.length} conflicts detected`, 'SourceConflictError');
-    conflictError.setData(conflicts);
-    throw conflictError;
+    throw new SourceConflictError(`${conflicts.length} conflicts detected`, conflicts);
   }
 };
 

src/shared/remoteSourceTrackingService.ts

@@ -7,10 +7,18 @@
 
 import * as path from 'node:path';
 import * as fs from 'node:fs';
+import { EOL } from 'node:os';
 import { retryDecorator, NotRetryableError } from 'ts-retry-promise';
-import { Logger, Org, Messages, Lifecycle, SfError, Connection } from '@salesforce/core';
+import { Logger, Org, Messages, Lifecycle, SfError, Connection, lockInit } from '@salesforce/core';
 import { env, Duration, parseJsonMap } from '@salesforce/kit';
-import { ChangeResult, RemoteChangeElement, MemberRevision, SourceMember, RemoteSyncInput } from './types';
+import {
+  ChangeResult,
+  RemoteChangeElement,
+  MemberRevision,
+  SourceMember,
+  RemoteSyncInput,
+  SourceMemberPollingEvent,
+} from './types';
 import { getMetadataKeyFromFileResponse, mappingsForSourceMemberTypesToMetadataType } from './metadataKeys';
 import { getMetadataKey } from './functions';
 import { calculateExpectedSourceMembers } from './expectedSourceMembers';
@@ -239,8 +247,8 @@ export class RemoteSourceTrackingService {
 
     const originalOutstandingSize = outstandingSourceMembers.size;
     // this will be the absolute timeout from the start of the poll.  We can also exit early if it doesn't look like more results are coming in
-    const pollingTimeout = calculateTimeout(outstandingSourceMembers.size);
     let highestRevisionSoFar = this.serverMaxRevisionCounter;
+    const pollingTimeout = calculateTimeout(outstandingSourceMembers.size);
     let pollAttempts = 0;
     let consecutiveEmptyResults = 0;
     let someResultsReturned = false;
@@ -279,6 +287,13 @@ export class RemoteSourceTrackingService {
         consecutiveEmptyResults++;
       }
 
+      await Lifecycle.getInstance().emit('sourceMemberPollingEvent', {
+        original: originalOutstandingSize,
+        remaining: outstandingSourceMembers.size,
+        attempts: pollAttempts,
+        consecutiveEmptyResults,
+      } satisfies SourceMemberPollingEvent);
+
       this.logger.debug(
         `[${pollAttempts}] Found ${
           originalOutstandingSize - outstandingSourceMembers.size
@@ -310,45 +325,44 @@ export class RemoteSourceTrackingService {
       delay: POLLING_DELAY_MS,
       retries: 'INFINITELY',
     });
+    const lc = Lifecycle.getInstance();
     try {
       await pollingFunction();
       this.logger.debug(`Retrieved all SourceMember data after ${pollAttempts} attempts`);
       // find places where the expectedSourceMembers might be too pruning too aggressively
       if (bonusTypes.size) {
-        void Lifecycle.getInstance().emitTelemetry({
+        void lc.emitTelemetry({
           eventName: 'sourceMemberBonusTypes',
           library: 'SourceTracking',
           deploymentSize: expectedMembers.length,
           bonusTypes: Array.from(bonusTypes).sort().join(','),
         });
       }
     } catch {
-      this.logger.warn(
-        `Polling for SourceMembers timed out after ${pollAttempts} attempts (last ${consecutiveEmptyResults} were empty) )`
-      );
-      if (outstandingSourceMembers.size < 51) {
-        this.logger.debug(
-          `Could not find ${outstandingSourceMembers.size} SourceMembers: ${Array.from(outstandingSourceMembers).join(
-            ','
-          )}`
-        );
-      } else {
-        this.logger.debug(`Could not find SourceMembers for ${outstandingSourceMembers.size} components`);
-      }
-      void Lifecycle.getInstance().emitTelemetry({
-        eventName: 'sourceMemberPollingTimeout',
-        library: 'SourceTracking',
-        timeoutSeconds: pollingTimeout.seconds,
-        attempts: pollAttempts,
-        consecutiveEmptyResults,
-        missingQuantity: outstandingSourceMembers.size,
-        deploymentSize: expectedMembers.length,
-        bonusTypes: Array.from(bonusTypes).sort().join(','),
-        types: [...new Set(Array.from(outstandingSourceMembers.values()).map((member) => member.type))]
-          .sort()
-          .join(','),
-        members: Array.from(outstandingSourceMembers.keys()).join(','),
-      });
+      await Promise.all([
+        lc.emitWarning(
+          `Polling for ${
+            outstandingSourceMembers.size
+          } SourceMembers timed out after ${pollAttempts} attempts (last ${consecutiveEmptyResults} were empty).
+
+Missing SourceMembers:
+${formatSourceMemberWarnings(outstandingSourceMembers)}`
+        ),
+        lc.emitTelemetry({
+          eventName: 'sourceMemberPollingTimeout',
+          library: 'SourceTracking',
+          timeoutSeconds: pollingTimeout.seconds,
+          attempts: pollAttempts,
+          consecutiveEmptyResults,
+          missingQuantity: outstandingSourceMembers.size,
+          deploymentSize: expectedMembers.length,
+          bonusTypes: Array.from(bonusTypes).sort().join(','),
+          types: [...new Set(Array.from(outstandingSourceMembers.values()).map((member) => member.type))]
+            .sort()
+            .join(','),
+          members: Array.from(outstandingSourceMembers.keys()).join(','),
+        }),
+      ]);
     }
   }
 
@@ -479,9 +493,8 @@ export class RemoteSourceTrackingService {
   }
 
   private async write(): Promise<void> {
-    await fs.promises.mkdir(path.dirname(this.filePath), { recursive: true });
-    return fs.promises.writeFile(
-      this.filePath,
+    const lockResult = await lockInit(this.filePath);
+    await lockResult.writeAndUnlock(
       JSON.stringify(
         {
           serverMaxRevisionCounter: this.serverMaxRevisionCounter,
@@ -585,3 +598,15 @@ const queryFn = async (conn: Connection, query: string): Promise<SourceMember[]>
     throw SfError.wrap(error as Error);
   }
 };
+
+/** organize by type and format for warning output */
+const formatSourceMemberWarnings = (outstandingSourceMembers: Map<string, RemoteSyncInput>): string => {
+  // ex: CustomObject : [Foo__c, Bar__c]
+  const mapByType = Array.from(outstandingSourceMembers.values()).reduce<Map<string, string[]>>((acc, value) => {
+    acc.set(value.type, [...(acc.get(value.type) ?? []), value.fullName]);
+    return acc;
+  }, new Map());
+  return Array.from(mapByType.entries())
+    .map(([type, names]) => `  - ${type}: ${names.join(', ')}`)
+    .join(EOL);
+};

src/shared/types.ts

@@ -64,11 +64,27 @@ export interface ConflictResponse {
   filePath: string;
 }
 
-export interface SourceConflictError extends SfError {
+// this and the related class are not enforced but a convention of this library.
+// This helps the consumers get correct typing--if the error name matches SourceConflictError,
+// there will be a data property of type ConflictResponse[]
+export interface SourceConflictErrorType extends SfError<ConflictResponse[]> {
   name: 'SourceConflictError';
-  data: ConflictResponse[];
 }
 
-export class SourceConflictError extends SfError implements SourceConflictError {}
+export class SourceConflictError extends SfError<ConflictResponse[]> implements SourceConflictErrorType {
+  public readonly name: SourceConflictErrorType['name'];
+  public constructor(message: string, data: ConflictResponse[]) {
+    super(message);
+    this.name = 'SourceConflictError';
+    this.data = data;
+  }
+}
 
 export type ChangeOptionType = ChangeResult | SourceComponent | string;
+
+export type SourceMemberPollingEvent = {
+  original: number;
+  remaining: number;
+  attempts: number;
+  consecutiveEmptyResults: number;
+};

test/unit/metadataKeys.test.ts

@@ -4,7 +4,6 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-import { Lifecycle } from '@salesforce/core';
 import { expect } from 'chai';
 import { ComponentStatus } from '@salesforce/source-deploy-retrieve';
 import { getMetadataKeyFromFileResponse, registrySupportsType } from '../../src/shared/metadataKeys';
@@ -117,14 +116,18 @@ describe('registrySupportsType', () => {
     expect(registrySupportsType('CustomObject')).to.equal(true);
     expect(registrySupportsType('ApexClass')).to.equal(true);
   });
-  it('bad type returns false and emits warning', () => {
-    let warningEmitted = false;
+  it('bad type returns false and emits warning', async () => {
+    const warningEmitted: string[] = [];
     const badType = 'NotARealType';
-    // eslint-disable-next-line @typescript-eslint/require-await
+    const { Lifecycle } = await import('@salesforce/core');
     Lifecycle.getInstance().onWarning(async (w): Promise<void> => {
-      warningEmitted = w.includes(badType);
+      warningEmitted.push(w);
+      return Promise.resolve();
     });
     expect(registrySupportsType(badType)).to.equal(false);
-    expect(warningEmitted, 'warning not emitted').to.equal(true);
+    expect(
+      warningEmitted.some((w) => w.includes(badType)),
+      'warning not emitted'
+    ).to.equal(true);
   });
 });

test/unit/remoteSourceTracking.test.ts

@@ -471,10 +471,21 @@ describe('remoteSourceTrackingService', () => {
     });
 
     describe('timeout handling', () => {
+      const warns = new Set<string>();
+
+      beforeEach(async () => {
+        warns.clear();
+        const { Lifecycle } = await import('@salesforce/core');
+        const lc = Lifecycle.getInstance();
+        lc.onWarning((w) => {
+          warns.add(w);
+          return Promise.resolve();
+        });
+      });
+
       it('should stop if the computed pollingTimeout is exceeded', async () => {
         // @ts-ignore
         const queryStub = $$.SANDBOX.stub(remoteSourceTrackingService, 'querySourceMembersFrom').resolves([]);
-        const warnSpy = $$.SANDBOX.spy($$.TEST_LOGGER, 'warn');
 
         // @ts-ignore
         const trackSpy = $$.SANDBOX.stub(remoteSourceTrackingService, 'trackSourceMembers');
@@ -483,9 +494,9 @@ describe('remoteSourceTrackingService', () => {
         await remoteSourceTrackingService.pollForSourceTracking(memberNames);
         // changed from toolbelt because each query result goes to tracking
         expect(trackSpy.callCount).to.equal(6);
-        expect(warnSpy.called).to.equal(true);
-        const expectedMsg = 'Polling for SourceMembers timed out after 6 attempts';
-        expect(warnSpy.calledWithMatch(expectedMsg)).to.equal(true);
+        expect(warns.size).to.be.greaterThan(0);
+        const expectedMsg = 'Polling for 3 SourceMembers timed out after 6 attempts';
+        expect(Array.from(warns).some((w) => w.includes(expectedMsg))).to.equal(true);
         expect(queryStub.called).to.equal(true);
       }).timeout(10000);
 
@@ -494,7 +505,6 @@ describe('remoteSourceTrackingService', () => {
         $$.SANDBOX.stub(kit.env, 'getString').callsFake(() => '3');
         // @ts-ignore
         const queryStub = $$.SANDBOX.stub(remoteSourceTrackingService, 'querySourceMembersFrom').resolves([]);
-        const warnSpy = $$.SANDBOX.spy($$.TEST_LOGGER, 'warn');
 
         // @ts-ignore
         const trackSpy = $$.SANDBOX.stub(remoteSourceTrackingService, 'trackSourceMembers');
@@ -503,10 +513,9 @@ describe('remoteSourceTrackingService', () => {
         await remoteSourceTrackingService.pollForSourceTracking(memberNames);
         expect(trackSpy.called).to.equal(true);
 
-        expect(warnSpy.called).to.equal(true);
-        const expectedMsg = 'Polling for SourceMembers timed out after 3 attempts';
-        expect(warnSpy.calledWithMatch(expectedMsg)).to.equal(true);
-        expect(warnSpy.calledOnce).to.equal(true);
+        expect(warns.size).to.be.greaterThan(0);
+        const expectedMsg = 'Polling for 3 SourceMembers timed out after 3 attempts';
+        expect(Array.from(warns).some((w) => w.includes(expectedMsg))).to.equal(true);
         expect(queryStub.called).to.equal(true);
       });
     });