Metecho UI changes to facilitate extraction of non source tracking component

docs/api/schema.yml

@@ -980,6 +980,42 @@ paths:
               schema:
                 $ref: '#/components/schemas/ScratchOrg'
           description: ''
+  /api/scratch-orgs/{id}/listmetadata/:
+    post:
+      operationId: scratch_orgs_listmetadata_create
+      description: Manage Salesforce scratch orgs.
+      parameters:
+      - in: path
+        name: id
+        schema:
+          type: string
+          format: HashID
+        description: A unique integer value identifying this scratch org.
+        required: true
+      tags:
+      - scratch-orgs
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ListMetadataRequest'
+          application/x-www-form-urlencoded:
+            schema:
+              $ref: '#/components/schemas/ListMetadataRequest'
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/ListMetadataRequest'
+        required: true
+      security:
+      - tokenAuth: []
+      - cookieAuth: []
+      responses:
+        '202':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ScratchOrg'
+          description: ''
   /api/scratch-orgs/{id}/log/:
     get:
       operationId: scratch_orgs_log_retrieve
@@ -2082,6 +2118,14 @@ components:
         enabled:
           type: boolean
         state: {}
+    ListMetadataRequest:
+      type: object
+      properties:
+        desired_type:
+          type: string
+          minLength: 1
+      required:
+      - desired_type
     MinimalUser:
       type: object
       properties:
@@ -2625,6 +2669,13 @@ components:
         is_omnistudio_installed:
           type: boolean
           readOnly: true
+        non_source_changes:
+          type: object
+          additionalProperties: {}
+          readOnly: true
+        has_non_source_changes:
+          type: boolean
+          readOnly: true
       required:
       - currently_parsing_datasets
       - currently_reassigning_user
@@ -2638,6 +2689,7 @@ components:
       - expires_at
       - has_been_visited
       - has_ignored_changes
+      - has_non_source_changes
       - has_unsaved_changes
       - id
       - ignored_changes
@@ -2649,6 +2701,7 @@ components:
       - latest_commit
       - latest_commit_at
       - latest_commit_url
+      - non_source_changes
       - org_config_name
       - org_type
       - owner

metecho/api/jobs.py

@@ -1,3 +1,4 @@
+import concurrent.futures
 import contextlib
 import logging
 import string
@@ -26,6 +27,10 @@
 from cumulusci.salesforce_api.org_schema import Field, Filters, Schema, get_org_schema
 from cumulusci.salesforce_api.utils import get_simple_salesforce_connection
 from cumulusci.tasks.github.util import CommitDir
+from cumulusci.tasks.salesforce.nonsourcetracking import (
+    ListComponents,
+    ListNonSourceTrackable,
+)
 from cumulusci.tasks.vlocity.vlocity import VlocityRetrieveTask
 from cumulusci.utils import temporary_dir
 from cumulusci.utils.http.requests_utils import safe_json_from_response
@@ -605,24 +610,63 @@ def refresh_scratch_org(scratch_org, *, originating_user_id):
 refresh_scratch_org_job = job(refresh_scratch_org)
 
 
+def unsaved_changes(scratch_org, originating_user_id):
+    old_revision_numbers = scratch_org.latest_revision_numbers
+
+    new_revision_numbers = get_latest_revision_numbers(
+        scratch_org, originating_user_id=originating_user_id
+    )
+    unsaved_changes = compare_revisions(old_revision_numbers, new_revision_numbers)
+    scratch_org.unsaved_changes = unsaved_changes
+
+
+def nonsource_types(scratch_org):
+    user = scratch_org.owner
+    repo_id = scratch_org.parent.get_repo_id()
+    commit_ish = scratch_org.parent.branch_name
+
+    with local_github_checkout(user, repo_id, commit_ish) as repo_root:
+        scratch_org.valid_target_directories, _ = get_valid_target_directories(
+            user,
+            scratch_org,
+            repo_root,
+        )
+    scratch_org.non_source_changes = {}
+    with dataset_env(scratch_org) as (
+        project_config,
+        org_config,
+        sf,
+        schema,
+        repo,
+    ):
+        try:
+            components = ListNonSourceTrackable(
+                org_config=org_config,
+                project_config=project_config,
+                task_config=TaskConfig({"options": {}}),
+            )()
+            for types in components:
+                scratch_org.non_source_changes[types] = []
+        except Exception as e:
+            logger.error(f"Error in listing non-source-trackable metadatatypes: {e}")
+
+
 def get_unsaved_changes(scratch_org, *, originating_user_id):
+
     try:
-        scratch_org.refresh_from_db()
-        old_revision_numbers = scratch_org.latest_revision_numbers
-        new_revision_numbers = get_latest_revision_numbers(
-            scratch_org, originating_user_id=originating_user_id
-        )
-        unsaved_changes = compare_revisions(old_revision_numbers, new_revision_numbers)
-        user = scratch_org.owner
-        repo_id = scratch_org.parent.get_repo_id()
-        commit_ish = scratch_org.parent.branch_name
-        with local_github_checkout(user, repo_id, commit_ish) as repo_root:
-            scratch_org.valid_target_directories, _ = get_valid_target_directories(
-                user,
-                scratch_org,
-                repo_root,
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            unsaved_changes_result = executor.submit(
+                unsaved_changes,
+                scratch_org=scratch_org,
+                originating_user_id=originating_user_id,
+            )
+            nonsource_types_result = executor.submit(
+                nonsource_types, scratch_org=scratch_org
             )
-        scratch_org.unsaved_changes = unsaved_changes
+            concurrent.futures.wait([unsaved_changes_result, nonsource_types_result])
+            unsaved_changes_result.result()
+            nonsource_types_result.result()
+
     except Exception as e:
         scratch_org.refresh_from_db()
         scratch_org.finalize_get_unsaved_changes(
@@ -640,6 +684,36 @@ def get_unsaved_changes(scratch_org, *, originating_user_id):
 get_unsaved_changes_job = job(get_unsaved_changes)
 
 
+def get_nonsource_components(*, scratch_org, desired_type, originating_user_id):
+    try:
+        scratch_org.refresh_from_db()
+        with dataset_env(scratch_org) as (project_config, org_config, sf, schema, repo):
+            components = ListComponents(
+                org_config=org_config,
+                project_config=project_config,
+                task_config=TaskConfig({"options": {"metadata_types": desired_type}}),
+            )()
+
+        scratch_org.non_source_changes[desired_type] = [
+            cmp["MemberName"] for cmp in components
+        ]
+    except Exception as e:
+        scratch_org.refresh_from_db()
+        scratch_org.finalize_get_nonsource_components(
+            error=e, originating_user_id=originating_user_id
+        )
+        tb = traceback.format_exc()
+        logger.error(tb)
+        raise
+    else:
+        scratch_org.finalize_get_nonsource_components(
+            originating_user_id=originating_user_id
+        )
+
+
+get_nonsource_components_job = job(get_nonsource_components)
+
+
 def commit_changes_from_org(
     *,
     scratch_org,
@@ -685,6 +759,10 @@ def commit_changes_from_org(
         latest_revision_numbers = get_latest_revision_numbers(
             scratch_org, originating_user_id=originating_user_id
         )
+        member_types = list(desired_changes.keys())
+        for member_type in member_types:
+            if member_type in scratch_org.non_source_changes:
+                del desired_changes[member_type]
         for member_type in desired_changes.keys():
             for member_name in desired_changes[member_type]:
                 try:

metecho/api/migrations/0119_scratchorg_non_source_changes.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.0.6 on 2024-01-31 13:15
+
+import django.core.serializers.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0118_project_deleted_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scratchorg",
+            name="non_source_changes",
+            field=models.JSONField(
+                blank=True,
+                default=dict,
+                encoder=django.core.serializers.json.DjangoJSONEncoder,
+            ),
+        ),
+    ]

metecho/api/models.py

@@ -1243,6 +1243,9 @@ class ScratchOrg(
     unsaved_changes = models.JSONField(
         default=dict, encoder=DjangoJSONEncoder, blank=True
     )
+    non_source_changes = models.JSONField(
+        default=dict, encoder=DjangoJSONEncoder, blank=True
+    )
     ignored_changes = models.JSONField(
         default=dict, encoder=DjangoJSONEncoder, blank=True
     )
@@ -1492,6 +1495,32 @@ def finalize_get_unsaved_changes(self, *, error=None, originating_user_id):
                 originating_user_id=originating_user_id,
             )
 
+    def queue_get_nonsource_components(self, *, originating_user_id, desired_type):
+        from .jobs import get_nonsource_components_job
+
+        self.currently_refreshing_changes = True
+        self.save()
+        self.notify_changed(originating_user_id=originating_user_id)
+        get_nonsource_components_job.delay(
+            scratch_org=self,
+            desired_type=desired_type,
+            originating_user_id=originating_user_id,
+        )
+
+    def finalize_get_nonsource_components(self, *, error=None, originating_user_id):
+        self.currently_refreshing_changes = False
+        if error is None:
+            self.save()
+            self.notify_changed(originating_user_id=originating_user_id)
+        else:
+            self.non_source_changes = {}
+            self.save()
+            self.notify_scratch_org_error(
+                error=error,
+                type_="SCRATCH_ORG_FETCH_CHANGES_FAILED",
+                originating_user_id=originating_user_id,
+            )
+
     def queue_commit_changes(
         self,
         *,

metecho/api/serializers.py

@@ -999,6 +999,8 @@ class ScratchOrgSerializer(HashIdModelSerializer):
     description_rendered = MarkdownField(source="description", read_only=True)
     unsaved_changes = serializers.SerializerMethodField()
     has_unsaved_changes = serializers.SerializerMethodField()
+    non_source_changes = serializers.SerializerMethodField()
+    has_non_source_changes = serializers.SerializerMethodField()
     total_unsaved_changes = serializers.SerializerMethodField()
     ignored_changes = serializers.SerializerMethodField()
     has_ignored_changes = serializers.SerializerMethodField()
@@ -1049,6 +1051,8 @@ class Meta:
             "currently_retrieving_omnistudio",
             "installed_packages",
             "is_omnistudio_installed",
+            "non_source_changes",
+            "has_non_source_changes",
         )
         extra_kwargs = {
             "last_modified_at": {"read_only": True},
@@ -1089,9 +1093,15 @@ def _total_X_changes(self, obj, kind) -> int:
     def get_unsaved_changes(self, obj) -> dict:
         return self._X_changes(obj, "unsaved")
 
+    def get_non_source_changes(self, obj) -> dict:
+        return self._X_changes(obj, "non_source")
+
     def get_has_unsaved_changes(self, obj) -> bool:
         return self._has_X_changes(obj, "unsaved")
 
+    def get_has_non_source_changes(self, obj) -> bool:
+        return self._has_X_changes(obj, "non_source")
+
     def get_total_unsaved_changes(self, obj) -> int:
         return self._total_X_changes(obj, "unsaved")
 
@@ -1143,6 +1153,10 @@ class CommitSerializer(serializers.Serializer):
     target_directory = serializers.CharField()
 
 
+class ListMetadataSerializer(serializers.Serializer):
+    desired_type = serializers.CharField()
+
+
 class CommitDatasetSerializer(serializers.Serializer):
     commit_message = serializers.CharField()
     dataset_name = serializers.CharField()