fix: data-items underscore properties

src/app/shared/services/dynamic-data/dynamic-data.service.spec.ts

@@ -7,7 +7,7 @@ import { AppDataService } from "../data/app-data.service";
 import { MockAppDataService } from "../data/app-data.service.spec";
 
 const TEST_DATA_ROWS = [
-  { id: "id1", number: 1, string: "hello", boolean: true },
+  { id: "id1", number: 1, string: "hello", boolean: true, _meta_field: { test: "hello" } },
   { id: "id2", number: 2, string: "goodbye", boolean: false },
   { id: "id0", number: 3, string: "goodbye", boolean: false },
 ];
@@ -144,6 +144,21 @@ describe("DynamicDataService", () => {
     expect(data[1].string).toEqual("sets an item correctly for a given _index");
   });
 
+  it("supports reading data with protected fields", async () => {
+    const obs = await service.query$("data_list", "test_flow");
+    const data = await firstValueFrom(obs);
+    expect(data[0]["_meta_field"]).toEqual({ test: "hello" });
+  });
+  it("ignores writes to protected fields", async () => {
+    await service.setItem({
+      context: SET_ITEM_CONTEXT,
+      writeableProps: { _meta_field: "updated", string: "updated" },
+    });
+    const obs = await service.query$("data_list", "test_flow");
+    const data = await firstValueFrom(obs);
+    expect(data[0]["string"]).toEqual("updated");
+    expect(data[0]["_meta_field"]).toEqual({ test: "hello" });
+  });
   it("adds metadata (row_index) to docs", async () => {
     const obs = await service.query$<any>("data_list", "test_flow");
     const data = await firstValueFrom(obs);

src/app/shared/services/dynamic-data/dynamic-data.service.ts

@@ -12,7 +12,7 @@ import { ReactiveMemoryAdapater, REACTIVE_SCHEMA_BASE } from "./adapters/reactiv
 import { TemplateActionRegistry } from "../../components/template/services/instance/template-action.registry";
 import { TopLevelProperty } from "rxdb/dist/types/types";
 
-type IDocWithID = { [key: string]: any; id: string };
+type IDocWithMeta = { id: string; APP_META?: Record<string, any> };
 
 @Injectable({ providedIn: "root" })
 /**
@@ -91,7 +91,7 @@ export class DynamicDataService extends AsyncServiceBase {
   }
 
   /** Watch for changes to a specific flow */
-  public async query$<T extends IDocWithID>(
+  public async query$<T extends IDocWithMeta>(
     flow_type: FlowTypes.FlowType,
     flow_name: string,
     queryObj?: MangoQuery
@@ -110,7 +110,8 @@ export class DynamicDataService extends AsyncServiceBase {
         return docs.map((doc) => {
           // we need mutable json so that we can replace dynamic references as required
           const data = doc.toMutableJSON();
-          return data as T;
+          // ensure any previously extracted metadata fields are repopulated
+          return this.populateMeta(data) as T;
         });
       })
     );
@@ -206,21 +207,26 @@ export class DynamicDataService extends AsyncServiceBase {
     delete this.collectionCreators[collectionName];
   }
 
-  /** Retrive json sheet data and merge with any user writes */
+  /** Retrieve json sheet data and merge with any user writes */
   private async getInitialData(flow_type: FlowTypes.FlowType, flow_name: string) {
     const flowData = await this.appDataService.getSheet(flow_type, flow_name);
     const writeData = this.writeCache.get(flow_type, flow_name) || {};
-    const writeDataArray: IDocWithID[] = Object.entries(writeData).map(([id, v]) => ({ ...v, id }));
+    const writeDataArray: IDocWithMeta[] = Object.entries(writeData).map(([id, v]) => ({
+      ...v,
+      id,
+    }));
     const mergedData = this.mergeData(flowData?.rows, writeDataArray);
-    return mergedData;
+    // HACK - rxdb can't write any fields prefixed with `_` so extract all to top-level APP_META key
+    const cleaned = mergedData.map((el) => this.extractMeta(el));
+    return cleaned;
   }
 
   /** When working with rxdb collections only alphanumeric lower case names allowed  */
   private normaliseCollectionName(flow_type: FlowTypes.FlowType, flow_name: string) {
     return `${flow_type}${flow_name}`.toLowerCase().replace(/[^a-z0-9]/g, "");
   }
 
-  private mergeData<T extends IDocWithID>(flowData: T[] = [], dbData: T[] = []) {
+  private mergeData<T extends IDocWithMeta>(flowData: T[] = [], dbData: T[] = []) {
     const flowHashmap = arrayToHashmap(flowData, "id");
     const dbDataHashmap = arrayToHashmap(dbData, "id");
     const merged = deepMergeObjects(flowHashmap, dbDataHashmap);
@@ -285,6 +291,29 @@ export class DynamicDataService extends AsyncServiceBase {
       console.warn(`[SET ITEM] - No item ${_id ? "with ID " + _id : "at index " + _index}`);
     }
   }
+
+  /**
+   * Iterate over a document's key-value pairs and populate any properties starting with
+   * an underscore to a single top-level APP_META property
+   */
+  private extractMeta(doc: IDocWithMeta) {
+    const APP_META: Record<string, any> = {};
+    for (const [key, value] of Object.entries(doc)) {
+      if (key.startsWith("_")) {
+        APP_META[key] = value;
+        delete doc[key];
+      }
+    }
+    if (Object.keys(APP_META).length > 0) {
+      doc.APP_META = APP_META;
+    }
+    return doc;
+  }
+  /** Populate any previously extracted APP_META properties back to document */
+  private populateMeta(doc: IDocWithMeta) {
+    const { APP_META, ...data } = doc;
+    return { ...data, ...APP_META };
+  }
 }
 
 /** the context for evaluating the target item to be updated, provided by the data-items component */