Fix(data): Parsing of tag model arrays on subject model (#222)

- Adds support for parsing subject model tag arrays
- Cleans up old "VerificationParser" terminoligy by renaming it to "SubjectParser"
- Fixes bug where falsy values (e.g. 0) would not be emitted in subject model
- Fix service tests not being included in unit test dev script
- Adds tests for SubjectParser

Fixes: #213

package.json

@@ -14,8 +14,8 @@
     "build:components": "vite build && tsc",
     "publish:components": "pnpm build:components && npm publish --access public",
     "test": "playwright test .",
-    "test:unit": "playwright test src/components/",
-    "test:e2e": "playwright test tests/",
+    "test:unit": "playwright test src/components/ src/services/",
+    "test:e2e": "playwright test src/tests/",
     "test:report": "playwright show-report test-results",
     "lint": "eslint .",
     "format": "prettier . --write"

public/test-items.json

@@ -9,7 +9,7 @@
     "Offset": 30,
     "AudioLink": "http://localhost:3000/example.flac",
     "Distance": "4.542739391326904",
-    "Tag": "koala"
+    "Tag": ["koala", "kangaroo"]
   },
   {
     "Filename": "20210527T140000+1000_Mt-Barney-Wet-B_515726.flac",

src/models/subject.ts

@@ -4,7 +4,7 @@ import { Verification } from "./decisions/verification";
 import { Tag, TagName } from "./tag";
 
 /** Original unprocessed data from the data source */
-export type Subject = Record<any, unknown>;
+export type Subject = Record<PropertyKey, unknown>;
 
 /**
  * @constructor

src/services/gridPageFetcher.ts

@@ -1,5 +1,5 @@
 import { SubjectWrapper } from "../models/subject";
-import { SubjectParser } from "./verificationParser";
+import { SubjectParser } from "./subjectParser";
 
 export interface IPageFetcherResponse<T> {
   subjects: SubjectWrapper[];

src/services/modelParser.ts

@@ -22,7 +22,9 @@ export abstract class ModelParser<T> {
 
     for (const [target, candidateKeys] of Object.entries(transformer)) {
       for (const candidateKey of candidateKeys) {
-        if (original[candidateKey]) {
+        const originalSubjectValue = original[candidateKey];
+
+        if (originalSubjectValue !== undefined) {
           model[target] = original[candidateKey];
         }
       }

src/services/subjectParser.spec.ts

@@ -0,0 +1,162 @@
+import { Subject } from "../models/subject";
+import { Tag } from "../models/tag";
+import { test } from "../tests/assertions";
+import { SubjectParser } from "./subjectParser";
+
+// each test must have an input, and either an expectedUrl and/or an expectedTag
+type VerificationParserTest = {
+  name: string;
+  input: Subject;
+} & ({ expectedUrl: string } | { expectedTag: Tag });
+
+const tests: VerificationParserTest[] = [
+  {
+    name: "empty subject",
+    input: {},
+    expectedUrl: "",
+    expectedTag: { text: "" },
+  },
+  {
+    name: "subject with a string tag",
+    input: {
+      src: "https://www.testing.com",
+      tags: "bird",
+    },
+    expectedUrl: "https://www.testing.com",
+    expectedTag: { text: "bird" },
+  },
+  {
+    // I have chosen to test the number zero here because it is a falsy value
+    // meaning that it shouldn't be caught by the null/undefined check to use
+    // the default tag
+    name: "subject with a falsy number tag",
+    input: {
+      tags: 0,
+    },
+    expectedTag: { text: "0" },
+  },
+  {
+    name: "subject with a number tag",
+    input: {
+      tags: 123,
+    },
+    expectedTag: { text: "123" },
+  },
+  {
+    name: "subject with a boolean tag",
+    input: {
+      tags: true,
+    },
+    expectedTag: { text: "true" },
+  },
+  {
+    name: "subject with a null tag",
+    input: {
+      tags: null,
+    },
+    expectedTag: { text: "" },
+  },
+  {
+    name: "subject with an undefined tag",
+    input: {
+      tags: undefined,
+    },
+    expectedTag: { text: "" },
+  },
+  {
+    name: "subject with an empty string tag",
+    input: {
+      tags: "",
+    },
+    expectedTag: { text: "" },
+  },
+  {
+    // I do not expect that users will pass in a symbol as a tag, but I have
+    // included this test to show that the parser can handle it
+    name: "subject with a symbol tag",
+    input: {
+      tags: Symbol("bird"),
+    },
+    expectedTag: { text: "Symbol(bird)" },
+  },
+  {
+    name: "array of string tags",
+    input: {
+      tags: ["bird", "sparrow", "finch"],
+    },
+    expectedTag: { text: "bird" },
+  },
+  {
+    name: "subject with an array of object tags",
+    input: {
+      tags: [{ text: "bird" }, { text: "sparrow" }, { text: "finch" }],
+    },
+    expectedTag: { text: "bird" },
+  },
+  {
+    name: "array of tag object and strings",
+    input: {
+      tags: ["bird", { text: "sparrow" }, "finch"],
+    },
+    expectedTag: { text: "bird" },
+  },
+  {
+    name: "nested array of tag objects",
+    input: {
+      tags: [
+        // prettier wants to inline this all on one line. However, I think that
+        // a one line object with nested arrays is hard to read
+        // I have therefore disabled prettier for these nested objects
+        // @prettier-ignore
+        [{ text: "nest" }, { text: "tree" }],
+        { text: "bird" },
+      ],
+    },
+    expectedTag: { text: "nest" },
+  },
+  {
+    name: "nested array of tag objects and strings",
+    input: {
+      tags: [
+        // @prettier-ignore
+        [{ text: "tree" }, "nest"],
+        "bird",
+        { text: "sparrow" },
+        "finch",
+      ],
+    },
+    expectedTag: { text: "tree" },
+  },
+  {
+    name: "empty array of tags",
+    input: {
+      tags: [],
+    },
+    expectedTag: { text: "" },
+  },
+  {
+    name: "nested empty array of tags",
+    input: {
+      tags: [[]],
+    },
+    expectedTag: { text: "" },
+  }
+];
+
+test.describe("SubjectParser", () => {
+  for (const testItem of tests) {
+    const result = SubjectParser.parse(testItem.input);
+
+    if ("expectedUrl" in testItem) {
+      test(`should have the correct url for a ${testItem.name}`, () => {
+        test.expect(result.url).toEqual(testItem.expectedUrl);
+      });
+    }
+
+    if ("expectedTag" in testItem) {
+      test(`should have the correct tag for a ${testItem.name}`, () => {
+        test.expect(result.tag).toEqual(testItem.expectedTag);
+      });
+    }
+  }
+});

src/services/subjectParser.ts

@@ -0,0 +1,97 @@
+import { Subject, SubjectWrapper } from "../models/subject";
+import { Tag } from "../models/tag";
+import { ModelParser } from "./modelParser";
+import { Transformer } from "./modelParser";
+
+export abstract class SubjectParser extends ModelParser<SubjectWrapper> {
+  private static emittedTagArrayWarning = false;
+
+  // we use "as const" here so that the type is inferred as a literal type
+  // and can be inlined by bundlers. We then use "satisfies" to ensure that
+  // the type is compatible with the Tag interface
+  // if we typed this as a Tag, it would reduce the type from a constant to a
+  // generic type, which would reduce linting and bundling optimizations
+  private static defaultTag = { text: "" } as const satisfies Tag;
+
+  public static parse(original: Subject): SubjectWrapper {
+    const transformer: Transformer = {
+      url: SubjectParser.keyTransformer(["src", "url", "audioLink"]),
+      tag: SubjectParser.keyTransformer([
+        // Common name formats
+        "tags",
+        "tag",
+        "label",
+        "classification",
+
+        // Raven format
+        "species",
+
+        // BirdNet format
+        "scientificName",
+        "commonName",
+
+        // Ecosounds annotation download formats
+        "commonNameTags",
+        "speciesNameTags",
+      ]),
+    };
+
+    const partialModel = SubjectParser.deriveModel(original, transformer);
+
+    const url = (partialModel.url as string) ?? "";
+
+    const tag: Tag = SubjectParser.tagParser(partialModel.tag);
+
+    return new SubjectWrapper(original as Subject, url, tag);
+  }
+
+  private static tagParser(subjectTag: any): Tag {
+    if (subjectTag === null || subjectTag === undefined) {
+      return SubjectParser.defaultTag;
+    }
+
+    const isTagString = typeof subjectTag === "string";
+    if (isTagString) {
+      return { text: subjectTag };
+    }
+
+    const isTagArray = subjectTag instanceof Array;
+    if (isTagArray) {
+      return SubjectParser.tagArrayParser(subjectTag as any[]);
+    }
+
+    // although arrays are objects, the condition above will catch arrays
+    // and early return, so we can safely assume that if we reach this point
+    // the value is an object
+    const isTagObject = typeof subjectTag === "object";
+    if (isTagObject) {
+      return subjectTag as any;
+    }
+
+    // the first guard of this function ensures that any value that gets to here
+    // is not null or undefined, meaning that we should attempt to convert it to
+    // a human readable format by calling toString()
+    //
+    // this case will be triggered if the value is an obscure type that we do
+    // not expect a subject tag to be e.g. a function, bigint, symbol, etc...
+    // this toString() call also handles numbers and booleans
+    const tagText = subjectTag.toString();
+    return { text: tagText };
+  }
+
+  private static tagArrayParser(subjectTags: unknown[]): Tag {
+    if (subjectTags.length === 0) {
+      return SubjectParser.defaultTag;
+    }
+
+    if (!SubjectParser.emittedTagArrayWarning) {
+      console.warn("Received a subject model with a tag array. Only the first tag will be used.");
+      SubjectParser.emittedTagArrayWarning = true;
+    }
+
+    // if we receive an array of tags, we only want to use the first one
+    const firstTag = subjectTags[0];
+    const tagModel = SubjectParser.tagParser(firstTag);
+    return tagModel;
+  }
+}