Bill Topic Updates (#1647)

* Switching bill topic format per dev feedback, extracting topic parser for later use, adding unit tests for topic parser

* Prettier fixes

components/db/bills.ts

@@ -36,6 +36,11 @@ export type BillContent = {
   DocumentText: string
 }
 
+export type BillTopic = {
+  category: string
+  topic: string
+}
+
 export type Bill = {
   id: string
   court: number
@@ -52,7 +57,7 @@ export type Bill = {
   history: BillHistory
   currentCommittee?: CurrentCommittee
   city?: string
-  topics?: string[]
+  topics?: BillTopic[]
   summary?: string
 }
 

functions/src/bills/search.ts

@@ -1,7 +1,7 @@
 import { isString } from "lodash"
 import { db } from "../firebase"
 import { createSearchIndexer } from "../search"
-import { Bill, CATEGORIES_BY_TOPIC } from "./types"
+import { Bill, BillTopic } from "./types"
 
 export const {
   syncToSearchIndex: syncBillToSearchIndex,
@@ -71,27 +71,10 @@ export const {
   }
 })
 
-const buildTopicsForSearch = (billTopics: string[] = []) => {
-  // We need to enrich the topics with the associated topic categories for the hierachical facets
-  const topicsByCategory = billTopics.reduce((acc, topic) => {
-    const category = CATEGORIES_BY_TOPIC[topic]
-    if (!category) {
-      console.error(`No category found for topic ${topic}`)
-      return acc
-    }
-    if (!acc[category]) acc[category] = []
-    acc[category].push(topic)
-    return acc
-  }, {} as { [key: string]: string[] })
-
-  const categories = Object.keys(topicsByCategory).sort()
-  const topicsSorted = Object.entries(topicsByCategory)
-    .reduce((acc, [category, topics]) => {
-      // Instantsearch needs lower hierarchical levels in the form "category > topic"
-      acc.push(...topics.map(topic => `${category} > ${topic}`))
-      return acc
-    }, [] as string[])
-    .sort()
+const buildTopicsForSearch = (billTopics: BillTopic[] = []) => {
+  const categoriesSorted = billTopics.map(t => t.category).sort()
 
-  return { categories, topics: topicsSorted }
+  // Instantsearch needs lower hierarchical levels in the form "category > topic"
+  const topicsSorted = billTopics.map(t => `${t.category} > ${t.topic}`).sort()
+  return { categories: categoriesSorted, topics: topicsSorted }
 }

functions/src/bills/topicParser.test.ts

@@ -0,0 +1,28 @@
+import { assignCategoriesToTopics } from "./topicParser"
+
+describe("assignCategoriesToTopics", () => {
+  it("assigns categories to topics", () => {
+    const topics = [
+      "Consumer protection",
+      "Correctional facilities",
+      "Property crimes"
+    ]
+    const categories = assignCategoriesToTopics(topics)
+    expect(categories).toEqual([
+      { category: "Commerce", topic: "Consumer protection" },
+      {
+        category: "Crime and Law Enforcement",
+        topic: "Correctional facilities"
+      },
+      { category: "Crime and Law Enforcement", topic: "Property crimes" }
+    ])
+  })
+
+  it("ignores topics with missing categories", () => {
+    const topics = ["Consumer protection", "Unknown topic"]
+    const categories = assignCategoriesToTopics(topics)
+    expect(categories).toEqual([
+      { category: "Commerce", topic: "Consumer protection" }
+    ])
+  })
+})

functions/src/bills/topicParser.ts

@@ -0,0 +1,15 @@
+import { BillTopic, CATEGORIES_BY_TOPIC } from "./types"
+
+// The ML model will return a list of topics without categories
+// We need to enrich the topics with the associated topic categories for the hierachical facets
+export const assignCategoriesToTopics = (billTopics: string[]) => {
+  return billTopics.reduce((acc, topic) => {
+    const category = CATEGORIES_BY_TOPIC[topic]
+    if (category) {
+      acc.push({ category, topic })
+    } else {
+      console.error(`No category found for topic ${topic}`)
+    }
+    return acc
+  }, [] as BillTopic[])
+}

functions/src/bills/types.ts

@@ -49,6 +49,12 @@ export const BillContent = Record({
   Cosponsors: Array(Record({ Name: Maybe(String) }))
 })
 
+export type BillTopic = Static<typeof BillTopic>
+export const BillTopic = Record({
+  category: String,
+  topic: String
+})
+
 /** Represents a missing timestamp value. This allows documents without values
  * to appear in results when sorting by that value. */
 export const MISSING_TIMESTAMP = Timestamp.fromMillis(0)
@@ -315,7 +321,7 @@ export const Bill = withDefaults(
     similar: Array(Id),
     currentCommittee: Optional(CurrentCommittee),
     city: Optional(String),
-    topics: Optional(Array(String)),
+    topics: Optional(Array(BillTopic)),
     summary: Optional(String)
   }),
   {

stories/organisms/billDetail/MockBillData.tsx

@@ -55,6 +55,17 @@ export const newBillContent: BillContent = {
   DocumentText: "this is the document text"
 }
 
+export const newBillTopics = [
+  {
+    category: "Crime and Law Enforcement",
+    topic: "Criminal investigation, prosecution, interrogation"
+  },
+  {
+    category: "Economics and Public Finance",
+    topic: "Budget process"
+  }
+]
+
 export const bill: Bill = {
   id: "123",
   court: 192,
@@ -76,9 +87,6 @@ export const bill: Bill = {
     }
   },
   city: "Boston",
-  topics: [
-    "Criminal investigation, prosecution, interrogation",
-    "Criminal sentencing"
-  ],
+  topics: newBillTopics,
   summary: "This is the summary"
 }