WIP: PoC successfull

Need to change to Document indexing instead of "normal" Index. Then add all the respective labels there. PoC was kind of successfull, now needs to be polished.
skohub-io · Aug 18, 2023 · 7749eae · 7749eae
1 parent da37696
commit 7749eae
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 24 deletions.
diff --git a/gatsby-node.js b/gatsby-node.js
@@ -10,7 +10,7 @@ const { DataFactory } = n3
 const { namedNode } = DataFactory
 const path = require("path")
 const fs = require("fs-extra")
-const { Index } = require("flexsearch")
+const { Index, Document } = require("flexsearch")
 const omitEmpty = require("omit-empty")
 const { i18n, getFilePath, parseLanguages } = require("./src/common")
 const context = require("./src/context")
@@ -56,11 +56,11 @@ jsonld.registerRDFParser("text/turtle", (ttlString) => {
 const createData = ({ path, data }) =>
   fs.outputFile(`public${path}`, data, (err) => err && console.error(err))
 
-const getTurtleFiles = function (dirPath, arrayOfFiles) {
+const getTurtleFiles = function(dirPath, arrayOfFiles) {
   const files = fs.readdirSync(dirPath)
   arrayOfFiles = arrayOfFiles || []
 
-  files.forEach(function (file) {
+  files.forEach(function(file) {
     if (fs.statSync(dirPath + "/" + file).isDirectory()) {
       arrayOfFiles = getTurtleFiles(dirPath + "/" + file, arrayOfFiles)
     } else {
@@ -79,7 +79,7 @@ const getTurtleFiles = function (dirPath, arrayOfFiles) {
  *
  **/
 const exportIndex = (index, conceptScheme, language) => {
-  index.export(function (key, data) {
+  index.export(function(key, data) {
     const path = getFilePath(
       conceptScheme.id + `/search/${language}/${key}`,
       `json`
@@ -138,10 +138,10 @@ exports.onPreBootstrap = async ({ createContentDigest, actions, getNode }) => {
       } = graph
       const type = Array.isArray(properties.type)
         ? properties.type.find((t) => [
-            "Concept",
-            "ConceptScheme",
-            "Collection",
-          ])
+          "Concept",
+          "ConceptScheme",
+          "Collection",
+        ])
         : properties.type
 
       const inSchemeNodes = [...(inScheme || []), ...(topConceptOf || [])]
@@ -272,9 +272,14 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => {
         const languagesOfCS = languagesByCS[conceptScheme.id]
         const indexes = Object.fromEntries(
           [...languagesOfCS].map((l) => {
-            const index = new Index({
+            const index = new Document({
               tokenize: tokenizer,
               charset: "latin",
+              id: "id",
+              index: [
+                "prefLabel",
+                "altLabel"
+              ]
             })
             return [l, index]
           })
@@ -322,10 +327,20 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => {
               data: JSON.stringify(jsonld, null, 2),
             })
           }
-          languagesOfCS.forEach((language) =>
-            indexes[language].add(concept.id, i18n(language)(concept.prefLabel))
-          )
+          // add labels to index
+          languagesOfCS.forEach((language) => {
+            const document = {
+              id: concept.id,
+              prefLabel: i18n(language)(concept.prefLabel),
+              ...(concept.altLabel && Object.hasOwn(concept.altLabel, language) && { altLabel: i18n(language)(concept.altLabel) }),
+            }
+            console.log(document)
+            indexes[language].add(document)
+          })
         })
+        console.log(indexes["de"].search("Konzept"))
+        console.log(indexes["de"].search("Konzept", ["prefLabel"]))
+        console.log(indexes["de"].search("Alterna"))
 
         languagesOfCS.forEach((language) =>
           createPage({

diff --git a/src/components/nestedList.jsx b/src/components/nestedList.jsx
@@ -25,6 +25,9 @@ const getNestedItems = (item) => {
  */
 
 const NestedList = ({ items, current, filter, highlight, language }) => {
+  console.log(filter && filter.length && filter.map(f => f.result))
+  // TODO put this in App.jsx and pass here just the ids as before
+  const filteredIds = filter && filter.length && filter.map(f => f.result)[0]
   const { config } = getConfigAndConceptSchemes()
   const style = css`
     list-style-type: none;
@@ -115,11 +118,12 @@ const NestedList = ({ items, current, filter, highlight, language }) => {
   `
   const filteredItems = filter
     ? items.filter(
-        (item) =>
-          !filter ||
-          filter.some((filter) => getNestedItems(item).includes(filter))
-      )
+      (item) =>
+        !filter ||
+        filteredIds.some((filter) => getNestedItems(item).includes(filter))
+    )
     : items
+  console.log(filteredItems)
   const t = i18n(language)
 
   const isExpanded = (item, truthy, falsy) => {
@@ -142,9 +146,9 @@ const NestedList = ({ items, current, filter, highlight, language }) => {
             dangerouslySetInnerHTML={{
               __html: highlight
                 ? t(item.prefLabel).replace(
-                    highlight,
-                    (str) => `<strong>${str}</strong>`
-                  )
+                  highlight,
+                  (str) => `<strong>${str}</strong>`
+                )
                 : t(item.prefLabel),
             }}
           />

diff --git a/src/templates/App.jsx b/src/templates/App.jsx
@@ -1,5 +1,5 @@
 import React, { useEffect, useState } from "react"
-import Index from "flexsearch/dist/module/index.js"
+import Document from "flexsearch/dist/module/document.js"
 import escapeRegExp from "lodash.escaperegexp"
 import { i18n, getFilePath } from "../common"
 import NestedList from "../components/nestedList"
@@ -19,7 +19,7 @@ const App = ({ pageContext, children }) => {
   const [conceptSchemeId, setConceptSchemeId] = useState(
     data?.currentScheme?.id
   )
-  const [index, setIndex] = useState(new Index())
+  const [index, setIndex] = useState({})
   const [query, setQuery] = useState(null)
   const [tree, setTree] = useState(
     pageContext.node.type === "ConceptScheme" ? pageContext.node : null
@@ -36,9 +36,30 @@ const App = ({ pageContext, children }) => {
   }
 
   const importIndex = async () => {
-    const idx = new Index()
-    const keys = ["cfg", "ctx", "map", "reg"]
-
+    /** FIXME the document options need to be imported from somewhere
+      * maybe store them in a separate file and create it in gatsby-node.js ? 
+      * maybe along with all the necessary keys
+      **/
+    const idx = new Document({
+      tokenize: "full",
+      charset: "latin",
+      id: "id",
+      index: [
+        "prefLabel",
+        "altLabel"
+      ]
+    })
+    const keys = [
+      "altLabel.cfg",
+      "altLabel.ctx",
+      "altLabel.map",
+      // "altLabel.store",
+      "prefLabel.cfg",
+      "prefLabel.ctx",
+      "prefLabel.map",
+      "reg"
+    ]
+    console.log(conceptSchemeId)
     for (let i = 0, key; i < keys.length; i += 1) {
       key = keys[i]
       const data = await fetch(
@@ -56,6 +77,8 @@ const App = ({ pageContext, children }) => {
     }
 
     setIndex(idx)
+    console.log(idx.search("Konzept"))
+    console.log(idx.search("Alternativ"))
   }
 
   // get concept scheme id from context