From 7749eae35ed9b93c2bd1206ef1953ae6322c8ec9 Mon Sep 17 00:00:00 2001 From: "@s.roertgen" Date: Fri, 18 Aug 2023 13:12:21 +0200 Subject: [PATCH] WIP: PoC successfull Need to change to Document indexing instead of "normal" Index. Then add all the respective labels there. PoC was kind of successfull, now needs to be polished. --- gatsby-node.js | 39 ++++++++++++++++++++++++----------- src/components/nestedList.jsx | 18 +++++++++------- src/templates/App.jsx | 33 ++++++++++++++++++++++++----- 3 files changed, 66 insertions(+), 24 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index fa79f6c..76e3ab6 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -10,7 +10,7 @@ const { DataFactory } = n3 const { namedNode } = DataFactory const path = require("path") const fs = require("fs-extra") -const { Index } = require("flexsearch") +const { Index, Document } = require("flexsearch") const omitEmpty = require("omit-empty") const { i18n, getFilePath, parseLanguages } = require("./src/common") const context = require("./src/context") @@ -56,11 +56,11 @@ jsonld.registerRDFParser("text/turtle", (ttlString) => { const createData = ({ path, data }) => fs.outputFile(`public${path}`, data, (err) => err && console.error(err)) -const getTurtleFiles = function (dirPath, arrayOfFiles) { +const getTurtleFiles = function(dirPath, arrayOfFiles) { const files = fs.readdirSync(dirPath) arrayOfFiles = arrayOfFiles || [] - files.forEach(function (file) { + files.forEach(function(file) { if (fs.statSync(dirPath + "/" + file).isDirectory()) { arrayOfFiles = getTurtleFiles(dirPath + "/" + file, arrayOfFiles) } else { @@ -79,7 +79,7 @@ const getTurtleFiles = function (dirPath, arrayOfFiles) { * **/ const exportIndex = (index, conceptScheme, language) => { - index.export(function (key, data) { + index.export(function(key, data) { const path = getFilePath( conceptScheme.id + `/search/${language}/${key}`, `json` @@ -138,10 +138,10 @@ exports.onPreBootstrap = async ({ createContentDigest, actions, getNode }) => { } = graph const type = Array.isArray(properties.type) ? properties.type.find((t) => [ - "Concept", - "ConceptScheme", - "Collection", - ]) + "Concept", + "ConceptScheme", + "Collection", + ]) : properties.type const inSchemeNodes = [...(inScheme || []), ...(topConceptOf || [])] @@ -272,9 +272,14 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => { const languagesOfCS = languagesByCS[conceptScheme.id] const indexes = Object.fromEntries( [...languagesOfCS].map((l) => { - const index = new Index({ + const index = new Document({ tokenize: tokenizer, charset: "latin", + id: "id", + index: [ + "prefLabel", + "altLabel" + ] }) return [l, index] }) @@ -322,10 +327,20 @@ exports.createPages = async ({ graphql, actions: { createPage } }) => { data: JSON.stringify(jsonld, null, 2), }) } - languagesOfCS.forEach((language) => - indexes[language].add(concept.id, i18n(language)(concept.prefLabel)) - ) + // add labels to index + languagesOfCS.forEach((language) => { + const document = { + id: concept.id, + prefLabel: i18n(language)(concept.prefLabel), + ...(concept.altLabel && Object.hasOwn(concept.altLabel, language) && { altLabel: i18n(language)(concept.altLabel) }), + } + console.log(document) + indexes[language].add(document) + }) }) + console.log(indexes["de"].search("Konzept")) + console.log(indexes["de"].search("Konzept", ["prefLabel"])) + console.log(indexes["de"].search("Alterna")) languagesOfCS.forEach((language) => createPage({ diff --git a/src/components/nestedList.jsx b/src/components/nestedList.jsx index d3dc7a0..a8896d7 100644 --- a/src/components/nestedList.jsx +++ b/src/components/nestedList.jsx @@ -25,6 +25,9 @@ const getNestedItems = (item) => { */ const NestedList = ({ items, current, filter, highlight, language }) => { + console.log(filter && filter.length && filter.map(f => f.result)) + // TODO put this in App.jsx and pass here just the ids as before + const filteredIds = filter && filter.length && filter.map(f => f.result)[0] const { config } = getConfigAndConceptSchemes() const style = css` list-style-type: none; @@ -115,11 +118,12 @@ const NestedList = ({ items, current, filter, highlight, language }) => { ` const filteredItems = filter ? items.filter( - (item) => - !filter || - filter.some((filter) => getNestedItems(item).includes(filter)) - ) + (item) => + !filter || + filteredIds.some((filter) => getNestedItems(item).includes(filter)) + ) : items + console.log(filteredItems) const t = i18n(language) const isExpanded = (item, truthy, falsy) => { @@ -142,9 +146,9 @@ const NestedList = ({ items, current, filter, highlight, language }) => { dangerouslySetInnerHTML={{ __html: highlight ? t(item.prefLabel).replace( - highlight, - (str) => `${str}` - ) + highlight, + (str) => `${str}` + ) : t(item.prefLabel), }} /> diff --git a/src/templates/App.jsx b/src/templates/App.jsx index 46c0a94..04f2f2d 100644 --- a/src/templates/App.jsx +++ b/src/templates/App.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from "react" -import Index from "flexsearch/dist/module/index.js" +import Document from "flexsearch/dist/module/document.js" import escapeRegExp from "lodash.escaperegexp" import { i18n, getFilePath } from "../common" import NestedList from "../components/nestedList" @@ -19,7 +19,7 @@ const App = ({ pageContext, children }) => { const [conceptSchemeId, setConceptSchemeId] = useState( data?.currentScheme?.id ) - const [index, setIndex] = useState(new Index()) + const [index, setIndex] = useState({}) const [query, setQuery] = useState(null) const [tree, setTree] = useState( pageContext.node.type === "ConceptScheme" ? pageContext.node : null @@ -36,9 +36,30 @@ const App = ({ pageContext, children }) => { } const importIndex = async () => { - const idx = new Index() - const keys = ["cfg", "ctx", "map", "reg"] - + /** FIXME the document options need to be imported from somewhere + * maybe store them in a separate file and create it in gatsby-node.js ? + * maybe along with all the necessary keys + **/ + const idx = new Document({ + tokenize: "full", + charset: "latin", + id: "id", + index: [ + "prefLabel", + "altLabel" + ] + }) + const keys = [ + "altLabel.cfg", + "altLabel.ctx", + "altLabel.map", + // "altLabel.store", + "prefLabel.cfg", + "prefLabel.ctx", + "prefLabel.map", + "reg" + ] + console.log(conceptSchemeId) for (let i = 0, key; i < keys.length; i += 1) { key = keys[i] const data = await fetch( @@ -56,6 +77,8 @@ const App = ({ pageContext, children }) => { } setIndex(idx) + console.log(idx.search("Konzept")) + console.log(idx.search("Alternativ")) } // get concept scheme id from context