From 75f69df7aac88714d10157c714ccafafc1162988 Mon Sep 17 00:00:00 2001 From: H4mzaCode Date: Thu, 27 Jul 2023 09:30:07 -0400 Subject: [PATCH] Now stems all words in the "item description" before searching --- package-lock.json | 177 +++++++++++++++++++++++++++++++++++++++++- package.json | 5 +- renderer/item_main.js | 7 +- 3 files changed, 186 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 53111bc..18850d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,8 +13,11 @@ "bootstrap": "^5.3.0", "electron-squirrel-startup": "^1.0.0", "exceljs": "^4.3.0", + "lancaster-stemmer": "^2.1.0", "lodash": "^4.17.0", - "luxon": "^3.3.0" + "luxon": "^3.3.0", + "natural": "^6.5.0", + "porter-stemmer": "^0.9.1" }, "devDependencies": { "@electron-forge/cli": "^6.2.1", @@ -972,6 +975,24 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "node_modules/afinn-165": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.4.tgz", + "integrity": "sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/afinn-165-financialmarketnews": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz", + "integrity": "sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -1050,6 +1071,17 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "dependencies": { + "sylvester": ">= 0.0.8" + }, + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -3355,6 +3387,18 @@ "json-buffer": "3.0.1" } }, + "node_modules/lancaster-stemmer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lancaster-stemmer/-/lancaster-stemmer-2.1.0.tgz", + "integrity": "sha512-EAy3W7D6mERl7EzDbbrmO0kF/n9fvqDtyM0UIB7TEoS8QrYE6oVjsxM3tsV08i+/Ks/MAAwLZb0w1NIunZXPuw==", + "bin": { + "lancaster-stemmer": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/lazystream": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", @@ -3978,6 +4022,24 @@ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" }, + "node_modules/natural": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/natural/-/natural-6.5.0.tgz", + "integrity": "sha512-EV9uSQ3zraz3CgUI2TbRfuEHEBnjkqcSKS183wqDo4hIeAi+e49xwPzDXBAWjXha8eZLKmftQ/dVZU7cW1gjsg==", + "dependencies": { + "afinn-165": "^1.0.2", + "afinn-165-financialmarketnews": "^3.0.0", + "apparatus": "^0.0.10", + "safe-stable-stringify": "^2.2.0", + "stopwords-iso": "^1.1.0", + "sylvester": "^0.0.12", + "underscore": "^1.9.1", + "wordnet-db": "^3.1.11" + }, + "engines": { + "node": ">=0.4.10" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -4507,6 +4569,14 @@ "node": ">=6" } }, + "node_modules/porter-stemmer": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/porter-stemmer/-/porter-stemmer-0.9.1.tgz", + "integrity": "sha512-vnSYgMMWKMo0+zCUumdcQAYE2RD55k2b27sPySqseetSRaGtXCCktv3llCB4EXyU1nQApMM5u+rGwVWcW72AHg==", + "engines": { + "node": "*" + } + }, "node_modules/prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -4963,6 +5033,14 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -5245,6 +5323,14 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/stopwords-iso": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stopwords-iso/-/stopwords-iso-1.1.0.tgz", + "integrity": "sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -5396,6 +5482,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sylvester": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz", + "integrity": "sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==", + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/tar": { "version": "6.1.15", "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.15.tgz", @@ -5601,6 +5695,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" + }, "node_modules/unique-filename": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz", @@ -5774,6 +5873,14 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, + "node_modules/wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==", + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -6737,6 +6844,16 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "afinn-165": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.4.tgz", + "integrity": "sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==" + }, + "afinn-165-financialmarketnews": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz", + "integrity": "sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==" + }, "agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -6791,6 +6908,14 @@ "color-convert": "^2.0.1" } }, + "apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "requires": { + "sylvester": ">= 0.0.8" + } + }, "aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -8553,6 +8678,11 @@ "json-buffer": "3.0.1" } }, + "lancaster-stemmer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lancaster-stemmer/-/lancaster-stemmer-2.1.0.tgz", + "integrity": "sha512-EAy3W7D6mERl7EzDbbrmO0kF/n9fvqDtyM0UIB7TEoS8QrYE6oVjsxM3tsV08i+/Ks/MAAwLZb0w1NIunZXPuw==" + }, "lazystream": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", @@ -9057,6 +9187,21 @@ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" }, + "natural": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/natural/-/natural-6.5.0.tgz", + "integrity": "sha512-EV9uSQ3zraz3CgUI2TbRfuEHEBnjkqcSKS183wqDo4hIeAi+e49xwPzDXBAWjXha8eZLKmftQ/dVZU7cW1gjsg==", + "requires": { + "afinn-165": "^1.0.2", + "afinn-165-financialmarketnews": "^3.0.0", + "apparatus": "^0.0.10", + "safe-stable-stringify": "^2.2.0", + "stopwords-iso": "^1.1.0", + "sylvester": "^0.0.12", + "underscore": "^1.9.1", + "wordnet-db": "^3.1.11" + } + }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -9436,6 +9581,11 @@ "xmlbuilder": "^15.1.1" } }, + "porter-stemmer": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/porter-stemmer/-/porter-stemmer-0.9.1.tgz", + "integrity": "sha512-vnSYgMMWKMo0+zCUumdcQAYE2RD55k2b27sPySqseetSRaGtXCCktv3llCB4EXyU1nQApMM5u+rGwVWcW72AHg==" + }, "prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -9759,6 +9909,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "safe-stable-stringify": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", + "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -9967,6 +10122,11 @@ "minipass": "^5.0.0" } }, + "stopwords-iso": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stopwords-iso/-/stopwords-iso-1.1.0.tgz", + "integrity": "sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==" + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -10079,6 +10239,11 @@ "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true }, + "sylvester": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.12.tgz", + "integrity": "sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==" + }, "tar": { "version": "6.1.15", "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.15.tgz", @@ -10249,6 +10414,11 @@ "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "dev": true }, + "underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" + }, "unique-filename": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz", @@ -10403,6 +10573,11 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, + "wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==" + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index f5a830c..373aa0e 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,11 @@ "bootstrap": "^5.3.0", "electron-squirrel-startup": "^1.0.0", "exceljs": "^4.3.0", + "lancaster-stemmer": "^2.1.0", "lodash": "^4.17.0", - "luxon": "^3.3.0" + "luxon": "^3.3.0", + "natural": "^6.5.0", + "porter-stemmer": "^0.9.1" }, "config": { "forge": { diff --git a/renderer/item_main.js b/renderer/item_main.js index fbcb796..02833a5 100644 --- a/renderer/item_main.js +++ b/renderer/item_main.js @@ -1,4 +1,6 @@ const { clipboard, ipcRenderer, shell } = require('electron'); +const natural = require('natural'); +const PorterStemmer = natural.PorterStemmer; // const { dialog } = require('electron').remote; const Database = require('../assets/indexDB'); const Validate = require('../assets/validators'); @@ -674,8 +676,11 @@ function validSingle(isExtended = false) { let bar = new ProgressBar(); bar.update(0, 'Starting Item Description Validation'); let raw_desc = document.getElementById("maximo-desc").value; + const tokens = raw_desc.split(' '); + const stemmedTokens = tokens.map(token => PorterStemmer.stem(token)); + let newDesc = stemmedTokens.join(' '); const worker = new WorkerHandler(); - worker.work(['validSingle', raw_desc], (result) => { + worker.work(['validSingle', newDesc], (result) => { showResult(result, isExtended) }); }