From 5a9709b7320f42c480ed4320851064208adb8f8d Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Fri, 26 Jun 2020 17:38:35 +0000 Subject: [PATCH 001/137] added function to parse exported JSON with questions into CSV using literal plainText Q&A format --- lambda/export/package-lock.json | 945 ++++++++++++++++++++++++++++ lambda/export/package.json | 12 +- lambda/export/parseJSON.js | 45 ++ lambda/export/test/parseJSONTest.js | 45 ++ lambda/export/test/qna_FAQ.csv | 26 + lambda/export/test/qna_export.json | 365 +++++++++++ 6 files changed, 1434 insertions(+), 4 deletions(-) create mode 100644 lambda/export/parseJSON.js create mode 100644 lambda/export/test/parseJSONTest.js create mode 100644 lambda/export/test/qna_FAQ.csv create mode 100644 lambda/export/test/qna_export.json diff --git a/lambda/export/package-lock.json b/lambda/export/package-lock.json index 8d61fd96f..e163a69e0 100644 --- a/lambda/export/package-lock.json +++ b/lambda/export/package-lock.json @@ -4,11 +4,393 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==" + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "array.prototype.map": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array.prototype.map/-/array.prototype.map-1.0.2.tgz", + "integrity": "sha512-Az3OYxgsa1g7xDYp86l0nnN4bcmuEITGe1rbdEBVkrqkzMgDcbdQ2R7r41pNzti+4NMces3H8gMmuioZUilLgw==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "es-array-method-boxes-properly": "^1.0.0", + "is-string": "^1.0.4" + } + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "binary-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", + "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==" + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "requires": { + "fill-range": "^7.0.1" + } + }, + "browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "dependencies": { + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "chokidar": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.3.1.tgz", + "integrity": "sha512-4QYCEWOcK3OJrxwvyyAOxFuhpvOVCYkr33LPfFNBjAD/w3sEzWsp2BUOkI4l9bHvWioAd0rc6NlHUOEaWkTeqg==", + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.3.0" + } + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "csv-parser": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-2.3.3.tgz", + "integrity": "sha512-czcyxc4/3Tt63w0oiK1zsnRgRD4PkqWaRSJ6eef63xC0f+5LVLuGdSYEcJwGp2euPgRHx+jmlH2Lb49anb1CGQ==", + "requires": { + "minimist": "^1.2.0", + "through2": "^3.0.1" + } + }, + "csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" + }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "^1.0.12" + } + }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "es-array-method-boxes-properly": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz", + "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" + }, + "es-get-iterator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.0.tgz", + "integrity": "sha512-UfrmHuWQlNMTs35e1ypnvikg6jCz3SK8v8ImvmDsh36fCVUR1MqoFDiyn0/k52C8NqO3YsO8Oe0azeesNuqSsQ==", + "requires": { + "es-abstract": "^1.17.4", + "has-symbols": "^1.0.1", + "is-arguments": "^1.0.4", + "is-map": "^2.0.1", + "is-set": "^2.0.1", + "is-string": "^1.0.5", + "isarray": "^2.0.5" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "flat": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/flat/-/flat-4.1.0.tgz", + "integrity": "sha512-Px/TiLIznH7gEDlPXcUD4KnBusa6kR6ayRUVcnEAbreRIuhkqow/mun59BuRXwoYk7ZQOLW1ZM05ilIvK38hFw==", + "requires": { + "is-buffer": "~2.0.3" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "fsevents": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "requires": { + "is-glob": "^4.0.1" + } + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==" + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" + }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "intercept-stdout": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/intercept-stdout/-/intercept-stdout-0.1.2.tgz", @@ -17,6 +399,129 @@ "lodash.toarray": "^3.0.0" } }, + "is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==" + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-buffer": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", + "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" + }, + "is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==" + }, + "is-date-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==" + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.1.tgz", + "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==" + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + }, + "is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "requires": { + "has-symbols": "^1.0.1" + } + }, + "is-set": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.1.tgz", + "integrity": "sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA==" + }, + "is-string": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", + "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==" + }, + "is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "requires": { + "has-symbols": "^1.0.1" + } + }, + "isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + }, + "iterate-iterator": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/iterate-iterator/-/iterate-iterator-1.0.1.tgz", + "integrity": "sha512-3Q6tudGN05kbkDQDI4CqjaBf4qf85w6W6GnuZDtUVYwKgtC1q8yxYX7CZed7N+tLzQqS6roujWvszf13T+n9aw==" + }, + "iterate-value": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/iterate-value/-/iterate-value-1.0.2.tgz", + "integrity": "sha512-A6fMAio4D2ot2r/TYzr4yUWrmwNdsN5xL7+HUiyACE4DXm+q8HtPcnFTp+NnW3k4N05tZ7FVYFFb2CR13NxyHQ==", + "requires": { + "es-get-iterator": "^1.0.2", + "iterate-iterator": "^1.0.1" + } + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", @@ -66,6 +571,446 @@ "lodash._basevalues": "^3.0.0", "lodash.keys": "^3.0.0" } + }, + "log-symbols": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz", + "integrity": "sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==", + "requires": { + "chalk": "^2.4.2" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "mocha": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.0.1.tgz", + "integrity": "sha512-vefaXfdYI8+Yo8nPZQQi0QO2o+5q9UIMX1jZ1XMmK3+4+CQjc7+B0hPdUeglXiTlr8IHMVRo63IhO9Mzt6fxOg==", + "requires": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.3.1", + "debug": "3.2.6", + "diff": "4.0.2", + "escape-string-regexp": "1.0.5", + "find-up": "4.1.0", + "glob": "7.1.6", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "3.13.1", + "log-symbols": "3.0.0", + "minimatch": "3.0.4", + "ms": "2.1.2", + "object.assign": "4.1.0", + "promise.allsettled": "1.0.2", + "serialize-javascript": "3.0.0", + "strip-json-comments": "3.0.1", + "supports-color": "7.1.0", + "which": "2.0.2", + "wide-align": "1.1.3", + "workerpool": "6.0.0", + "yargs": "13.3.2", + "yargs-parser": "13.1.2", + "yargs-unparser": "1.6.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "object-inspect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==" + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "requires": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==" + }, + "promise.allsettled": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/promise.allsettled/-/promise.allsettled-1.0.2.tgz", + "integrity": "sha512-UpcYW5S1RaNKT6pd+s9jp9K9rlQge1UXKskec0j6Mmuq7UJCvlS2J2/s/yuPN8ehftf9HXMxWlKiPbGGUzpoRg==", + "requires": { + "array.prototype.map": "^1.0.1", + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "iterate-value": "^1.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "readdirp": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.3.0.tgz", + "integrity": "sha512-zz0pAkSPOXXm1viEwygWIPSPkcBYjW1xU5j/JBh5t9bGCJwa6f9+BJa6VaB2g+b55yVrmXzqkyLf4xaWYM0IkQ==", + "requires": { + "picomatch": "^2.0.7" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "save-dev": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/save-dev/-/save-dev-0.0.1-security.tgz", + "integrity": "sha512-k6knZTDNK8PKKbIqnvxiOveJinuw2LcQjqDoaorZWP9M5AR2EPsnpDeSbeoZZ0pHr5ze1uoaKdK8NBGQrJ34Uw==" + }, + "serialize-javascript": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.0.0.tgz", + "integrity": "sha512-skZcHYw2vEX4bw90nAr2iTTsz6x2SrHEnfxgKYmZlvJYBEZrvbKtobJWlQ20zczKb3bsHHXXTYt48zBA7ni9cw==" + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-json-comments": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", + "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==" + }, + "supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "requires": { + "has-flag": "^4.0.0" + }, + "dependencies": { + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + } + } + }, + "through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + } + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "requires": { + "is-number": "^7.0.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + } + }, + "workerpool": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.0.0.tgz", + "integrity": "sha512-fU2OcNA/GVAJLLyKUoHkAgIhKb0JoCpSjLC/G2vYKxUjVmQwGbRVeoPJ1a8U4pnVofz4AQV5Y/NEw8oKqxEBtA==" + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==" + }, + "yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "yargs-unparser": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-1.6.0.tgz", + "integrity": "sha512-W9tKgmSn0DpSatfri0nx52Joq5hVXgeLiqR/5G0sZNDoLZFOr/xjBUDcShCOGNsBnEMNo1KAMBkTej1Hm62HTw==", + "requires": { + "flat": "^4.1.0", + "lodash": "^4.17.15", + "yargs": "^13.3.0" + } } } } diff --git a/lambda/export/package.json b/lambda/export/package.json index 8ebbd71a4..4138e8aa2 100644 --- a/lambda/export/package.json +++ b/lambda/export/package.json @@ -3,14 +3,18 @@ "version": "1.0.0", "description": "", "main": "index.js", - "scripts": { - "test": "nodeunit test.js" - }, "author": "", "license": "ISC", "dependencies": { "bluebird": "^3.5.1", + "csv-parser": "^2.3.3", + "csv-writer": "^1.6.0", + "intercept-stdout": "^0.1.2", "lodash": "^4.17.15", - "intercept-stdout": "^0.1.2" + "mocha": "^8.0.1", + "save-dev": "0.0.1-security" + }, + "scripts": { + "test": "mocha" } } diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js new file mode 100644 index 000000000..80f6b3e8e --- /dev/null +++ b/lambda/export/parseJSON.js @@ -0,0 +1,45 @@ +// parseJson.js + +/** + * Function to parse JSON of configurations/questions from QNA Content Designer and write an output CSV file + * @param input_path : the input file path of the exported JSON + * @param output_path : the output file path to write the CSV + * @param plainText : a boolean to write the standard Q&A format or the JSON meta-data format + * @returns output_path + */ +function qnaJsonParser(input_path, output_path, plainText) { + // sets default behavior to construct a plain text FAQ which does NOT string-ify the entire JSON field into the 'Answer' column + if (plainText == undefined) { + plainText = true; + } + + const createCsvWriter = require('./node_modules/csv-writer').createObjectCsvWriter; + const csvWriter = createCsvWriter({ + path: output_path, + header: [ + {id: 'question', title: 'Question'}, + {id: 'answer', title: 'Answer'}, + {id: 'link', title: 'Document URL (optional)'}, + ] + }); + + const data = []; + + const q_list = require(input_path).qna; + q_list.forEach(function(elem) { + elem['q'].forEach(function(ques) { + var entry = {question:ques, answer:elem['a'], link:''}; + data.push(entry); + }); + }); + + csvWriter + .writeRecords(data) + .then(()=> console.log('The CSV file ' + output_path + ' was written successfully')); + + return output_path; +} + +exports.handler = async (input, output, plainText) => { + return qnaJsonParser(input, output, plainText); +}; \ No newline at end of file diff --git a/lambda/export/test/parseJSONTest.js b/lambda/export/test/parseJSONTest.js new file mode 100644 index 000000000..b86d68211 --- /dev/null +++ b/lambda/export/test/parseJSONTest.js @@ -0,0 +1,45 @@ +var assert = require('assert'); + +// json parsing test + +async function test_parser() { + const parseJSON = require('../parseJSON.js'); + const csv_path = await parseJSON.handler('./test/qna_export.json', './test/qna_FAQ.csv', true); + + const fs = require('fs') + + try { + if (fs.existsSync(csv_path)) { + //file exists + return 'exists'; + } + } catch(err) { + // console.error(err) + return 'false'; + } + + // TODO: CHECK CONTENTS OF CSV ROWS TO VALIDATE FORMAT + // const csv = require('csv-parser'); + // const fs = require('fs'); + // const csv_text = []; + + // fs.createReadStream(csv_path) + // .pipe(csv()) + // .on('data', (data) => csv_text.push(data)) + // .on('end', () => { + // console.log('CSV processed'); + // // [ + // // { NAME: 'Daffy Duck', AGE: '24' }, + // // { NAME: 'Bugs Bunny', AGE: '22' } + // // ] + // }); +} + + +describe('#test automate-sync()', () => { + it('test_parser', async function() { + let resp = await test_parser(); + assert(resp, "CSV file does not exist!"); + }) +}); + diff --git a/lambda/export/test/qna_FAQ.csv b/lambda/export/test/qna_FAQ.csv new file mode 100644 index 000000000..bbb7d0968 --- /dev/null +++ b/lambda/export/test/qna_FAQ.csv @@ -0,0 +1,26 @@ +Question,Answer,Document URL (optional) +help,"I am the QnA bot, ask me a question and I will try my best to answer it.", +help me,"I am the QnA bot, ask me a question and I will try my best to answer it.", +need help,"I am the QnA bot, ask me a question and I will try my best to answer it.", +no_hits,The Kendra Fallback search was not able to identify any results, +Thumbs down,Thank you for your feedback - we will try to improve this answer., +Bad answer,Thank you for your feedback - we will try to improve this answer., +Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", +How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive", +How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C., +How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", +Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", +What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers., +How wide is the sun?,The Sun's diameter is 1.4 million kilometers., +Thumbs up,"Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", +Good answer,"Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", +Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", +How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.", +What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.", +What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", +what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", +Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", +I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", +How old is the Sun?,Our Sun is 4.6 billion years old., +What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/., +no_hits_orig,{{Settings.EMPTYMESSAGE}}, diff --git a/lambda/export/test/qna_export.json b/lambda/export/test/qna_export.json new file mode 100644 index 000000000..b1fde4418 --- /dev/null +++ b/lambda/export/test/qna_export.json @@ -0,0 +1,365 @@ +{ + "qna": [ + { + "a": "I am the QnA bot, ask me a question and I will try my best to answer it.", + "qid": "Help", + "type": "qna", + "q": [ + "help", + "help me", + "need help" + ] + }, + { + "qid": "KendraFallback", + "a": "The Kendra Fallback search was not able to identify any results", + "l": "QNA:EXTKendraFallback", + "type": "qna", + "q": [ + "no_hits" + ] + }, + { + "a": "Thank you for your feedback - we will try to improve this answer.", + "qid": "Feedback.001", + "args": [ + "incorrect" + ], + "l": "QNA:ExamplePYTHONLambdaFeedback", + "next": "", + "r": { + "subTitle": "", + "title": "", + "url": "", + "text": "", + "imageUrl": "", + "buttons": [ + { + "text": "", + "value": "" + } + ] + }, + "t": "", + "alt": { + "ssml": "", + "markdown": "" + }, + "type": "qna", + "q": [ + "Thumbs down", + "Bad answer" + ] + }, + { + "next": "", + "args": [ + "" + ], + "a": "When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", + "r": { + "subTitle": "", + "imageUrl": "", + "title": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.4", + "type": "qna", + "selected": false, + "q": [ + "Will the Sun ever consume the earth?" + ] + }, + { + "a": "The Sun is 109 times wider than the Earth and 330,000 times as massive", + "r": { + "imageUrl": "", + "title": "", + "subTitle": "" + }, + "qid": "sun.6", + "type": "qna", + "q": [ + "How large is the sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "At its centre the Sun reaches temperatures of 15 million °C.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.3", + "type": "qna", + "q": [ + "How hot is it at the center of the Sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.10", + "type": "qna", + "q": [ + "How many planets orbit the Sun?", + "Which planets orbit the Sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "The Sun's diameter is 1.4 million kilometers.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.11", + "type": "qna", + "q": [ + "What is the diameter of the Sun?", + "How wide is the sun?" + ] + }, + { + "a": "Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", + "qid": "Feedback.002", + "args": [ + "correct" + ], + "l": "QNA:ExamplePYTHONLambdaFeedback", + "type": "qna", + "q": [ + "Thumbs up", + "Good answer" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.1", + "type": "qna", + "q": [ + "Tell me about the sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "The sun is 149,598,262 km (1 AU) from earth.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.5", + "type": "qna", + "selected": false, + "q": [ + "How far away is the sun?", + "What distance is the sun from the earth?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.9", + "type": "qna", + "q": [ + "What are solar flares?", + "what is a solar flare?", + "Define solar flares.", + "I'd like to know more about solar flares." + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Our Sun is 4.6 billion years old.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.2", + "type": "qna", + "q": [ + "How old is the Sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.7", + "type": "qna", + "selected": false, + "q": [ + "What other locations on the internet have information about the sun?" + ] + }, + { + "qid": "CustomNoMatches", + "a": "{{Settings.EMPTYMESSAGE}}", + "type": "qna", + "q": [ + "no_hits_orig" + ] + } + ] +} \ No newline at end of file From f7a32cca4acaca3afa9acd0f08c5143fb3241f93 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 29 Jun 2020 04:47:33 +0000 Subject: [PATCH 002/137] in process of shortening SSML to longest highlight --- .../KendraFallback/KendraFallback.js | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index 23e664fd4..9c486b67d 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -81,6 +81,27 @@ function mergeIntervals(intervals) { } + +/** + * Function to return the longest interval from a list of sorted intervals + * @param intervals + * @returns {*} + */ +function longestInterval(intervals) { + // test if there are at least 2 intervals + if (intervals.length == 0) { + return intervals; + } else if (intervals.length == 1) { + return intervals[0]; + } + + // sort the intervals based on their length + intervals.sort(function(a, b) {return (a[1]-a[0]) - (b[1]-b[0])}); + return intervals[0]; + +} + + /** Function that processes kendra requests and handles response. Decides whether to handle SNS * events or Lambda Hook events from QnABot. * @param event - input event passed to the Lambda Handler @@ -148,6 +169,7 @@ async function routeKendraRequest(event, context) { let faqanswerMessage = 'Answer from Amazon Kendra FAQ.'; let faqanswerMessageMd = '*Answer from Amazon Kendra FAQ.* \n '; let markdownAnswer = ""; + let speechMessage = ""; let helpfulLinksMsg = 'Source Link'; let extractedTextMsg = 'Discovered Text'; let moreResultsMsg = 'Additional Search Results'; @@ -177,6 +199,7 @@ async function routeKendraRequest(event, context) { element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text) { answerMessage += '\n\n ' + element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); + // Emboldens the highlighted phrases returned by the Kendra response API in markdown format let answerTextMd = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); // iterates over the answer highlights in sorted order of BeginOffset, merges the overlapping intervals var sorted_highlights = mergeIntervals(element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Highlights); @@ -200,6 +223,11 @@ async function routeKendraRequest(event, context) { }; answerMessageMd = answerMessageMd + '\n\n' + answerTextMd; + // Shortens the speech response to contain say the longest highlighted phrase + var longest_highlight = longestInterval(sorted_highlights); + let answerText = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); + speechMessage = answerText.substring(longest_highlight.BeginOffset, longest_highlight.EndOffset); + answerDocumentUris.add(element.DocumentURI); kendraQueryId = res.QueryId; // store off the QueryId to use as a session attribute for feedback kendraIndexId = res.originalKendraIndexId; // store off the Kendra IndexId to use as a session attribute for feedback @@ -264,6 +292,10 @@ async function routeKendraRequest(event, context) { if (foundAnswerCount > 0) { event.res.message = answerMessage; let ssmlMessage = `${answerMessage.substring(0,600).replace(/\r?\n|\r/g, " ")}`; + if (speechMessage != "") { + ssmlMessage = `${speechMessage.substring(0,600).replace(/\r?\n|\r/g, " ")}`; + } + let lastIndex = ssmlMessage.lastIndexOf('.'); if (lastIndex > 0) { ssmlMessage = ssmlMessage.substring(0,lastIndex); From d7d9c840edcdd2dab8615a83e3cc7aef8db8184a Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 29 Jun 2020 04:54:43 +0000 Subject: [PATCH 003/137] limiting number of search results --- .../extensions/js_lambda_hooks/KendraFallback/KendraFallback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index 9e3121022..36055c6af 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -150,7 +150,7 @@ async function routeKendraRequest(event, context) { let markdownAnswer = ""; let helpfulLinksMsg = 'Possible Links'; let extractedTextMsg = 'Discovered Text'; - let maxDocumentCount = 4; + let maxDocumentCount = 2; var seenTop = false; let foundAnswerCount = 0; From 80cd7a20950a5803dada16032bf3c1d0462df92a Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 29 Jun 2020 16:42:31 +0000 Subject: [PATCH 004/137] when kendra returns search results, the SSML response should only speak the longest highlight from the answer --- .../KendraFallback/KendraFallback.js | 22 +++++++++++++------ .../KendraFallback/test/run_test.js | 4 ++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index 9c486b67d..b5c607b37 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -164,7 +164,10 @@ async function routeKendraRequest(event, context) { // process kendra query responses and update answer content /* default message text - can be overridden using QnABot SSM Parameter Store Custom Property */ - let answerMessage = 'While I did not find an exact answer, these search results from Amazon Kendra might be helpful. '; + let topAnswerMessage = "Amazon Kendra suggested answer. \n\n "; + let topAnswerMessageMd = "*Amazon Kendra suggested answer.* \n "; + let docSearchMessage = 'While I did not find an exact answer, these search results from Amazon Kendra might be helpful. '; + let answerMessage = docSearchMessage; let answerMessageMd = '*While I did not find an exact answer, these search results from Amazon Kendra might be helpful.* \n '; let faqanswerMessage = 'Answer from Amazon Kendra FAQ.'; let faqanswerMessageMd = '*Answer from Amazon Kendra FAQ.* \n '; @@ -213,8 +216,8 @@ async function routeKendraRequest(event, context) { if (elem.TopAnswer == true) { // if top answer is found, then answer is abbreviated to this phrase seenTop = true; - answerMessage = 'Amazon Kendra suggested answer. \n\n ' + highlight + '.'; - answerMessageMd = '*Amazon Kendra suggested answer.* \n '; + answerMessage = topAnswerMessage + highlight + '.'; + answerMessageMd = topAnswerMessageMd; answerTextMd = '**' + highlight + '** '; break; } else { @@ -223,10 +226,14 @@ async function routeKendraRequest(event, context) { }; answerMessageMd = answerMessageMd + '\n\n' + answerTextMd; - // Shortens the speech response to contain say the longest highlighted phrase - var longest_highlight = longestInterval(sorted_highlights); - let answerText = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); - speechMessage = answerText.substring(longest_highlight.BeginOffset, longest_highlight.EndOffset); + // Shortens the speech response to contain say the longest highlighted phrase ONLY IF top answer not found + if (seenTop == false) { + var longest_highlight = longestInterval(sorted_highlights); + let answerText = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); + speechMessage = answerText.substring(longest_highlight.BeginOffset, longest_highlight.EndOffset); + speechMessage = docSearchMessage + speechMessage + '.'; + } + answerDocumentUris.add(element.DocumentURI); kendraQueryId = res.QueryId; // store off the QueryId to use as a session attribute for feedback @@ -301,6 +308,7 @@ async function routeKendraRequest(event, context) { ssmlMessage = ssmlMessage.substring(0,lastIndex); } ssmlMessage = ` ${ssmlMessage} `; + event.res.session.appContext.altMessages.markdown = answerMessageMd; event.res.session.appContext.altMessages.ssml = ssmlMessage; if (event.req["_event"].outputDialogMode !== 'Text') { diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js index 0dfafbe6a..dc33ca1c2 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js @@ -53,6 +53,10 @@ describe('#test_kendra_highlights()', () => { // tests that when querying an unstructured document, highlights are boldened in the excerpts when no top answer is found assert.equal(resp.res.session.appContext.altMessages.markdown, "*While I did not find an exact answer, these search results from Amazon Kendra might be helpful.* \n \n\nAnatomy of the Sun Mysteries of the Sun **The Sun is an incandescent mass of hydrogen, helium, and other heavier elements**. While it appears constant and unchanging from our vantage point on Earth, it actually has a dynamic and variable system of twisting magnetic fields that cause solar events of almost unimaginable power. The Convection Zone Energy continues to move toward the surface through convection currents of heated and cooled gas in the convection zone. The Radiative Zone Energy moves slowly outward—taking more than 170,000 years to radiate through the layer of the **Sun** known as the radiative zone.\n\n Source Link: [https://s3.us-east-1.amazonaws.com/explore-kendra-solar/637244main_MysteriesOfTheSun_Book.pdf](https://s3.us-east-1.amazonaws.com/explore-kendra-solar/637244main_MysteriesOfTheSun_Book.pdf)\n\n***\n\n
\n\n ...Mass 1.989 × 1030 kg Density 1.409 g/cm3 **Composition** 92.1% hydrogen, 7.8% helium, 0.1% other elements Surface Temperature (Photosphere) 5,500 deg C (10,000 deg F) Luminosity* 3.83 × 1033 ergs/sec *The total energy radiated by the **Sun** (or any star) per second at all wavelengths...\n\n Source Link: [https://s3.us-east-1.amazonaws.com/explore-kendra-solar/Sun_Lithograph.pdf](https://s3.us-east-1.amazonaws.com/explore-kendra-solar/Sun_Lithograph.pdf)\n\n***\n\n
\n\n ...gases may contribute to a change in tempera- ture or water **composition** in the atmosphere. Stratosphere 10–31 Miles The ozone layer lies within the stratosphere and absorbs ultraviolet radiation from the **Sun**. Troposphere 0–10 Miles The troposphere is the layer of the...\n\n Source Link: [https://s3.us-east-1.amazonaws.com/explore-kendra-solar/637244main_MysteriesOfTheSun_Book.pdf](https://s3.us-east-1.amazonaws.com/explore-kendra-solar/637244main_MysteriesOfTheSun_Book.pdf)"); + + // tests that when querying an unstructured document, the SSML response returns the longest highlight from the response when no top answer is found + assert.equal(resp.res.session.appContext.altMessages.ssml, + " While I did not find an exact answer, these search results from Amazon Kendra might be helpful. The Sun is an incandescent mass of hydrogen, helium, and other heavier elements "); }); }); From a4ae33375c2ba30bd90330ab03034ea146549028 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 1 Jul 2020 13:48:04 +0000 Subject: [PATCH 005/137] updates: testing JSON stringify, debugging CSV-->FAQ --- lambda/export/createFAQ.js | 109 ++++++++++++++++++ lambda/export/package-lock.json | 6 +- lambda/export/parseJSON.js | 9 +- lambda/export/sync.js | 16 +++ .../{parseJSONTest.js => automateSyncTest.js} | 37 +++--- 5 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 lambda/export/createFAQ.js create mode 100644 lambda/export/sync.js rename lambda/export/test/{parseJSONTest.js => automateSyncTest.js} (55%) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js new file mode 100644 index 000000000..4b0e1152f --- /dev/null +++ b/lambda/export/createFAQ.js @@ -0,0 +1,109 @@ +// createFAQ.js + +// TODO: don't hardcode + + +const AWSKendra = require('aws-sdk/clients/kendra'); +const AWSS3 = require('aws-sdk/clients/s3'); + + +/** + * Function to upload CSV to S3 bucket, return Promise + * @param s3Client + * @param params + * @returns {*} + */ +function s3Uploader(s3Client,params) { + return new Promise(function(resolve, reject) { + s3Client.putObject(params, function(err, data) { + if (err) { + console.log(err, err.stack); // an error occurred + reject(err); + } + else { + console.log('Uploaded CSV to S3 successfully: '); + console.log(data); // successful response + resolve(data); + } + }); + }); +} + + +/** + * Function to convert uploaded CSV into Kendra FAQ, return Promise + * @param kendraClient + * @param params + * @returns {*} + */ +function faqConverter(kendraClient,params) { + return new Promise(function(resolve, reject) { + kendraClient.createFaq(params, function(err, data) { + console.log('FAQ conversion params are : ') + console.log(params); + if (err) { + console.log(err, err.stack); // an error occurred + reject(err); + } + else { + console.log('Converted CSV to FAQ successfully: '); + console.log(data); // successful response + resolve(data); + } + }); + }); +} + + + + +/** + * Function to upload CSV into S3 bucket and convert into Kendra FAQ, return Promise + * @returns {*} + */ +async function createFAQ() { + let kendraClient = (process.env.REGION ? + new AWSKendra({apiVersion: '2019-02-03', region: process.env.REGION}) : + new AWSKendra({apiVersion: '2019-02-03'})); + let s3Client = (process.env.REGION ? + new AWSS3({apiVersion: '2006-03-01', region: process.env.REGION}) : + new AWSS3({apiVersion: '2006-03-01'})); + + console.log('clients created'); + + + // read in CSV and upload to S3 bucket + var fs = require('fs'); + let csv_path = './test/qna_FAQ.csv'; + + var s3_params = { + Bucket: 'explore-kendra-solar', + Key: 'qna_FAQ.csv', + ACL: 'private', + Body: fs.createReadStream(csv_path), // use read stream option in case file is large + }; + await s3Uploader(s3Client, s3_params); + + + // create the FAQ + var faq_params = { + IndexId: 'e1c23860-e5c8-4409-ae26-b05bd6ced00a', + Name: 'explore-qnabot', + RoleArn: 'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer', + S3Path: { + Bucket: 'explore-kendra-solar', + Key: 'qna_FAQ.csv' + }, + Description: 'Exported FAQ of questions from QnABot designer console', + Tags: [] // no tags + }; + console.log('starting FAQ conversion'); + var faq_response = await faqConverter(kendraClient, faq_params); + + return faq_response; +} + +// TODO: event and context will be triggered by the clicking on the menu item +exports.handler = async() => { + return createFAQ(); +} \ No newline at end of file diff --git a/lambda/export/package-lock.json b/lambda/export/package-lock.json index e163a69e0..cab2ba9e0 100644 --- a/lambda/export/package-lock.json +++ b/lambda/export/package-lock.json @@ -56,9 +56,9 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "binary-extensions": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", - "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==" }, "bluebird": { "version": "3.7.2", diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 80f6b3e8e..a721b74d5 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -1,4 +1,4 @@ -// parseJson.js +// parseJSON.js /** * Function to parse JSON of configurations/questions from QNA Content Designer and write an output CSV file @@ -28,7 +28,12 @@ function qnaJsonParser(input_path, output_path, plainText) { const q_list = require(input_path).qna; q_list.forEach(function(elem) { elem['q'].forEach(function(ques) { - var entry = {question:ques, answer:elem['a'], link:''}; + var entry = {question:ques, answer:elem, link:''}; // entire JSON field in the answer field + var entry = {question:ques, answer:elem['a'], link:elem}; // entire JSON field in the URL field + // var entry = {question:ques, answer:elem['a'], link:'QID: ' + elem['qid']}; // QID in the URL field + // var entry = {question:ques, answer:elem['a'], link:''}; // explore changing link display text + + // var entry = {question:ques, answer:elem['a'], link:''}; // original data.push(entry); }); }); diff --git a/lambda/export/sync.js b/lambda/export/sync.js new file mode 100644 index 000000000..1547c5940 --- /dev/null +++ b/lambda/export/sync.js @@ -0,0 +1,16 @@ +// https://docs.aws.amazon.com/kendra/latest/dg/gs-prerequisites.html +// if no IAM role allowing access to Kendra, S3 then set that up first +// TODO: add cases for creating a Kendra index & S3 bucket if it does not already exist + +const executeJava = () => { + return new Promise((resolve, reject) => { + const child = exec('java -jar createFAQ.jar', function (error, stdout, stderr) { + console.log('Value at stdout is: ' + stdout); // here you get your result. In my case I did'nt needed to pass arguments to java program. + resolve(stdout); + if (error !== null) { + console.log('exec error: ' + error); + reject(error); + } + }); + }) +} diff --git a/lambda/export/test/parseJSONTest.js b/lambda/export/test/automateSyncTest.js similarity index 55% rename from lambda/export/test/parseJSONTest.js rename to lambda/export/test/automateSyncTest.js index b86d68211..5f437bd24 100644 --- a/lambda/export/test/parseJSONTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -1,45 +1,44 @@ var assert = require('assert'); // json parsing test - async function test_parser() { const parseJSON = require('../parseJSON.js'); const csv_path = await parseJSON.handler('./test/qna_export.json', './test/qna_FAQ.csv', true); - const fs = require('fs') try { if (fs.existsSync(csv_path)) { //file exists return 'exists'; + } else { + return false; } } catch(err) { // console.error(err) - return 'false'; + return false; } // TODO: CHECK CONTENTS OF CSV ROWS TO VALIDATE FORMAT - // const csv = require('csv-parser'); - // const fs = require('fs'); - // const csv_text = []; - - // fs.createReadStream(csv_path) - // .pipe(csv()) - // .on('data', (data) => csv_text.push(data)) - // .on('end', () => { - // console.log('CSV processed'); - // // [ - // // { NAME: 'Daffy Duck', AGE: '24' }, - // // { NAME: 'Bugs Bunny', AGE: '22' } - // // ] - // }); +} + + +// create FAQ test +async function test_create_faq() { + const createFAQ = require('../createFAQ.js'); + return createFAQ.handler(); + } describe('#test automate-sync()', () => { - it('test_parser', async function() { + it('test_json_parser', async function() { let resp = await test_parser(); assert(resp, "CSV file does not exist!"); - }) + }); + + it('test_create_faq', async function() { + let resp = await test_create_faq(); + assert(resp, undefined); + }); }); From 88e779ac70667b4443b7f85c7295f8b310abeb4e Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 1 Jul 2020 18:55:40 +0000 Subject: [PATCH 006/137] debugging --- lambda/export/createFAQ.js | 11 +- lambda/export/parseJSON.js | 13 ++- lambda/export/test/qna_FAQ.csv | 21 +--- lambda/export/test/qna_export.json | 182 +++++++++-------------------- 4 files changed, 76 insertions(+), 151 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 4b0e1152f..1ebce2d6f 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -82,8 +82,11 @@ async function createFAQ() { ACL: 'private', Body: fs.createReadStream(csv_path), // use read stream option in case file is large }; - await s3Uploader(s3Client, s3_params); - + var s3_response = await s3Uploader(s3Client, s3_params); + // if (s3_response) { + // return s3_response; + // } + // create the FAQ var faq_params = { @@ -106,4 +109,6 @@ async function createFAQ() { // TODO: event and context will be triggered by the clicking on the menu item exports.handler = async() => { return createFAQ(); -} \ No newline at end of file +} + +createFAQ(); \ No newline at end of file diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index a721b74d5..6d1038a14 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -27,13 +27,14 @@ function qnaJsonParser(input_path, output_path, plainText) { const q_list = require(input_path).qna; q_list.forEach(function(elem) { - elem['q'].forEach(function(ques) { - var entry = {question:ques, answer:elem, link:''}; // entire JSON field in the answer field - var entry = {question:ques, answer:elem['a'], link:elem}; // entire JSON field in the URL field - // var entry = {question:ques, answer:elem['a'], link:'QID: ' + elem['qid']}; // QID in the URL field - // var entry = {question:ques, answer:elem['a'], link:''}; // explore changing link display text + console.log(); + console.log('ELEM IS ' + JSON.stringify(elem.q[0])); + elem.q.forEach(function(ques) { - // var entry = {question:ques, answer:elem['a'], link:''}; // original + // var ans = JSON.stringify(elem); + // var entry = {question:ques, answer: ans, link:''}; // entire JSON field in the answer field + + var entry = {question:ques, answer:elem.a, link:''}; // original data.push(entry); }); }); diff --git a/lambda/export/test/qna_FAQ.csv b/lambda/export/test/qna_FAQ.csv index bbb7d0968..0a3ad2caf 100644 --- a/lambda/export/test/qna_FAQ.csv +++ b/lambda/export/test/qna_FAQ.csv @@ -1,26 +1,17 @@ Question,Answer,Document URL (optional) -help,"I am the QnA bot, ask me a question and I will try my best to answer it.", -help me,"I am the QnA bot, ask me a question and I will try my best to answer it.", -need help,"I am the QnA bot, ask me a question and I will try my best to answer it.", -no_hits,The Kendra Fallback search was not able to identify any results, -Thumbs down,Thank you for your feedback - we will try to improve this answer., -Bad answer,Thank you for your feedback - we will try to improve this answer., Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive", How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C., -How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", -Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", -What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers., -How wide is the sun?,The Sun's diameter is 1.4 million kilometers., -Thumbs up,"Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", -Good answer,"Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", +How old is the Sun?,Our Sun is 4.6 billion years old., How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.", What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.", +What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/., What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", -How old is the Sun?,Our Sun is 4.6 billion years old., -What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/., -no_hits_orig,{{Settings.EMPTYMESSAGE}}, +How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", +Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", +What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers., +How wide is the sun?,The Sun's diameter is 1.4 million kilometers., diff --git a/lambda/export/test/qna_export.json b/lambda/export/test/qna_export.json index b1fde4418..59f5c4dd7 100644 --- a/lambda/export/test/qna_export.json +++ b/lambda/export/test/qna_export.json @@ -1,56 +1,5 @@ { "qna": [ - { - "a": "I am the QnA bot, ask me a question and I will try my best to answer it.", - "qid": "Help", - "type": "qna", - "q": [ - "help", - "help me", - "need help" - ] - }, - { - "qid": "KendraFallback", - "a": "The Kendra Fallback search was not able to identify any results", - "l": "QNA:EXTKendraFallback", - "type": "qna", - "q": [ - "no_hits" - ] - }, - { - "a": "Thank you for your feedback - we will try to improve this answer.", - "qid": "Feedback.001", - "args": [ - "incorrect" - ], - "l": "QNA:ExamplePYTHONLambdaFeedback", - "next": "", - "r": { - "subTitle": "", - "title": "", - "url": "", - "text": "", - "imageUrl": "", - "buttons": [ - { - "text": "", - "value": "" - } - ] - }, - "t": "", - "alt": { - "ssml": "", - "markdown": "" - }, - "type": "qna", - "q": [ - "Thumbs down", - "Bad answer" - ] - }, { "next": "", "args": [ @@ -78,9 +27,9 @@ { "a": "The Sun is 109 times wider than the Earth and 330,000 times as massive", "r": { - "imageUrl": "", - "title": "", - "subTitle": "" + "subTitle": "Image referenced from space-facts.com", + "imageUrl": "https://space-facts.com/wp-content/uploads/sun-size.png", + "title": "Image" }, "qid": "sun.6", "type": "qna", @@ -97,20 +46,20 @@ "r": { "buttons": [ { - "text": "", - "value": "" + "text": "How far away is the sun?", + "value": "How far away is the sun?" } ], - "subTitle": "", - "imageUrl": "", - "title": "", + "subTitle": "Image referenced from solarsystem.nasa.gov", + "imageUrl": "https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg", + "title": "Sun Image", "text": "", "url": "" }, "t": "", "alt": { - "markdown": "", - "ssml": "" + "markdown": "# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**.", + "ssml": "\nAt its centre the Sun reaches temperatures of 15 million °C\n" }, "l": "", "qid": "sun.3", @@ -124,31 +73,30 @@ "" ], "next": "", - "a": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", + "a": "Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", "r": { "buttons": [ { - "text": "", - "value": "" + "text": "How hot is the sun", + "value": "How hot is the sun" } ], "subTitle": "", "imageUrl": "", - "title": "", + "title": "Options", "text": "", "url": "" }, "t": "", "alt": { - "markdown": "", - "ssml": "" + "markdown": "## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths", + "ssml": " Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths." }, "l": "", - "qid": "sun.10", + "qid": "sun.1", "type": "qna", "q": [ - "How many planets orbit the Sun?", - "Which planets orbit the Sun?" + "Tell me about the sun?" ] }, { @@ -156,7 +104,7 @@ "" ], "next": "", - "a": "The Sun's diameter is 1.4 million kilometers.", + "a": "Our Sun is 4.6 billion years old.", "r": { "buttons": [ { @@ -176,24 +124,10 @@ "ssml": "" }, "l": "", - "qid": "sun.11", - "type": "qna", - "q": [ - "What is the diameter of the Sun?", - "How wide is the sun?" - ] - }, - { - "a": "Thank you for your positive feedback on this answer, your feedback helps us continuously improve.", - "qid": "Feedback.002", - "args": [ - "correct" - ], - "l": "QNA:ExamplePYTHONLambdaFeedback", + "qid": "sun.2", "type": "qna", "q": [ - "Thumbs up", - "Good answer" + "How old is the Sun?" ] }, { @@ -201,30 +135,30 @@ "" ], "next": "", - "a": "Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", + "a": "The sun is 149,598,262 km (1 AU) from earth.", "r": { "buttons": [ { - "text": "", - "value": "" + "text": "How large is the sun?", + "value": "How large is the sun?" } ], - "subTitle": "", - "imageUrl": "", - "title": "", - "text": "", - "url": "" + "subTitle": "Image referenced from blogspot.com", + "imageUrl": "https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif", + "title": "Map" }, "t": "", "alt": { - "markdown": "", + "markdown": "### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth", "ssml": "" }, "l": "", - "qid": "sun.1", + "qid": "sun.5", "type": "qna", + "selected": false, "q": [ - "Tell me about the sun?" + "How far away is the sun?", + "What distance is the sun from the earth?" ] }, { @@ -232,7 +166,7 @@ "" ], "next": "", - "a": "The sun is 149,598,262 km (1 AU) from earth.", + "a": "Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.", "r": { "buttons": [ { @@ -242,20 +176,19 @@ ], "subTitle": "", "imageUrl": "", - "title": "" + "title": "Options" }, "t": "", "alt": { - "markdown": "", + "markdown": "[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \"Nasa's Sun Page\") is also a great source of information on the Sun.", "ssml": "" }, "l": "", - "qid": "sun.5", + "qid": "sun.7", "type": "qna", "selected": false, "q": [ - "How far away is the sun?", - "What distance is the sun from the earth?" + "What other locations on the internet have information about the sun?" ] }, { @@ -279,8 +212,8 @@ }, "t": "", "alt": { - "markdown": "", - "ssml": "" + "markdown": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')", + "ssml": "A brief eruption of intense high-energy radiation from the sun's surface." }, "l": "", "qid": "sun.9", @@ -297,17 +230,17 @@ "" ], "next": "", - "a": "Our Sun is 4.6 billion years old.", + "a": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", "r": { "buttons": [ { - "text": "", - "value": "" + "text": "What is a solar flare?", + "value": "What is a solar flare" } ], "subTitle": "", "imageUrl": "", - "title": "", + "title": "Suggestions", "text": "", "url": "" }, @@ -317,10 +250,11 @@ "ssml": "" }, "l": "", - "qid": "sun.2", + "qid": "sun.10", "type": "qna", "q": [ - "How old is the Sun?" + "How many planets orbit the Sun?", + "Which planets orbit the Sun?" ] }, { @@ -328,17 +262,19 @@ "" ], "next": "", - "a": "Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.", + "a": "The Sun's diameter is 1.4 million kilometers.", "r": { "buttons": [ { - "text": "", - "value": "" + "text": "How many planets orbit the Sun?", + "value": "How many planets orbit the Sun" } ], "subTitle": "", "imageUrl": "", - "title": "" + "title": "Suggestions", + "text": "", + "url": "" }, "t": "", "alt": { @@ -346,19 +282,11 @@ "ssml": "" }, "l": "", - "qid": "sun.7", - "type": "qna", - "selected": false, - "q": [ - "What other locations on the internet have information about the sun?" - ] - }, - { - "qid": "CustomNoMatches", - "a": "{{Settings.EMPTYMESSAGE}}", + "qid": "sun.11", "type": "qna", "q": [ - "no_hits_orig" + "What is the diameter of the Sun?", + "How wide is the sun?" ] } ] From f369e889c6ed936848ba0c678f1a8cd2f9f68a6e Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 1 Jul 2020 22:06:33 +0000 Subject: [PATCH 007/137] framework for uploading CSV and converting to FAQ in place --- lambda/export/createFAQ.js | 107 ++++++++++++++++++++++++++++++------- lambda/export/parseJSON.js | 2 - 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 1ebce2d6f..86b685ea5 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -21,7 +21,7 @@ function s3Uploader(s3Client,params) { reject(err); } else { - console.log('Uploaded CSV to S3 successfully: '); + console.log('Uploaded CSV to S3 successfully:'); console.log(data); // successful response resolve(data); } @@ -39,14 +39,12 @@ function s3Uploader(s3Client,params) { function faqConverter(kendraClient,params) { return new Promise(function(resolve, reject) { kendraClient.createFaq(params, function(err, data) { - console.log('FAQ conversion params are : ') - console.log(params); if (err) { console.log(err, err.stack); // an error occurred reject(err); } else { - console.log('Converted CSV to FAQ successfully: '); + console.log('Converted CSV to FAQ successfully:'); console.log(data); // successful response resolve(data); } @@ -55,6 +53,52 @@ function faqConverter(kendraClient,params) { } +/** + * Function to delete old FAQ from Kendra index, return Promise + * @param kendraClient + * @param params + * @returns {*} + */ +function faqDeleter(kendraClient,params) { + return new Promise(function(resolve, reject) { + kendraClient.deleteFaq(params, function(err, data) { + if (err) { + console.log(err, err.stack); // an error occurred + reject(err); + } + else { + console.log('Deleted old FAQ successfully. New list of FAQs in index ' + params.IndexId + ':'); + console.log(data); // successful response + resolve(data); + } + }); + }); +} + + +/** + * Function to list existing FAQs in a Kendra index, return Promise + * @param kendraClient + * @param params + * @returns {*} + */ +function faqLister(kendraClient,params) { + return new Promise(function(resolve, reject) { + kendraClient.listFaqs(params, function(err, data) { + if (err) { + console.log(err, err.stack); // an error occurred + reject(err); + } + else { + console.log('Checked for pre-existing FAQ successfully. List of FAQs for index ' + params.IndexId + ':'); + console.log(data); // successful response + resolve(data); + } + }); + }); +} + + /** @@ -64,43 +108,68 @@ function faqConverter(kendraClient,params) { async function createFAQ() { let kendraClient = (process.env.REGION ? new AWSKendra({apiVersion: '2019-02-03', region: process.env.REGION}) : - new AWSKendra({apiVersion: '2019-02-03'})); + new AWSKendra({apiVersion: '2019-02-03', region: "us-east-1"}) + ); let s3Client = (process.env.REGION ? new AWSS3({apiVersion: '2006-03-01', region: process.env.REGION}) : new AWSS3({apiVersion: '2006-03-01'})); - console.log('clients created'); - // read in CSV and upload to S3 bucket - var fs = require('fs'); + // FAQ attributes // TODO: don't hardcode (also for the region) + let faq_name = 'solar-facts'; + let faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; let csv_path = './test/qna_FAQ.csv'; + + // read in CSV and upload to S3 bucket + var fs = require('fs'); var s3_params = { Bucket: 'explore-kendra-solar', Key: 'qna_FAQ.csv', - ACL: 'private', + ACL: 'private', // TODO: do we want this to be publicly readable by linking QID's? Body: fs.createReadStream(csv_path), // use read stream option in case file is large }; var s3_response = await s3Uploader(s3Client, s3_params); - // if (s3_response) { - // return s3_response; - // } - + + + // if FAQ exists already: then the old one and update it + var index_params = { + IndexId: faq_index_id, + MaxResults: '30' // default max number of FAQs in developer edition + // NextToken: '' // TODO: for when the number of FAQs goes over a page... + }; + // TODO: this errors when there are no FAQs already + var list_faq_response = await faqLister(kendraClient, index_params); + // TODO: does it always have this field? + var j, elem, index=null; + for (j=0; j { return createFAQ(); } - -createFAQ(); \ No newline at end of file diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 6d1038a14..44736e5db 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -27,8 +27,6 @@ function qnaJsonParser(input_path, output_path, plainText) { const q_list = require(input_path).qna; q_list.forEach(function(elem) { - console.log(); - console.log('ELEM IS ' + JSON.stringify(elem.q[0])); elem.q.forEach(function(ques) { // var ans = JSON.stringify(elem); From 1bf170a98f64b10ce5b9ac10527e7961e9595eb8 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 1 Jul 2020 22:12:38 +0000 Subject: [PATCH 008/137] cleanup --- lambda/export/createFAQ.js | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 86b685ea5..1a986a157 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -106,9 +106,21 @@ function faqLister(kendraClient,params) { * @returns {*} */ async function createFAQ() { + + // FAQ attributes // TODO: don't hardcode (also for the region) + let faq_name = 'solar-facts'; + let faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; + let csv_path = './test/qna_FAQ.csv'; + let csv_name = 'qna_FAQ.csv'; + let s3_bucket = 'explore-kendra-solar'; + let kendra_s3_access_role = 'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer'; + let region = 'us-east-1'; + + + // create kendra and s3 clients let kendraClient = (process.env.REGION ? new AWSKendra({apiVersion: '2019-02-03', region: process.env.REGION}) : - new AWSKendra({apiVersion: '2019-02-03', region: "us-east-1"}) + new AWSKendra({apiVersion: '2019-02-03', region: region}) ); let s3Client = (process.env.REGION ? new AWSS3({apiVersion: '2006-03-01', region: process.env.REGION}) : @@ -116,32 +128,25 @@ async function createFAQ() { console.log('clients created'); - // FAQ attributes // TODO: don't hardcode (also for the region) - let faq_name = 'solar-facts'; - let faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; - let csv_path = './test/qna_FAQ.csv'; - - // read in CSV and upload to S3 bucket var fs = require('fs'); var s3_params = { - Bucket: 'explore-kendra-solar', - Key: 'qna_FAQ.csv', + Bucket: s3_bucket, + Key: csv_name, ACL: 'private', // TODO: do we want this to be publicly readable by linking QID's? Body: fs.createReadStream(csv_path), // use read stream option in case file is large }; var s3_response = await s3Uploader(s3Client, s3_params); - // if FAQ exists already: then the old one and update it + // if FAQ exists already, delete the old one and update it var index_params = { IndexId: faq_index_id, MaxResults: '30' // default max number of FAQs in developer edition // NextToken: '' // TODO: for when the number of FAQs goes over a page... }; - // TODO: this errors when there are no FAQs already + // TODO: this errors when there are no FAQs already (empty arrays --> throttling exceptions) var list_faq_response = await faqLister(kendraClient, index_params); - // TODO: does it always have this field? var j, elem, index=null; for (j=0; j Date: Thu, 2 Jul 2020 15:15:36 +0000 Subject: [PATCH 009/137] adding json structure of specific question to URL field of FAQ csv so it can be used for a query response during the alternative engine integration --- lambda/export/createFAQ.js | 7 ++----- lambda/export/parseJSON.js | 6 +++--- .../KendraFallback/KendraFallback.js | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 1a986a157..008d8a4b4 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -1,7 +1,5 @@ // createFAQ.js -// TODO: don't hardcode - const AWSKendra = require('aws-sdk/clients/kendra'); const AWSS3 = require('aws-sdk/clients/s3'); @@ -107,8 +105,8 @@ function faqLister(kendraClient,params) { */ async function createFAQ() { - // FAQ attributes // TODO: don't hardcode (also for the region) - let faq_name = 'solar-facts'; + // FAQ attributes // TODO: don't hardcode + let faq_name = 'qna-facts'; let faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; let csv_path = './test/qna_FAQ.csv'; let csv_name = 'qna_FAQ.csv'; @@ -145,7 +143,6 @@ async function createFAQ() { MaxResults: '30' // default max number of FAQs in developer edition // NextToken: '' // TODO: for when the number of FAQs goes over a page... }; - // TODO: this errors when there are no FAQs already (empty arrays --> throttling exceptions) var list_faq_response = await faqLister(kendraClient, index_params); var j, elem, index=null; for (j=0; j'}; // entire JSON field in the answer field + var json_doc = JSON.stringify(elem); + var entry = {question:ques, answer:elem.a, link:json_doc}; // entire JSON structure in URL field - var entry = {question:ques, answer:elem.a, link:''}; // original + // var entry = {question:ques, answer:elem.a, link:''}; // original data.push(entry); }); }); diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index c582f7365..dce2f5fe6 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -81,6 +81,25 @@ function mergeIntervals(intervals) { } + +/** Function that returns if a string has JSON structure + * @param str - input string + * @returns boolean true or false + */ +function hasJsonStructure(str) { + if (typeof str !== 'string') return false; + try { + const result = JSON.parse(str); + const type = Object.prototype.toString.call(result); + return type === '[object Object]' + || type === '[object Array]'; + } catch (err) { + return false; + } +} + + + /** Function that processes kendra requests and handles response. Decides whether to handle SNS * events or Lambda Hook events from QnABot. * @param event - input event passed to the Lambda Handler @@ -232,6 +251,8 @@ async function routeKendraRequest(event, context) { kendraResultId = element.Id; // store off resultId to use as a session attribute for feedback foundAnswerCount++; + // TODO: use json structure for query response from doc URL field + } else if (element.Type === 'DOCUMENT' && element.DocumentExcerpt.Text && element.DocumentURI) { const docInfo = {} From da9fbc9875a9ae887531efd8aa5bb7735f867dee Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 2 Jul 2020 16:20:07 +0000 Subject: [PATCH 010/137] json in the url field completed --- lambda/export/test/qna_FAQ.csv | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lambda/export/test/qna_FAQ.csv b/lambda/export/test/qna_FAQ.csv index 0a3ad2caf..394b6c61d 100644 --- a/lambda/export/test/qna_FAQ.csv +++ b/lambda/export/test/qna_FAQ.csv @@ -1,17 +1,17 @@ Question,Answer,Document URL (optional) -Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", -How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive", -How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C., -Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", -How old is the Sun?,Our Sun is 4.6 billion years old., -How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.", -What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.", -What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/., -What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", -what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", -Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", -I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", -How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", -Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", -What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers., -How wide is the sun?,The Sun's diameter is 1.4 million kilometers., +Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.","{""next"":"""",""args"":[""""],""a"":""When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant."",""r"":{""subTitle"":"""",""imageUrl"":"""",""title"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.4"",""type"":""qna"",""selected"":false,""q"":[""Will the Sun ever consume the earth?""]}" +How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive","{""a"":""The Sun is 109 times wider than the Earth and 330,000 times as massive"",""r"":{""subTitle"":""Image referenced from space-facts.com"",""imageUrl"":""https://space-facts.com/wp-content/uploads/sun-size.png"",""title"":""Image""},""qid"":""sun.6"",""type"":""qna"",""q"":[""How large is the sun?""]}" +How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C.,"{""args"":[""""],""next"":"""",""a"":""At its centre the Sun reaches temperatures of 15 million °C."",""r"":{""buttons"":[{""text"":""How far away is the sun?"",""value"":""How far away is the sun?""}],""subTitle"":""Image referenced from solarsystem.nasa.gov"",""imageUrl"":""https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg"",""title"":""Sun Image"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**."",""ssml"":""\nAt its centre the Sun reaches temperatures of 15 million °C\n""},""l"":"""",""qid"":""sun.3"",""type"":""qna"",""q"":[""How hot is it at the center of the Sun?""]}" +Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!","{""args"":[""""],""next"":"""",""a"":""Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!"",""r"":{""buttons"":[{""text"":""How hot is the sun"",""value"":""How hot is the sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths"",""ssml"":"" Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths.""},""l"":"""",""qid"":""sun.1"",""type"":""qna"",""q"":[""Tell me about the sun?""]}" +How old is the Sun?,Our Sun is 4.6 billion years old.,"{""args"":[""""],""next"":"""",""a"":""Our Sun is 4.6 billion years old."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.2"",""type"":""qna"",""q"":[""How old is the Sun?""]}" +How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" +What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" +What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.,"{""args"":[""""],""next"":"""",""a"":""Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options""},""t"":"""",""alt"":{""markdown"":""[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \""Nasa's Sun Page\"") is also a great source of information on the Sun."",""ssml"":""""},""l"":"""",""qid"":""sun.7"",""type"":""qna"",""selected"":false,""q"":[""What other locations on the internet have information about the sun?""]}" +What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" +Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" +What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" +How wide is the sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" From b1d863de058f3022fa1a8a038d591f4cc0b16794 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 2 Jul 2020 16:24:50 +0000 Subject: [PATCH 011/137] added comments --- .../KendraFallback/KendraFallback.js | 3 +- .../KendraFallback/test/context_FAQ_md.json | 10 ++-- .../KendraFallback/test/event_FAQ_md.json | 42 ++++++--------- .../test/kendraData_FAQ_md.json | 18 +++---- .../KendraFallback/test/run_test.js | 2 +- .../KendraFallback/test/sample_return.json | 52 ++++++++----------- 6 files changed, 56 insertions(+), 71 deletions(-) diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index dce2f5fe6..04b021484 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -252,6 +252,7 @@ async function routeKendraRequest(event, context) { foundAnswerCount++; // TODO: use json structure for query response from doc URL field + // var json_struct = JSON.parse(element.DocumentURI); } else if (element.Type === 'DOCUMENT' && element.DocumentExcerpt.Text && element.DocumentURI) { @@ -327,7 +328,7 @@ async function routeKendraRequest(event, context) { event.res.session.kendraIndexId = kendraIndexId; event.res.session.kendraResultId = kendraResultId; } - console.log("final return: " + JSON.stringify(event,null,2)); + // console.log("final return: " + JSON.stringify(event,null,2)); return event; } diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/context_FAQ_md.json b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/context_FAQ_md.json index fe15158c2..9fa21fa26 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/context_FAQ_md.json +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/context_FAQ_md.json @@ -1,10 +1,10 @@ { "callbackWaitsForEmptyEventLoop": true, "functionVersion": "$LATEST", - "functionName": "QNAbot-dev-dev-dev-master-4-Exam-EXTKendraFallback-T1KJ602HXNIW", + "functionName": "QNA-dev-dev-dev-master-2-Example-EXTKendraFallback-1NKAGNPBF6VVY", "memoryLimitInMB": "2048", - "logGroupName": "/aws/lambda/QNAbot-dev-dev-dev-master-4-Exam-EXTKendraFallback-T1KJ602HXNIW", - "logStreamName": "2020/06/23/[$LATEST]9b1c9c2803c64f3db0ee8d14e6ac22d6", - "invokedFunctionArn": "arn:aws:lambda:us-east-1:425742325899:function:QNAbot-dev-dev-dev-master-4-Exam-EXTKendraFallback-T1KJ602HXNIW", - "awsRequestId": "9ab26fab-6b8c-4c14-8287-cbc8a48179f3" + "logGroupName": "/aws/lambda/QNA-dev-dev-dev-master-2-Example-EXTKendraFallback-1NKAGNPBF6VVY", + "logStreamName": "2020/07/02/[$LATEST]e5eaf1b6bade474a82aa21a378b9a71f", + "invokedFunctionArn": "arn:aws:lambda:us-east-1:425742325899:function:QNA-dev-dev-dev-master-2-Example-EXTKendraFallback-1NKAGNPBF6VVY", + "awsRequestId": "a0665572-c2b8-424f-a22e-502f6bb317cb" } diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/event_FAQ_md.json b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/event_FAQ_md.json index 11d45b59c..3c6c071e4 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/event_FAQ_md.json +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/event_FAQ_md.json @@ -3,17 +3,17 @@ "_event": { "messageVersion": "1.0", "invocationSource": "FulfillmentCodeHook", - "userId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "userId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "sessionAttributes": {}, "requestAttributes": null, "bot": { - "name": "QNAbot_dev_dev_dev_master_four_BotoNDIv", + "name": "QNA_dev_dev_dev_master_two_BotUSFgl", "alias": "live", "version": "1" }, "outputDialogMode": "Text", "currentIntent": { - "name": "fulfilment_IntentrDEldIYchy", + "name": "fulfilment_IntentfiuhxvkSil", "slots": { "slot": "How many planets orbit the sun" }, @@ -29,7 +29,6 @@ "recentIntentSummaryView": null, "sentimentResponse": null, "kendraResponse": null, - "origQuestion": "How many planets orbit the sun?", "errorFound": false }, "_settings": { @@ -50,7 +49,7 @@ "SMS_HINT_REMINDER": " (Feedback? Reply THUMBS UP or THUMBS DOWN. Ask HELP ME at any time)", "SMS_HINT_REMINDER_INTERVAL_HRS": "24", "IDENTITY_PROVIDER_JWKS_URLS": [], - "ENABLE_MULTI_LANGUAGE_SUPPORT": "true", + "ENABLE_MULTI_LANGUAGE_SUPPORT": "false", "MINIMUM_CONFIDENCE_SCORE": 0.6, "ALT_SEARCH_KENDRA_INDEXES": "[\"e1c23860-e5c8-4409-ae26-b05bd6ced00a\"]", "ELICIT_RESPONSE_MAX_RETRIES": 3, @@ -59,30 +58,26 @@ "ELICIT_RESPONSE_DEFAULT_MSG": "Ok. ", "ENABLE_REDACTING": "false", "REDACTING_REGEX": "\\b\\d{4}\\b(?![-])|\\b\\d{9}\\b|\\b\\d{3}-\\d{2}-\\d{4}\\b", - "DEFAULT_USER_POOL_JWKS_URL": "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_EZg5Ia82Z/.well-known/jwks.json" + "DEFAULT_USER_POOL_JWKS_URL": "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_jt2cXkCEb/.well-known/jwks.json" }, "_type": "LEX", - "_userId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "_userId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "question": "How many planets orbit the sun?", - "session": { - "userDetectedLocale": "en", - "userDetectedLocaleConfidence": 0.9948722720146179, - "userLocale": "en" - }, + "session": {}, "_preferredResponseType": "PlainText", "_userInfo": { - "UserId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "UserId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "InteractionCount": 0, - "TimeSinceLastInteraction": 1592902513.847 + "TimeSinceLastInteraction": 1593663418.209 }, "_info": { "es": { - "address": "search-qnabot-elasti-4n2sjlh1tlwf-ziyzyejzynvs6rs2cvx7yaqxma.us-east-1.es.amazonaws.com", - "index": "qnabot-dev-dev-dev-master-4", + "address": "search-qna-dev-elasti-mtl6ojldf3ja-w5begr25vpfuyidnlgb4izlbtq.us-east-1.es.amazonaws.com", + "index": "qna-dev-dev-dev-master-2", "type": "qna", "service": { - "qid": "QNAbot-dev-dev-dev-master-4-ESQidLambda-636ARD1AGUJH", - "proxy": "QNAbot-dev-dev-dev-master-4-ESProxyLambda-NF5QMNB8RE00" + "qid": "QNA-dev-dev-dev-master-2-ESQidLambda-1I9JEHU6U4K9I", + "proxy": "QNA-dev-dev-dev-master-2-ESProxyLambda-1NMAY6LNBRJDY" } } } @@ -91,9 +86,6 @@ "type": "PlainText", "message": "The Kendra Fallback search was not able to identify any results", "session": { - "userDetectedLocale": "en", - "userDetectedLocaleConfidence": 0.9948722720146179, - "userLocale": "en", "appContext": { "altMessages": {} }, @@ -116,11 +108,11 @@ "url": "" }, "_userInfo": { - "UserId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "UserId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "InteractionCount": 1, - "TimeSinceLastInteraction": 1592902513.847, - "FirstSeen": "Tue Jun 23 2020 20:55:13 GMT+0000 (Coordinated Universal Time)", - "LastSeen": "Tue Jun 23 2020 20:55:13 GMT+0000 (Coordinated Universal Time)" + "TimeSinceLastInteraction": 1593663418.209, + "FirstSeen": "Thu Jul 02 2020 16:16:58 GMT+0000 (Coordinated Universal Time)", + "LastSeen": "Thu Jul 02 2020 16:16:58 GMT+0000 (Coordinated Universal Time)" }, "got_hits": 0, "result": { diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/kendraData_FAQ_md.json b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/kendraData_FAQ_md.json index ab0c63db0..c2d3098da 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/kendraData_FAQ_md.json +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/kendraData_FAQ_md.json @@ -1,8 +1,8 @@ { - "QueryId": "ec35a915-8990-4ef7-a5da-f490911cad42", + "QueryId": "b904542d-af85-447d-808b-d86b5df16c06", "ResultItems": [ { - "Id": "ec35a915-8990-4ef7-a5da-f490911cad42-2db7aa5b-1fcb-4910-a2fc-b3922a6069e3", + "Id": "b904542d-af85-447d-808b-d86b5df16c06-ee0db51d-15f4-4786-9144-b51d29d37f66", "Type": "QUESTION_ANSWER", "AdditionalAttributes": [ { @@ -10,7 +10,7 @@ "ValueType": "TEXT_WITH_HIGHLIGHTS_VALUE", "Value": { "TextWithHighlightsValue": { - "Text": "How many planets orbit the Sun", + "Text": "How many planets orbit the Sun?", "Highlights": [ { "BeginOffset": 4, @@ -41,7 +41,7 @@ "ValueType": "TEXT_WITH_HIGHLIGHTS_VALUE", "Value": { "TextWithHighlightsValue": { - "Text": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto", + "Text": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", "Highlights": [ { "BeginOffset": 6, @@ -63,26 +63,26 @@ } } ], - "DocumentId": "c8e0762c86ee7fc92f253ce925b1ecbd101479541164cc5afac09fb027c659dd1592335726833", + "DocumentId": "a212c9bbe37131a9777efc915d837915ee33e5b327ec7df4645cb96d6d165f4a1593702980960", "DocumentTitle": { "Text": "" }, "DocumentExcerpt": { - "Text": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto", + "Text": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", "Highlights": [ { "BeginOffset": 0, - "EndOffset": 146, + "EndOffset": 147, "TopAnswer": false } ] }, - "DocumentURI": "", + "DocumentURI": "{\"args\":[\"\"],\"next\":\"\",\"a\":\"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.\",\"r\":{\"buttons\":[{\"text\":\"What is a solar flare?\",\"value\":\"What is a solar flare\"}],\"subTitle\":\"\",\"imageUrl\":\"\",\"title\":\"Suggestions\",\"text\":\"\",\"url\":\"\"},\"t\":\"\",\"alt\":{\"markdown\":\"\",\"ssml\":\"\"},\"l\":\"\",\"qid\":\"sun.10\",\"type\":\"qna\",\"q\":[\"How many planets orbit the Sun?\",\"Which planets orbit the Sun?\"]}", "DocumentAttributes": [ { "Key": "_source_uri", "Value": { - "StringValue": "" + "StringValue": "{\"args\":[\"\"],\"next\":\"\",\"a\":\"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.\",\"r\":{\"buttons\":[{\"text\":\"What is a solar flare?\",\"value\":\"What is a solar flare\"}],\"subTitle\":\"\",\"imageUrl\":\"\",\"title\":\"Suggestions\",\"text\":\"\",\"url\":\"\"},\"t\":\"\",\"alt\":{\"markdown\":\"\",\"ssml\":\"\"},\"l\":\"\",\"qid\":\"sun.10\",\"type\":\"qna\",\"q\":[\"How many planets orbit the Sun?\",\"Which planets orbit the Sun?\"]}" } } ] diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js index 0dfafbe6a..47b79b6ef 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/run_test.js @@ -34,7 +34,7 @@ describe('#test_kendra_highlights()', () => { // tests that in markdown format, highlights are boldened assert.equal(resp.res.session.appContext.altMessages.markdown, - "*Answer from Amazon Kendra FAQ.* \n \n\nEight **planets** **orbit** the **sun**. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto"); + "*Answer from Amazon Kendra FAQ.* \n \n\nEight **planets** **orbit** the **sun**. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."); }); it('test_top_answer', async function() { let resp = await test_top_ans(); diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/sample_return.json b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/sample_return.json index 64be59056..17364c5c4 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/sample_return.json +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/test/sample_return.json @@ -3,17 +3,17 @@ "_event": { "messageVersion": "1.0", "invocationSource": "FulfillmentCodeHook", - "userId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "userId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "sessionAttributes": {}, "requestAttributes": null, "bot": { - "name": "QNAbot_dev_dev_dev_master_four_BotoNDIv", + "name": "QNA_dev_dev_dev_master_two_BotUSFgl", "alias": "live", "version": "1" }, "outputDialogMode": "Text", "currentIntent": { - "name": "fulfilment_IntentrDEldIYchy", + "name": "fulfilment_IntentfiuhxvkSil", "slots": { "slot": "How many planets orbit the sun" }, @@ -29,7 +29,6 @@ "recentIntentSummaryView": null, "sentimentResponse": null, "kendraResponse": null, - "origQuestion": "How many planets orbit the sun?", "errorFound": false }, "_settings": { @@ -50,7 +49,7 @@ "SMS_HINT_REMINDER": " (Feedback? Reply THUMBS UP or THUMBS DOWN. Ask HELP ME at any time)", "SMS_HINT_REMINDER_INTERVAL_HRS": "24", "IDENTITY_PROVIDER_JWKS_URLS": [], - "ENABLE_MULTI_LANGUAGE_SUPPORT": "true", + "ENABLE_MULTI_LANGUAGE_SUPPORT": "false", "MINIMUM_CONFIDENCE_SCORE": 0.6, "ALT_SEARCH_KENDRA_INDEXES": "[\"e1c23860-e5c8-4409-ae26-b05bd6ced00a\"]", "ELICIT_RESPONSE_MAX_RETRIES": 3, @@ -59,45 +58,38 @@ "ELICIT_RESPONSE_DEFAULT_MSG": "Ok. ", "ENABLE_REDACTING": "false", "REDACTING_REGEX": "\\b\\d{4}\\b(?![-])|\\b\\d{9}\\b|\\b\\d{3}-\\d{2}-\\d{4}\\b", - "DEFAULT_USER_POOL_JWKS_URL": "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_EZg5Ia82Z/.well-known/jwks.json" + "DEFAULT_USER_POOL_JWKS_URL": "https://cognito-idp.us-east-1.amazonaws.com/us-east-1_jt2cXkCEb/.well-known/jwks.json" }, "_type": "LEX", - "_userId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "_userId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "question": "How many planets orbit the sun?", - "session": { - "userDetectedLocale": "en", - "userDetectedLocaleConfidence": 0.9948722720146179, - "userLocale": "en" - }, + "session": {}, "_preferredResponseType": "PlainText", "_userInfo": { - "UserId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "UserId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "InteractionCount": 0, - "TimeSinceLastInteraction": 1592902513.847 + "TimeSinceLastInteraction": 1593663418.209 }, "_info": { "es": { - "address": "search-qnabot-elasti-4n2sjlh1tlwf-ziyzyejzynvs6rs2cvx7yaqxma.us-east-1.es.amazonaws.com", - "index": "qnabot-dev-dev-dev-master-4", + "address": "search-qna-dev-elasti-mtl6ojldf3ja-w5begr25vpfuyidnlgb4izlbtq.us-east-1.es.amazonaws.com", + "index": "qna-dev-dev-dev-master-2", "type": "qna", "service": { - "qid": "QNAbot-dev-dev-dev-master-4-ESQidLambda-636ARD1AGUJH", - "proxy": "QNAbot-dev-dev-dev-master-4-ESProxyLambda-NF5QMNB8RE00" + "qid": "QNA-dev-dev-dev-master-2-ESQidLambda-1I9JEHU6U4K9I", + "proxy": "QNA-dev-dev-dev-master-2-ESProxyLambda-1NMAY6LNBRJDY" } } } }, "res": { "type": "PlainText", - "message": "Answer from Amazon Kendra FAQ.\n\n Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto", + "message": "Answer from Amazon Kendra FAQ.\n\n Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", "session": { - "userDetectedLocale": "en", - "userDetectedLocaleConfidence": 0.9948722720146179, - "userLocale": "en", "appContext": { "altMessages": { - "markdown": "*Answer from Amazon Kendra FAQ.* \n \n\n Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto", - "ssml": " Answer from Amazon Kendra FAQ. Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune " + "markdown": "*Answer from Amazon Kendra FAQ.* \n \n\nEight **planets** **orbit** the **sun**. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", + "ssml": " Answer from Amazon Kendra FAQ. Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto " } }, "previous": { @@ -112,9 +104,9 @@ "hasParent": true }, "kendraResponsibleQid": "KendraFallback", - "kendraQueryId": "ec35a915-8990-4ef7-a5da-f490911cad42", + "kendraQueryId": "b904542d-af85-447d-808b-d86b5df16c06", "kendraIndexId": "e1c23860-e5c8-4409-ae26-b05bd6ced00a", - "kendraResultId": "ec35a915-8990-4ef7-a5da-f490911cad42-2db7aa5b-1fcb-4910-a2fc-b3922a6069e3" + "kendraResultId": "b904542d-af85-447d-808b-d86b5df16c06-ee0db51d-15f4-4786-9144-b51d29d37f66" }, "card": { "send": false, @@ -123,11 +115,11 @@ "url": "" }, "_userInfo": { - "UserId": "us-east-1:05aef6ae-bc99-41a5-98ad-3d74df5b4bc1", + "UserId": "us-east-1:7ace124d-bfd1-4356-ac5a-ed8a98785fd9", "InteractionCount": 1, - "TimeSinceLastInteraction": 1592902513.847, - "FirstSeen": "Tue Jun 23 2020 20:55:13 GMT+0000 (Coordinated Universal Time)", - "LastSeen": "Tue Jun 23 2020 20:55:13 GMT+0000 (Coordinated Universal Time)" + "TimeSinceLastInteraction": 1593663418.209, + "FirstSeen": "Thu Jul 02 2020 16:16:58 GMT+0000 (Coordinated Universal Time)", + "LastSeen": "Thu Jul 02 2020 16:16:58 GMT+0000 (Coordinated Universal Time)" }, "got_hits": 0, "result": { From aa8efb1523b7acd608c10cd8cd02d7e3b859eb9d Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 7 Jul 2020 04:41:50 +0000 Subject: [PATCH 012/137] adding export trigger --- lambda/export/ExportJSON.js | 110 ++++++++++ lambda/export/createFAQ.js | 23 +- lambda/export/kendraSync.js | 51 +++++ lambda/export/parseJSON.js | 13 +- lambda/export/sync.js | 16 -- lambda/export/test/automateSyncTest.js | 40 +++- lambda/export/test/qna_FAQ.csv | 26 +-- lambda/export/test/qna_export.json | 278 ++----------------------- templates/export/bucket.js | 31 +++ templates/export/index.js | 3 +- templates/export/resources.js | 51 +++++ templates/public/index.js | 3 +- 12 files changed, 329 insertions(+), 316 deletions(-) create mode 100644 lambda/export/ExportJSON.js create mode 100644 lambda/export/kendraSync.js delete mode 100644 lambda/export/sync.js diff --git a/lambda/export/ExportJSON.js b/lambda/export/ExportJSON.js new file mode 100644 index 000000000..67e783e02 --- /dev/null +++ b/lambda/export/ExportJSON.js @@ -0,0 +1,110 @@ +var aws=require('./util/aws') +var Promise=require('bluebird') +var s3=new aws.S3() + + +async function waitForExport(s3params, timeout){ + console.log("Checking the status of export") + var now = Date.now(); + var stoptime = now + timeout ; + var complete = false; + var timedout = false ; + var body; + do { + await new Promise(resolve => setTimeout(resolve, 1000)); + var res = await s3.getObject(s3params).promise() ; + body = JSON.parse(res.Body.toString()); + console.log(body.status); + complete = (body.status == "Completed") ? true : false ; + timedout = (Date.now() > stoptime) ? true : false ; + } while( !complete && !timedout ); + if (!complete && timedout ) { + console.log("Timed out."); + } + return complete; +} + + +async function run_export(params,reply) { + const ID="kendraSyncExport"; + var data={ + "bucket":params.bucket, + "index":params.index, + "faq_index_id":params.faq_index_id, + "id":params.id, + "config":"kendra-status/"+params.id, + "tmp":"kendra-tmp/"+params.id, + "key":"kendra-data/"+params.id, + "filter":"", + "status":"Started" + }; + var s3params = { + Bucket: data.bucket, + Key: data.config, + Body: JSON.stringify(data), + }; + const statusfile=data.bucket+'/'+data.config ; + console.log("Running content export to sync to Kendra FAQ."); + + // TODO: what happens if bucket is not yet created? + // Create object in export bucket to trigger export lambda + var res = await s3.putObject(s3params).promise() + console.log("Wait up to 60 seconds for status to be completed"); + delete s3params.Body; + var complete = await waitForExport(s3params,60000); + if (complete) { + console.log("Export completed: ",statusfile); + reply(null,ID) ; + } else { + console.log("Export did NOT complete: ",statusfile); + reply(null,ID) ; + } +} + +module.exports=class KendraSyncExport { + constructor(){ + } + + // converts JSON to CSV to Kendra FAQ + Convert(params) { + const statusfile = params.bucket+"/kendra-status/"+params.id; + + console.log("Beginning parsing of JSON into CSV", statusfile); + params.input_path = params.bucket+'/'+params.key; // s3 bucket location...TODO: read from s3 bucket + params.csv_name = 'qna_FAQ.csv'; + params.output = './test/' + params.csv_name; // TODO: put into the s3 bucket! + require('../parseJSON.js').handler(params); + console.log("Completed parsing of JSON into CSV", statusfile); + + + console.log("Beginning conversion of CSV into Kendra FAQ", statusfile); + params.faq_name = 'qna-facts'; + params.faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; + params.csv_path = params.output; + params.s3_bucket = 'explore-kendra-solar'; + params.kendra_s3_access_role = 'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer'; + params.region = 'us-east-1'; + require('../createFAQ.js').handler(params); + console.log("Completed", statusfile); + } + + Export(params,reply){ + run_export(params,reply) + const statusfile = params.bucket+"/kendra-status/"+params.id; + console.log("JSON export completed, now parsing into CSV", statusfile); + // the lambda will have to find the status file and add trigger to CloudFormation + // if I know the ID it is at the ending part of this key, tack on kendra-status + // will know the bucket/key for the source JSON file + + // TODO: this should trigger parseJSON + this.Convert(params); + + // TODO: confirm that the UI is polling the status file for completion + // important to write the status file at the end of all the phases signaling completion + // challenge: finding status file and updating status... given that we will pass the job file + + } + + +}; + diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 008d8a4b4..4cd3d28a6 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -103,16 +103,16 @@ function faqLister(kendraClient,params) { * Function to upload CSV into S3 bucket and convert into Kendra FAQ, return Promise * @returns {*} */ -async function createFAQ() { +async function createFAQ(params) { // FAQ attributes // TODO: don't hardcode - let faq_name = 'qna-facts'; - let faq_index_id = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; - let csv_path = './test/qna_FAQ.csv'; - let csv_name = 'qna_FAQ.csv'; - let s3_bucket = 'explore-kendra-solar'; - let kendra_s3_access_role = 'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer'; - let region = 'us-east-1'; + let faq_name = params.faq_name; //'qna-facts'; + let faq_index_id = params.faq_index_id; //'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; + let csv_path = params.csv_path; //'./test/qna_FAQ.csv'; + let csv_name = params.csv_name; //'qna_FAQ.csv'; + let s3_bucket = params.s3_bucket; //'explore-kendra-solar'; + let kendra_s3_access_role = params.kendra_s3_access_role; //'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer'; + let region = params.region; //'us-east-1'; // create kendra and s3 clients @@ -141,7 +141,6 @@ async function createFAQ() { var index_params = { IndexId: faq_index_id, MaxResults: '30' // default max number of FAQs in developer edition - // NextToken: '' // TODO: for when the number of FAQs goes over a page... }; var list_faq_response = await faqLister(kendraClient, index_params); var j, elem, index=null; @@ -171,7 +170,7 @@ async function createFAQ() { Key: csv_name }, Description: 'Exported FAQ of questions from QnABot designer console' - // if no tags, just comment it out because empty arrays cause throttling exceptions + // if no tags, delete parameter because empty arrays cause throttling exceptions }; var faq_response = await faqConverter(kendraClient, faq_params); @@ -179,6 +178,6 @@ async function createFAQ() { } // TODO: event and context will be triggered by the clicking on the menu item -exports.handler = async() => { - return createFAQ(); +exports.handler = async(params) => { + return createFAQ(params); } diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js new file mode 100644 index 000000000..bd4c9d5fa --- /dev/null +++ b/lambda/export/kendraSync.js @@ -0,0 +1,51 @@ +var Promise=require('bluebird') +var aws=require("aws-sdk") +aws.config.setPromisesDependency(Promise) +aws.config.region=process.env.AWS_REGION + +var s3=new aws.S3() +var lambda=new aws.Lambda() +var stride=parseInt(process.env.STRIDE) +var _=require('lodash') +var start=require('./lib/start') +var step=require('./lib/step') +var join=require('./lib/join') +var clean=require('./lib/clean') + +exports.performKendraSync=function(event,context,cb){ + console.log("step") + console.log("Request",JSON.stringify(event,null,2)) + var Bucket=event.Records[0].s3.bucket.name + var Key=decodeURI(event.Records[0].s3.object.key) + var VersionId=_.get(event,"Records[0].s3.object.versionId") + console.log(Bucket,Key) + + s3.waitFor('objectExists',{Bucket,Key,VersionId}).promise() + .then(()=>s3.getObject({Bucket,Key,VersionId}).promise()) + .then(x=>JSON.parse(x.Body.toString())) + .then(function(config){ + if(config.status!=="Error" && config.status!=="Completed"){ + return Promise.try(function(){ + console.log("Config:",JSON.stringify(config,null,2)) + switch(config.status){ + case 'Started': + return start(config); + case 'InProgress': + return step(config); + case "Join": + return join(config); + case "Clean": + return clean(config); + } + }) + .catch(error=>{ + console.log(error) + config.status="Error" + config.message=_.get(error,'message',JSON.stringify(error)) + }) + .then(()=>s3.putObject({Bucket,Key,Body:JSON.stringify(config)}).promise()) + } + }) + .catch(cb) + // TODO: another trigger based on the second s3 bucket... +} diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 862ee0203..563ce1483 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -4,15 +4,9 @@ * Function to parse JSON of configurations/questions from QNA Content Designer and write an output CSV file * @param input_path : the input file path of the exported JSON * @param output_path : the output file path to write the CSV - * @param plainText : a boolean to write the standard Q&A format or the JSON meta-data format * @returns output_path */ -function qnaJsonParser(input_path, output_path, plainText) { - // sets default behavior to construct a plain text FAQ which does NOT string-ify the entire JSON field into the 'Answer' column - if (plainText == undefined) { - plainText = true; - } - +function qnaJsonParser(input_path, output_path) { const createCsvWriter = require('./node_modules/csv-writer').createObjectCsvWriter; const csvWriter = createCsvWriter({ path: output_path, @@ -32,7 +26,6 @@ function qnaJsonParser(input_path, output_path, plainText) { var json_doc = JSON.stringify(elem); var entry = {question:ques, answer:elem.a, link:json_doc}; // entire JSON structure in URL field - // var entry = {question:ques, answer:elem.a, link:''}; // original data.push(entry); }); }); @@ -44,6 +37,6 @@ function qnaJsonParser(input_path, output_path, plainText) { return output_path; } -exports.handler = async (input, output, plainText) => { - return qnaJsonParser(input, output, plainText); +exports.handler = async (params) => { + return qnaJsonParser(params.input_path, params.output); }; \ No newline at end of file diff --git a/lambda/export/sync.js b/lambda/export/sync.js deleted file mode 100644 index 1547c5940..000000000 --- a/lambda/export/sync.js +++ /dev/null @@ -1,16 +0,0 @@ -// https://docs.aws.amazon.com/kendra/latest/dg/gs-prerequisites.html -// if no IAM role allowing access to Kendra, S3 then set that up first -// TODO: add cases for creating a Kendra index & S3 bucket if it does not already exist - -const executeJava = () => { - return new Promise((resolve, reject) => { - const child = exec('java -jar createFAQ.jar', function (error, stdout, stderr) { - console.log('Value at stdout is: ' + stdout); // here you get your result. In my case I did'nt needed to pass arguments to java program. - resolve(stdout); - if (error !== null) { - console.log('exec error: ' + error); - reject(error); - } - }); - }) -} diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index 5f437bd24..5a7f5b946 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -26,7 +26,40 @@ async function test_parser() { async function test_create_faq() { const createFAQ = require('../createFAQ.js'); return createFAQ.handler(); +} + +async function test_trigger_export() { + const triggerExport = require('../ExportJSON.js'); + const KendraSyncExport = triggerExport.KendraSyncExport(); + + var data={ + "bucket":params.bucket, + "index":params.index, + "id":params.id, + "config":"status/"+params.id, + "tmp":"tmp/"+params.id, + "key":"data/"+params.id, + "filter":"", + "status":"Started" + }; + + var params = { + "bucket":"explore-kendra-solar", + "index":"", // what index is this? + "faq_index":"e1c23860-e5c8-4409-ae26-b05bd6ced00a", + "id":"", + "config":"kendra-status/"+params.id, + "tmp":"tmp/"+params.id, + "key":"data/"+params.id, + "filter":"", + "status":"Started" + }; + + const trigger_export = KendraSyncExport.Export(params); + // const read = KendraSyncExport.Read(params); + // const convert = KendraSyncExport.Convert(params); + // const create = KendraSyncExport.Create(params); } @@ -38,7 +71,12 @@ describe('#test automate-sync()', () => { it('test_create_faq', async function() { let resp = await test_create_faq(); - assert(resp, undefined); + assert(resp, undefined); // TODO: fill in later }); + + // it('test_trigger_export', async function() { + // let resp = await test_trigger_export(); + // console.log(resp); + // }) }); diff --git a/lambda/export/test/qna_FAQ.csv b/lambda/export/test/qna_FAQ.csv index 394b6c61d..14d6b98e1 100644 --- a/lambda/export/test/qna_FAQ.csv +++ b/lambda/export/test/qna_FAQ.csv @@ -1,17 +1,11 @@ Question,Answer,Document URL (optional) -Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.","{""next"":"""",""args"":[""""],""a"":""When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant."",""r"":{""subTitle"":"""",""imageUrl"":"""",""title"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.4"",""type"":""qna"",""selected"":false,""q"":[""Will the Sun ever consume the earth?""]}" -How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive","{""a"":""The Sun is 109 times wider than the Earth and 330,000 times as massive"",""r"":{""subTitle"":""Image referenced from space-facts.com"",""imageUrl"":""https://space-facts.com/wp-content/uploads/sun-size.png"",""title"":""Image""},""qid"":""sun.6"",""type"":""qna"",""q"":[""How large is the sun?""]}" -How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C.,"{""args"":[""""],""next"":"""",""a"":""At its centre the Sun reaches temperatures of 15 million °C."",""r"":{""buttons"":[{""text"":""How far away is the sun?"",""value"":""How far away is the sun?""}],""subTitle"":""Image referenced from solarsystem.nasa.gov"",""imageUrl"":""https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg"",""title"":""Sun Image"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**."",""ssml"":""\nAt its centre the Sun reaches temperatures of 15 million °C\n""},""l"":"""",""qid"":""sun.3"",""type"":""qna"",""q"":[""How hot is it at the center of the Sun?""]}" -Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!","{""args"":[""""],""next"":"""",""a"":""Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!"",""r"":{""buttons"":[{""text"":""How hot is the sun"",""value"":""How hot is the sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths"",""ssml"":"" Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths.""},""l"":"""",""qid"":""sun.1"",""type"":""qna"",""q"":[""Tell me about the sun?""]}" -How old is the Sun?,Our Sun is 4.6 billion years old.,"{""args"":[""""],""next"":"""",""a"":""Our Sun is 4.6 billion years old."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.2"",""type"":""qna"",""q"":[""How old is the Sun?""]}" -How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" -What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" -What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.,"{""args"":[""""],""next"":"""",""a"":""Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options""},""t"":"""",""alt"":{""markdown"":""[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \""Nasa's Sun Page\"") is also a great source of information on the Sun."",""ssml"":""""},""l"":"""",""qid"":""sun.7"",""type"":""qna"",""selected"":false,""q"":[""What other locations on the internet have information about the sun?""]}" -What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" -what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" -Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" -I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" -How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" -Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" -What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" -How wide is the sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" +no_hits,The Kendra Fallback search was not able to identify any results,"{""qid"":""KendraFallback"",""a"":""The Kendra Fallback search was not able to identify any results"",""l"":""QNA:EXTKendraFallback"",""type"":""qna"",""q"":[""no_hits""]}" +How do I calculate expectation?,"There are 2 main methods to calculate the expected value of a random variable. +1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens. +2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.","{""qid"":""stat.4"",""a"":""There are 2 main methods to calculate the expected value of a random variable.\n1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens.\n2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation."",""type"":""qna"",""q"":[""How do I calculate expectation?""]}" +What are the types of categorical variables?,"There are two types of categorical variables: ordinal and nominal variables. +Ordinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating. +Nominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.","{""qid"":""stat.5"",""a"":""There are two types of categorical variables: ordinal and nominal variables.\nOrdinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating.\nNominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name."",""type"":""qna"",""q"":[""What are the types of categorical variables?""]}" +What is the general addition theory?,P(A or B) = P(A) + P(B) - P(A and B),"{""qid"":""stat.1"",""a"":""P(A or B) = P(A) + P(B) - P(A and B)"",""type"":""qna"",""q"":[""What is the general addition theory?""]}" +Define correlation,Correlation is a measure of the strength of a linear relationship.,"{""qid"":""stat.2"",""a"":""Correlation is a measure of the strength of a linear relationship."",""type"":""qna"",""q"":[""Define correlation""]}" +What is a normal distribution?,"The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.","{""qid"":""stat.3"",""a"":""The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation."",""type"":""qna"",""q"":[""What is a normal distribution?""]}" diff --git a/lambda/export/test/qna_export.json b/lambda/export/test/qna_export.json index 59f5c4dd7..2acd48e40 100644 --- a/lambda/export/test/qna_export.json +++ b/lambda/export/test/qna_export.json @@ -1,292 +1,52 @@ { "qna": [ { - "next": "", - "args": [ - "" - ], - "a": "When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", - "r": { - "subTitle": "", - "imageUrl": "", - "title": "" - }, - "t": "", - "alt": { - "markdown": "", - "ssml": "" - }, - "l": "", - "qid": "sun.4", + "qid": "KendraFallback", + "a": "The Kendra Fallback search was not able to identify any results", + "l": "QNA:EXTKendraFallback", "type": "qna", - "selected": false, "q": [ - "Will the Sun ever consume the earth?" + "no_hits" ] }, { - "a": "The Sun is 109 times wider than the Earth and 330,000 times as massive", - "r": { - "subTitle": "Image referenced from space-facts.com", - "imageUrl": "https://space-facts.com/wp-content/uploads/sun-size.png", - "title": "Image" - }, - "qid": "sun.6", + "qid": "stat.4", + "a": "There are 2 main methods to calculate the expected value of a random variable.\n1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens.\n2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.", "type": "qna", "q": [ - "How large is the sun?" + "How do I calculate expectation?" ] }, { - "args": [ - "" - ], - "next": "", - "a": "At its centre the Sun reaches temperatures of 15 million °C.", - "r": { - "buttons": [ - { - "text": "How far away is the sun?", - "value": "How far away is the sun?" - } - ], - "subTitle": "Image referenced from solarsystem.nasa.gov", - "imageUrl": "https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg", - "title": "Sun Image", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**.", - "ssml": "\nAt its centre the Sun reaches temperatures of 15 million °C\n" - }, - "l": "", - "qid": "sun.3", + "qid": "stat.5", + "a": "There are two types of categorical variables: ordinal and nominal variables.\nOrdinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating.\nNominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.", "type": "qna", "q": [ - "How hot is it at the center of the Sun?" + "What are the types of categorical variables?" ] }, { - "args": [ - "" - ], - "next": "", - "a": "Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", - "r": { - "buttons": [ - { - "text": "How hot is the sun", - "value": "How hot is the sun" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "Options", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths", - "ssml": " Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths." - }, - "l": "", - "qid": "sun.1", + "qid": "stat.1", + "a": "P(A or B) = P(A) + P(B) - P(A and B)", "type": "qna", "q": [ - "Tell me about the sun?" + "What is the general addition theory?" ] }, { - "args": [ - "" - ], - "next": "", - "a": "Our Sun is 4.6 billion years old.", - "r": { - "buttons": [ - { - "text": "", - "value": "" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "", - "ssml": "" - }, - "l": "", - "qid": "sun.2", + "qid": "stat.2", + "a": "Correlation is a measure of the strength of a linear relationship.", "type": "qna", "q": [ - "How old is the Sun?" + "Define correlation" ] }, { - "args": [ - "" - ], - "next": "", - "a": "The sun is 149,598,262 km (1 AU) from earth.", - "r": { - "buttons": [ - { - "text": "How large is the sun?", - "value": "How large is the sun?" - } - ], - "subTitle": "Image referenced from blogspot.com", - "imageUrl": "https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif", - "title": "Map" - }, - "t": "", - "alt": { - "markdown": "### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth", - "ssml": "" - }, - "l": "", - "qid": "sun.5", + "qid": "stat.3", + "a": "The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.", "type": "qna", - "selected": false, "q": [ - "How far away is the sun?", - "What distance is the sun from the earth?" - ] - }, - { - "args": [ - "" - ], - "next": "", - "a": "Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.", - "r": { - "buttons": [ - { - "text": "", - "value": "" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "Options" - }, - "t": "", - "alt": { - "markdown": "[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \"Nasa's Sun Page\") is also a great source of information on the Sun.", - "ssml": "" - }, - "l": "", - "qid": "sun.7", - "type": "qna", - "selected": false, - "q": [ - "What other locations on the internet have information about the sun?" - ] - }, - { - "args": [ - "" - ], - "next": "", - "a": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", - "r": { - "buttons": [ - { - "text": "", - "value": "" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')", - "ssml": "A brief eruption of intense high-energy radiation from the sun's surface." - }, - "l": "", - "qid": "sun.9", - "type": "qna", - "q": [ - "What are solar flares?", - "what is a solar flare?", - "Define solar flares.", - "I'd like to know more about solar flares." - ] - }, - { - "args": [ - "" - ], - "next": "", - "a": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", - "r": { - "buttons": [ - { - "text": "What is a solar flare?", - "value": "What is a solar flare" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "Suggestions", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "", - "ssml": "" - }, - "l": "", - "qid": "sun.10", - "type": "qna", - "q": [ - "How many planets orbit the Sun?", - "Which planets orbit the Sun?" - ] - }, - { - "args": [ - "" - ], - "next": "", - "a": "The Sun's diameter is 1.4 million kilometers.", - "r": { - "buttons": [ - { - "text": "How many planets orbit the Sun?", - "value": "How many planets orbit the Sun" - } - ], - "subTitle": "", - "imageUrl": "", - "title": "Suggestions", - "text": "", - "url": "" - }, - "t": "", - "alt": { - "markdown": "", - "ssml": "" - }, - "l": "", - "qid": "sun.11", - "type": "qna", - "q": [ - "What is the diameter of the Sun?", - "How wide is the sun?" + "What is a normal distribution?" ] } ] diff --git a/templates/export/bucket.js b/templates/export/bucket.js index 975af3835..0a873bd3d 100644 --- a/templates/export/bucket.js +++ b/templates/export/bucket.js @@ -28,3 +28,34 @@ module.exports={ } } + +module.kendra_sync_exports={ + "KendraSyncFromS3":{ + "Type": "Custom::S3Lambda", + "Properties": { + "ServiceToken": { "Ref" : "CFNLambda"}, + "Bucket":{"Ref":"ExportBucket"}, + NotificationConfiguration:{ + LambdaFunctionConfigurations:[{ + LambdaFunctionArn:{"Fn::GetAtt":["KendraSyncLambda","Arn"]}, + Events:["s3:ObjectCreated:*"], + Filter:{Key:{FilterRules:[{ + Name:"prefix", + Value:"status" + }]}} + }] + } + } + }, + "KendraSyncPermission":{ + "Type": "AWS::Lambda::Permission", + "Properties": { + "FunctionName":{"Fn::GetAtt":["KendraSyncLambda","Arn"]}, + "Action": "lambda:InvokeFunction", + "Principal": "s3.amazonaws.com", + "SourceAccount": {"Ref": "AWS::AccountId"}, + "SourceArn":{"Fn::Sub":"arn:aws:s3:::${ExportBucket}"} + } + } + +} diff --git a/templates/export/index.js b/templates/export/index.js index e87dbcb55..9c7ca5599 100644 --- a/templates/export/index.js +++ b/templates/export/index.js @@ -19,6 +19,7 @@ module.exports={ "VarIndex": {"Type":"String"}, "EsEndpoint": {"Type":"String"}, "EsProxyLambda": {"Type":"String"}, - "ExportBucket": {"Type":"String"} + "ExportBucket": {"Type":"String"}, + "KendraIndexParameter": {"Type":"String"} } } \ No newline at end of file diff --git a/templates/export/resources.js b/templates/export/resources.js index 1e2bfd083..e81956949 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -91,6 +91,57 @@ module.exports=Object.assign( "ServiceToken": { "Ref" : "CFNLambda" }, "Bucket":{"Ref":"ExportBucket"} } + }, + "KendraSyncExportRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Path": "/", + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + {"Ref":"ExportPolicy"} + ] + } + }, + "KendraySyncExportPolicy": { + "Type": "AWS::IAM::ManagedPolicy", + "Properties": { + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource":[{"Fn::Sub":"arn:aws:s3:::${ExportBucket}*"}] + },{ + "Effect": "Allow", + "Action": [ + "lambda:InvokeFunction" + ], + "Resource":[{"Ref":"EsProxyLambda"}] + }] + } + } + }, + "SyncKendraExportClear":{ + "Type": "Custom::S3Clear", + "Properties": { + "ServiceToken": { "Ref" : "CFNLambda" }, + "Bucket":{"Ref":"ExportBucket"} + } } + }) diff --git a/templates/public/index.js b/templates/public/index.js index e702507c4..b7278c11c 100644 --- a/templates/public/index.js +++ b/templates/public/index.js @@ -29,7 +29,8 @@ module.exports=Promise.resolve(require('../master')).then(function(base){ "Intent", "IntentFallback", "DefaultSettingsSSMParameterName", - "CustomSettingsSSMParameterName" + "CustomSettingsSSMParameterName", + "KendraIndexParameter" ]) base.Parameters=_.pick(base.Parameters,[ From 35bb635b0000552198c7632370cf8bd49ff1973f Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 7 Jul 2020 14:08:40 +0000 Subject: [PATCH 013/137] trying to add the secondary trigger --- lambda/export/kendraSync.js | 29 ++++++++++++++++++++++++++++- lambda/export/parseJSON.js | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js index bd4c9d5fa..e1e721236 100644 --- a/lambda/export/kendraSync.js +++ b/lambda/export/kendraSync.js @@ -4,6 +4,7 @@ aws.config.setPromisesDependency(Promise) aws.config.region=process.env.AWS_REGION var s3=new aws.S3() +const AWSS3 = require('aws-sdk/clients/s3') var lambda=new aws.Lambda() var stride=parseInt(process.env.STRIDE) var _=require('lodash') @@ -11,6 +12,8 @@ var start=require('./lib/start') var step=require('./lib/step') var join=require('./lib/join') var clean=require('./lib/clean') +var parse=require('./parseJSON') +var create=require('./createFAQ') exports.performKendraSync=function(event,context,cb){ console.log("step") @@ -47,5 +50,29 @@ exports.performKendraSync=function(event,context,cb){ } }) .catch(cb) - // TODO: another trigger based on the second s3 bucket... + + + // TODO: another trigger to start converting JSON into FAQ + // note that once the S3 bucket has the kendra-data file, you are good to go + var parseJSONparams = { + csv_name:'qna_FAQ.csv', + input_path:Bucket+'/'+Key, + s3_bucket:'explore-kendra-solar', + output_path:parseJSONparams.s3_bucket + '/' + parseJSONparams.csv_name + } + parse.handler(parseJSONparams).promise() + .done(console.log('Completed parsing JSON')); + + var createFAQparams = { + faq_name:'qna-facts', + faq_index_id:'e1c23860-e5c8-4409-ae26-b05bd6ced00a', + csv_path:parseJSONparams.output_path, + csv_name:parseJSONparams.csv_name, + s3_bucket:parseJSONparams.s3_bucket, + kendra_s3_access_role:'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer', + region:'use-east-1' + } + create.handler(createFAQparams) + .done(console.log('Completed converting to CSV')); + } diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 563ce1483..179fd2c50 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -38,5 +38,5 @@ function qnaJsonParser(input_path, output_path) { } exports.handler = async (params) => { - return qnaJsonParser(params.input_path, params.output); + return qnaJsonParser(params.input_path, params.output_path); }; \ No newline at end of file From 9b5314400152416df47481c1e593cf34fd655236 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 7 Jul 2020 14:11:47 +0000 Subject: [PATCH 014/137] changed config --- templates/export/bucket.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/export/bucket.js b/templates/export/bucket.js index 0a873bd3d..b6f853485 100644 --- a/templates/export/bucket.js +++ b/templates/export/bucket.js @@ -41,7 +41,7 @@ module.kendra_sync_exports={ Events:["s3:ObjectCreated:*"], Filter:{Key:{FilterRules:[{ Name:"prefix", - Value:"status" + Value:"kendradata" }]}} }] } From 20e9b8137f85e090192c8c16e6f534dfd4de5a4b Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 8 Jul 2020 04:19:05 +0000 Subject: [PATCH 015/137] debugging, updating tests --- lambda/export/createFAQ.js | 42 ++-- lambda/export/kendraSync.js | 79 +++---- lambda/export/parseJSON.js | 12 +- lambda/export/test/automateSyncTest.js | 71 +++---- lambda/export/test/qna_export.json | 278 +++++++++++++++++++++++-- templates/export/bucket.js | 6 +- templates/export/resources.js | 88 ++++++-- templates/master/exportstack.js | 1 + templates/master/index.js | 4 + templates/public/index.js | 3 +- 10 files changed, 415 insertions(+), 169 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 4cd3d28a6..707ba93e0 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -105,48 +105,38 @@ function faqLister(kendraClient,params) { */ async function createFAQ(params) { - // FAQ attributes // TODO: don't hardcode - let faq_name = params.faq_name; //'qna-facts'; - let faq_index_id = params.faq_index_id; //'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; - let csv_path = params.csv_path; //'./test/qna_FAQ.csv'; - let csv_name = params.csv_name; //'qna_FAQ.csv'; - let s3_bucket = params.s3_bucket; //'explore-kendra-solar'; - let kendra_s3_access_role = params.kendra_s3_access_role; //'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer'; - let region = params.region; //'us-east-1'; - - // create kendra and s3 clients let kendraClient = (process.env.REGION ? - new AWSKendra({apiVersion: '2019-02-03', region: process.env.REGION}) : - new AWSKendra({apiVersion: '2019-02-03', region: region}) + new AWSKendra({apiVersion:'2019-02-03', region:process.env.REGION}) : + new AWSKendra({apiVersion:'2019-02-03', region:params.region}) ); let s3Client = (process.env.REGION ? - new AWSS3({apiVersion: '2006-03-01', region: process.env.REGION}) : - new AWSS3({apiVersion: '2006-03-01'})); + new AWSS3({apiVersion:'2006-03-01', region:process.env.REGION}) : + new AWSS3({apiVersion:'2006-03-01', region:params.region})); console.log('clients created'); // read in CSV and upload to S3 bucket var fs = require('fs'); var s3_params = { - Bucket: s3_bucket, - Key: csv_name, - ACL: 'private', // TODO: do we want this to be publicly readable by linking QID's? - Body: fs.createReadStream(csv_path), // use read stream option in case file is large + Bucket: params.s3_bucket, + Key: params.s3_key, + ACL: 'bucket-owner-read', // TODO: should this param be public? + Body: fs.createReadStream(params.csv_path), // use read stream option in case file is large }; var s3_response = await s3Uploader(s3Client, s3_params); // if FAQ exists already, delete the old one and update it var index_params = { - IndexId: faq_index_id, + IndexId: params.faq_index_id, MaxResults: '30' // default max number of FAQs in developer edition }; var list_faq_response = await faqLister(kendraClient, index_params); var j, elem, index=null; for (j=0; js3.getObject({Bucket,Key,VersionId}).promise()) .then(x=>JSON.parse(x.Body.toString())) - .then(function(config){ - if(config.status!=="Error" && config.status!=="Completed"){ - return Promise.try(function(){ - console.log("Config:",JSON.stringify(config,null,2)) - switch(config.status){ - case 'Started': - return start(config); - case 'InProgress': - return step(config); - case "Join": - return join(config); - case "Clean": - return clean(config); - } - }) - .catch(error=>{ - console.log(error) - config.status="Error" - config.message=_.get(error,'message',JSON.stringify(error)) - }) - .then(()=>s3.putObject({Bucket,Key,Body:JSON.stringify(config)}).promise()) + .then(async function(content){ + var parseJSONparams = { + csv_name:'qna_FAQ.csv', + content:content, + output_path:'/tmp/qna_FAQ.csv', } + parse.handler(parseJSONparams); + console.log("Parsed content JSON into CSV stored locally"); + + var createFAQparams = { + faq_name:'qna-facts', + faq_index_id:process.env.KENDRA_INDEX, + csv_path:parseJSONparams.output_path, + csv_name:parseJSONparams.csv_name, + s3_bucket:process.env.OUTPUT_S3_BUCKET, + s3_key:"kendra_csv" + "/" + parseJSONparams.csv_name, + kendra_s3_access_role:process.env.KENDRA_ROLE, + region:process.env.REGION + } + console.log('kendra role is ' + createFAQparams.kendra_s3_access_role); + create.handler(createFAQparams) + console.log('Completed CSV converting to FAQ'); }) .catch(cb) - - - // TODO: another trigger to start converting JSON into FAQ - // note that once the S3 bucket has the kendra-data file, you are good to go - var parseJSONparams = { - csv_name:'qna_FAQ.csv', - input_path:Bucket+'/'+Key, - s3_bucket:'explore-kendra-solar', - output_path:parseJSONparams.s3_bucket + '/' + parseJSONparams.csv_name - } - parse.handler(parseJSONparams).promise() - .done(console.log('Completed parsing JSON')); - - var createFAQparams = { - faq_name:'qna-facts', - faq_index_id:'e1c23860-e5c8-4409-ae26-b05bd6ced00a', - csv_path:parseJSONparams.output_path, - csv_name:parseJSONparams.csv_name, - s3_bucket:parseJSONparams.s3_bucket, - kendra_s3_access_role:'arn:aws:iam::425742325899:role/service-role/AmazonKendra-question-bucketer', - region:'use-east-1' - } - create.handler(createFAQparams) - .done(console.log('Completed converting to CSV')); - } diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 179fd2c50..61185c7a3 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -6,10 +6,10 @@ * @param output_path : the output file path to write the CSV * @returns output_path */ -function qnaJsonParser(input_path, output_path) { +function qnaJsonParser(params) { const createCsvWriter = require('./node_modules/csv-writer').createObjectCsvWriter; const csvWriter = createCsvWriter({ - path: output_path, + path: params.output_path, header: [ {id: 'question', title: 'Question'}, {id: 'answer', title: 'Answer'}, @@ -19,7 +19,7 @@ function qnaJsonParser(input_path, output_path) { const data = []; - const q_list = require(input_path).qna; + const q_list = params.content.qna; q_list.forEach(function(elem) { elem.q.forEach(function(ques) { @@ -32,11 +32,11 @@ function qnaJsonParser(input_path, output_path) { csvWriter .writeRecords(data) - .then(()=> console.log('The CSV file ' + output_path + ' was written successfully')); + .then(()=> console.log('The CSV file ' + params.output_path + ' was written successfully')); - return output_path; + return params.output_path; } exports.handler = async (params) => { - return qnaJsonParser(params.input_path, params.output_path); + return qnaJsonParser(params); }; \ No newline at end of file diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index 5a7f5b946..741169a7c 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -3,7 +3,14 @@ var assert = require('assert'); // json parsing test async function test_parser() { const parseJSON = require('../parseJSON.js'); - const csv_path = await parseJSON.handler('./test/qna_export.json', './test/qna_FAQ.csv', true); + var content = require('./qna_export.json'); + var parseJSONparams = { + csv_name:'qna_FAQ.csv', + content:content, + output_path:'/tmp/qna_FAQ.csv', + } + const csv_path = await parseJSON.handler(parseJSONparams); + const fs = require('fs') try { @@ -14,7 +21,7 @@ async function test_parser() { return false; } } catch(err) { - // console.error(err) + console.error(err) return false; } @@ -24,44 +31,27 @@ async function test_parser() { // create FAQ test async function test_create_faq() { - const createFAQ = require('../createFAQ.js'); - return createFAQ.handler(); -} - -async function test_trigger_export() { - const triggerExport = require('../ExportJSON.js'); - const KendraSyncExport = triggerExport.KendraSyncExport(); - - - var data={ - "bucket":params.bucket, - "index":params.index, - "id":params.id, - "config":"status/"+params.id, - "tmp":"tmp/"+params.id, - "key":"data/"+params.id, - "filter":"", - "status":"Started" - }; - - var params = { - "bucket":"explore-kendra-solar", - "index":"", // what index is this? - "faq_index":"e1c23860-e5c8-4409-ae26-b05bd6ced00a", - "id":"", - "config":"kendra-status/"+params.id, - "tmp":"tmp/"+params.id, - "key":"data/"+params.id, - "filter":"", - "status":"Started" - }; - - const trigger_export = KendraSyncExport.Export(params); - // const read = KendraSyncExport.Read(params); - // const convert = KendraSyncExport.Convert(params); - // const create = KendraSyncExport.Create(params); + const create = require('../createFAQ.js'); + var content = require('./qna_export.json'); + var parseJSONparams = { + csv_name:'qna_FAQ.csv', + content:content, + output_path:'/tmp/qna_FAQ.csv', + } + var createFAQparams = { + faq_name:'qna-facts', + faq_index_id:'e1c23860-e5c8-4409-ae26-b05bd6ced00a', + csv_path:parseJSONparams.output_path, + csv_name:parseJSONparams.csv_name, + s3_bucket:'qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe', + s3_key:"kendra_csv" + "/" + parseJSONparams.csv_name, + kendra_s3_access_role:'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-2-ExportStac-KendraSyncRole-1G3IEI1JF7L3S', + region:'us-east-1' + } + return create.handler(createFAQparams); } +// TODO: test performSync describe('#test automate-sync()', () => { it('test_json_parser', async function() { @@ -73,10 +63,5 @@ describe('#test automate-sync()', () => { let resp = await test_create_faq(); assert(resp, undefined); // TODO: fill in later }); - - // it('test_trigger_export', async function() { - // let resp = await test_trigger_export(); - // console.log(resp); - // }) }); diff --git a/lambda/export/test/qna_export.json b/lambda/export/test/qna_export.json index 2acd48e40..59f5c4dd7 100644 --- a/lambda/export/test/qna_export.json +++ b/lambda/export/test/qna_export.json @@ -1,52 +1,292 @@ { "qna": [ { - "qid": "KendraFallback", - "a": "The Kendra Fallback search was not able to identify any results", - "l": "QNA:EXTKendraFallback", + "next": "", + "args": [ + "" + ], + "a": "When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.", + "r": { + "subTitle": "", + "imageUrl": "", + "title": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.4", "type": "qna", + "selected": false, "q": [ - "no_hits" + "Will the Sun ever consume the earth?" ] }, { - "qid": "stat.4", - "a": "There are 2 main methods to calculate the expected value of a random variable.\n1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens.\n2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.", + "a": "The Sun is 109 times wider than the Earth and 330,000 times as massive", + "r": { + "subTitle": "Image referenced from space-facts.com", + "imageUrl": "https://space-facts.com/wp-content/uploads/sun-size.png", + "title": "Image" + }, + "qid": "sun.6", "type": "qna", "q": [ - "How do I calculate expectation?" + "How large is the sun?" ] }, { - "qid": "stat.5", - "a": "There are two types of categorical variables: ordinal and nominal variables.\nOrdinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating.\nNominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.", + "args": [ + "" + ], + "next": "", + "a": "At its centre the Sun reaches temperatures of 15 million °C.", + "r": { + "buttons": [ + { + "text": "How far away is the sun?", + "value": "How far away is the sun?" + } + ], + "subTitle": "Image referenced from solarsystem.nasa.gov", + "imageUrl": "https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg", + "title": "Sun Image", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**.", + "ssml": "\nAt its centre the Sun reaches temperatures of 15 million °C\n" + }, + "l": "", + "qid": "sun.3", "type": "qna", "q": [ - "What are the types of categorical variables?" + "How hot is it at the center of the Sun?" ] }, { - "qid": "stat.1", - "a": "P(A or B) = P(A) + P(B) - P(A and B)", + "args": [ + "" + ], + "next": "", + "a": "Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!", + "r": { + "buttons": [ + { + "text": "How hot is the sun", + "value": "How hot is the sun" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "Options", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths", + "ssml": " Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths." + }, + "l": "", + "qid": "sun.1", "type": "qna", "q": [ - "What is the general addition theory?" + "Tell me about the sun?" ] }, { - "qid": "stat.2", - "a": "Correlation is a measure of the strength of a linear relationship.", + "args": [ + "" + ], + "next": "", + "a": "Our Sun is 4.6 billion years old.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.2", "type": "qna", "q": [ - "Define correlation" + "How old is the Sun?" ] }, { - "qid": "stat.3", - "a": "The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.", + "args": [ + "" + ], + "next": "", + "a": "The sun is 149,598,262 km (1 AU) from earth.", + "r": { + "buttons": [ + { + "text": "How large is the sun?", + "value": "How large is the sun?" + } + ], + "subTitle": "Image referenced from blogspot.com", + "imageUrl": "https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif", + "title": "Map" + }, + "t": "", + "alt": { + "markdown": "### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth", + "ssml": "" + }, + "l": "", + "qid": "sun.5", "type": "qna", + "selected": false, "q": [ - "What is a normal distribution?" + "How far away is the sun?", + "What distance is the sun from the earth?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "Options" + }, + "t": "", + "alt": { + "markdown": "[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \"Nasa's Sun Page\") is also a great source of information on the Sun.", + "ssml": "" + }, + "l": "", + "qid": "sun.7", + "type": "qna", + "selected": false, + "q": [ + "What other locations on the internet have information about the sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.", + "r": { + "buttons": [ + { + "text": "", + "value": "" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')", + "ssml": "A brief eruption of intense high-energy radiation from the sun's surface." + }, + "l": "", + "qid": "sun.9", + "type": "qna", + "q": [ + "What are solar flares?", + "what is a solar flare?", + "Define solar flares.", + "I'd like to know more about solar flares." + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.", + "r": { + "buttons": [ + { + "text": "What is a solar flare?", + "value": "What is a solar flare" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "Suggestions", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.10", + "type": "qna", + "q": [ + "How many planets orbit the Sun?", + "Which planets orbit the Sun?" + ] + }, + { + "args": [ + "" + ], + "next": "", + "a": "The Sun's diameter is 1.4 million kilometers.", + "r": { + "buttons": [ + { + "text": "How many planets orbit the Sun?", + "value": "How many planets orbit the Sun" + } + ], + "subTitle": "", + "imageUrl": "", + "title": "Suggestions", + "text": "", + "url": "" + }, + "t": "", + "alt": { + "markdown": "", + "ssml": "" + }, + "l": "", + "qid": "sun.11", + "type": "qna", + "q": [ + "What is the diameter of the Sun?", + "How wide is the sun?" ] } ] diff --git a/templates/export/bucket.js b/templates/export/bucket.js index b6f853485..1dae5fec2 100644 --- a/templates/export/bucket.js +++ b/templates/export/bucket.js @@ -25,11 +25,7 @@ module.exports={ "SourceAccount": {"Ref": "AWS::AccountId"}, "SourceArn":{"Fn::Sub":"arn:aws:s3:::${ExportBucket}"} } - } - -} - -module.kendra_sync_exports={ + }, "KendraSyncFromS3":{ "Type": "Custom::S3Lambda", "Properties": { diff --git a/templates/export/resources.js b/templates/export/resources.js index e81956949..234632198 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -16,6 +16,15 @@ module.exports=Object.assign( "BuildDate":(new Date()).toISOString() } }, + "SyncCodeVersion":{ + "Type": "Custom::S3Version", + "Properties": { + "ServiceToken": { "Ref" : "CFNLambda" }, + "Bucket": {"Ref":"BootstrapBucket"}, + "Key": {"Fn::Sub":"${BootstrapPrefix}/lambda/export.zip"}, + "BuildDate":(new Date()).toISOString() + } + }, "ExportStepLambda": { "Type": "AWS::Lambda::Function", "Properties": { @@ -92,7 +101,60 @@ module.exports=Object.assign( "Bucket":{"Ref":"ExportBucket"} } }, - "KendraSyncExportRole": { + "ExportStepLambda": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": {"Ref":"BootstrapBucket"}, + "S3Key": {"Fn::Sub":"${BootstrapPrefix}/lambda/export.zip"}, + "S3ObjectVersion":{"Ref":"ExportCodeVersion"} + }, + "Environment": { + "Variables": { + ES_INDEX:{"Ref":"VarIndex"}, + ES_ENDPOINT:{"Ref":"EsEndpoint"}, + ES_PROXY:{"Ref":"EsProxyLambda"} + } + }, + "Handler": "index.step", + "MemorySize": "1024", + "Role": {"Fn::GetAtt": ["ExportRole","Arn"]}, + "Runtime": "nodejs10.x", + "Timeout": 300, + "Tags":[{ + Key:"Type", + Value:"Export" + }] + } + }, + "KendraSyncLambda": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": {"Ref":"BootstrapBucket"}, + "S3Key": {"Fn::Sub":"${BootstrapPrefix}/lambda/export.zip"}, + "S3ObjectVersion":{"Ref":"SyncCodeVersion"} + }, + "Environment": { + "Variables": { + "KENDRA_INDEX":{"Ref":"KendraIndexParameter"}, + "OUTPUT_S3_BUCKET":{"Ref":"ExportBucket"}, + "KENDRA_ROLE":{"Ref":"KendraSyncRole"}, + "REGION":{"Ref":"AWS::Region"} // "$AWS_REGION" + } + }, + "Handler": "kendraSync.performSync", + "MemorySize": "1024", + "Role": {"Fn::GetAtt": ["KendraSyncRole","Arn"]}, + "Runtime": "nodejs10.x", + "Timeout": 300, + "Tags":[{ + Key:"Type", + Value:"Sync" + }] + } + }, + "KendraSyncRole": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -110,11 +172,11 @@ module.exports=Object.assign( "Path": "/", "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - {"Ref":"ExportPolicy"} + {"Ref":"KendraSyncPolicy"} ] } }, - "KendraySyncExportPolicy": { + "KendraSyncPolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { @@ -125,23 +187,15 @@ module.exports=Object.assign( "s3:*" ], "Resource":[{"Fn::Sub":"arn:aws:s3:::${ExportBucket}*"}] - },{ - "Effect": "Allow", - "Action": [ - "lambda:InvokeFunction" - ], - "Resource":[{"Ref":"EsProxyLambda"}] + // },{ + // "Effect": "Allow", + // "Action": [ + // "lambda:InvokeFunction" + // ], + // "Resource":[{"Ref":"EsProxyLambda"}] }] } } - }, - "SyncKendraExportClear":{ - "Type": "Custom::S3Clear", - "Properties": { - "ServiceToken": { "Ref" : "CFNLambda" }, - "Bucket":{"Ref":"ExportBucket"} - } } - }) diff --git a/templates/master/exportstack.js b/templates/master/exportstack.js index a0bd0a79d..363249e6e 100644 --- a/templates/master/exportstack.js +++ b/templates/master/exportstack.js @@ -12,6 +12,7 @@ module.exports={ "EsEndpoint": {"Fn::GetAtt": ["ESVar", "ESAddress"]}, "EsProxyLambda": {"Fn::GetAtt":["ESProxyLambda","Arn"]}, "ExportBucket": {"Ref":"ExportBucket"}, + "KendraIndexParameter": {"Ref":"KendraIndexParameter"} } } } diff --git a/templates/master/index.js b/templates/master/index.js index 729bafdd4..9ae84264a 100644 --- a/templates/master/index.js +++ b/templates/master/index.js @@ -167,6 +167,10 @@ module.exports={ "BuildExamples":{ "Type":"String", "Default":"TRUE" + }, + "KendraIndexParameter":{ + "Type":"String", + "Default":"" } }, "Conditions":{ diff --git a/templates/public/index.js b/templates/public/index.js index b7278c11c..5cda0aec0 100644 --- a/templates/public/index.js +++ b/templates/public/index.js @@ -35,7 +35,8 @@ module.exports=Promise.resolve(require('../master')).then(function(base){ base.Parameters=_.pick(base.Parameters,[ "Email", - "Username" + "Username", + "KendraIndexParameter" ]) base.Conditions.Public={"Fn::Equals":[true,true]} base.Conditions.AdminSignUp={"Fn::Equals":[true,true]} From 9453a1eaa6bb84d5d54eda4b3832a593492d3c85 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 8 Jul 2020 18:21:33 +0000 Subject: [PATCH 016/137] debugging export lambda --- bin/config.js | 3 ++- lambda/export/kendraSync.js | 36 ++++++++++++++----------- lambda/export/parseJSON.js | 8 +++--- lambda/export/test/automateSyncTest.js | 31 +++++++++++++++------ lambda/export/test/syncEvent.json | 37 ++++++++++++++++++++++++++ templates/dev/master.js | 1 + templates/export/index.js | 2 +- templates/export/resources.js | 24 ++++++++++------- templates/master/exportstack.js | 2 +- templates/master/index.js | 12 +++++++-- templates/public/index.js | 5 ++-- 11 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 lambda/export/test/syncEvent.json diff --git a/bin/config.js b/bin/config.js index 1f3fdd9ce..03044927b 100644 --- a/bin/config.js +++ b/bin/config.js @@ -18,7 +18,8 @@ module.exports={ "publicPrefix":"artifacts/aws-ai-qna-bot", "devEmail":"", "namespace":"dev", - "stackNamePrefix":"QNA" + "stackNamePrefix":"QNA", + "KendraIndex":"" } if (require.main === module) { diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js index fc3338763..886d1a971 100644 --- a/lambda/export/kendraSync.js +++ b/lambda/export/kendraSync.js @@ -32,22 +32,26 @@ exports.performSync=function(event,context,cb){ content:content, output_path:'/tmp/qna_FAQ.csv', } - parse.handler(parseJSONparams); - console.log("Parsed content JSON into CSV stored locally"); - - var createFAQparams = { - faq_name:'qna-facts', - faq_index_id:process.env.KENDRA_INDEX, - csv_path:parseJSONparams.output_path, - csv_name:parseJSONparams.csv_name, - s3_bucket:process.env.OUTPUT_S3_BUCKET, - s3_key:"kendra_csv" + "/" + parseJSONparams.csv_name, - kendra_s3_access_role:process.env.KENDRA_ROLE, - region:process.env.REGION - } - console.log('kendra role is ' + createFAQparams.kendra_s3_access_role); - create.handler(createFAQparams) - console.log('Completed CSV converting to FAQ'); + parse.handler(parseJSONparams).then(()=>{ + console.log("Parsed content JSON into CSV stored locally"); + + var createFAQparams = { + faq_name:'qna-facts', + faq_index_id:process.env.KENDRA_INDEX, + csv_path:parseJSONparams.output_path, + csv_name:parseJSONparams.csv_name, + s3_bucket:process.env.OUTPUT_S3_BUCKET, + s3_key:"kendra_csv" + "/" + parseJSONparams.csv_name, + kendra_s3_access_role:process.env.KENDRA_ROLE, + region:process.env.REGION + } + console.log('s3 bucket is ' + createFAQparams.s3_bucket); + console.log('kendra role is ' + createFAQparams.kendra_s3_access_role); + create.handler(createFAQparams) + console.log('Completed CSV converting to FAQ'); + }); }) .catch(cb) + + return 'Synced'; } diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 61185c7a3..83a46374d 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -7,7 +7,7 @@ * @returns output_path */ function qnaJsonParser(params) { - const createCsvWriter = require('./node_modules/csv-writer').createObjectCsvWriter; + const createCsvWriter = require('csv-writer').createObjectCsvWriter; const csvWriter = createCsvWriter({ path: params.output_path, header: [ @@ -29,12 +29,12 @@ function qnaJsonParser(params) { data.push(entry); }); }); + console.log(`data is ${data}`); - csvWriter + return csvWriter .writeRecords(data) .then(()=> console.log('The CSV file ' + params.output_path + ' was written successfully')); - - return params.output_path; + } exports.handler = async (params) => { diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index 741169a7c..a4a6777e9 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -51,17 +51,32 @@ async function test_create_faq() { return create.handler(createFAQparams); } -// TODO: test performSync +//performSync test +async function test_performSync() { + const kendraSync = require('../kendraSync.js'); + const event = require('./syncEvent.json'); + var context = undefined; + var cb = undefined; + process.env.OUTPUT_S3_BUCKET = 'qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe' + process.env.KENDRA_INDEX = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; + process.env.KENDRA_ROLE = 'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-2-ExportStac-KendraSyncRole-1G3IEI1JF7L3S' + return kendraSync.performSync(event, context, cb); +} describe('#test automate-sync()', () => { - it('test_json_parser', async function() { - let resp = await test_parser(); - assert(resp, "CSV file does not exist!"); - }); + // it('test_json_parser', async function() { + // let resp = await test_parser(); + // assert(resp, "CSV file does not exist!"); + // }); - it('test_create_faq', async function() { - let resp = await test_create_faq(); - assert(resp, undefined); // TODO: fill in later + // it('test_create_faq', async function() { + // let resp = await test_create_faq(); + // assert(resp, undefined); // TODO: fill in later + // }); + + it('test_perform_sync', async function() { + let resp = await test_performSync(); + assert(resp, 'Synced'); // TODO: fill in later }); }); diff --git a/lambda/export/test/syncEvent.json b/lambda/export/test/syncEvent.json new file mode 100644 index 000000000..55ef1b5da --- /dev/null +++ b/lambda/export/test/syncEvent.json @@ -0,0 +1,37 @@ +{ + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "EXAMPLE" + }, + "requestParameters": { + "sourceIPAddress": "127.0.0.1" + }, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe", + "ownerIdentity": { + "principalId": "EXAMPLE" + }, + "arn": "arn:aws:s3:::example-bucket" + }, + "object": { + "key": "qna_FAQ.json", + "size": 1024, + "eTag": "8f9ad97afed386ca0ff9452ef2ca198a" + } + } + } + ] +} diff --git a/templates/dev/master.js b/templates/dev/master.js index 49e228f78..64132e684 100644 --- a/templates/dev/master.js +++ b/templates/dev/master.js @@ -10,6 +10,7 @@ module.exports=Promise.join( base.Parameters.BootstrapBucket.Default=output.Bucket base.Parameters.BootstrapPrefix.Default=output.Prefix base.Parameters.Email.Default=config.devEmail + // base.Parameters.KendraIndex=config.KendraIndex return base }) diff --git a/templates/export/index.js b/templates/export/index.js index 9c7ca5599..44e2002ac 100644 --- a/templates/export/index.js +++ b/templates/export/index.js @@ -20,6 +20,6 @@ module.exports={ "EsEndpoint": {"Type":"String"}, "EsProxyLambda": {"Type":"String"}, "ExportBucket": {"Type":"String"}, - "KendraIndexParameter": {"Type":"String"} + "KendraIndex": {"Type":"String"} } } \ No newline at end of file diff --git a/templates/export/resources.js b/templates/export/resources.js index 234632198..11d7caa9a 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -137,10 +137,10 @@ module.exports=Object.assign( }, "Environment": { "Variables": { - "KENDRA_INDEX":{"Ref":"KendraIndexParameter"}, + "KENDRA_INDEX":{"Ref":"KendraIndex"}, "OUTPUT_S3_BUCKET":{"Ref":"ExportBucket"}, - "KENDRA_ROLE":{"Ref":"KendraSyncRole"}, - "REGION":{"Ref":"AWS::Region"} // "$AWS_REGION" + "KENDRA_ROLE":{"Fn::GetAtt": ["KendraSyncRole","Arn"]}, + "REGION":{"Ref":"AWS::Region"} } }, "Handler": "kendraSync.performSync", @@ -166,6 +166,12 @@ module.exports=Object.assign( "Service": "lambda.amazonaws.com" }, "Action": "sts:AssumeRole" + },{ + "Effect": "Allow", + "Principal": { + "Service": "kendra.amazonaws.com" + }, + "Action": "sts:AssumeRole" } ] }, @@ -187,12 +193,12 @@ module.exports=Object.assign( "s3:*" ], "Resource":[{"Fn::Sub":"arn:aws:s3:::${ExportBucket}*"}] - // },{ - // "Effect": "Allow", - // "Action": [ - // "lambda:InvokeFunction" - // ], - // "Resource":[{"Ref":"EsProxyLambda"}] + },{ + "Effect": "Allow", + "Action": [ + "" + ], + "Resource":[{"Ref":"KendraIndex"}] }] } } diff --git a/templates/master/exportstack.js b/templates/master/exportstack.js index 363249e6e..e54ed66da 100644 --- a/templates/master/exportstack.js +++ b/templates/master/exportstack.js @@ -12,7 +12,7 @@ module.exports={ "EsEndpoint": {"Fn::GetAtt": ["ESVar", "ESAddress"]}, "EsProxyLambda": {"Fn::GetAtt":["ESProxyLambda","Arn"]}, "ExportBucket": {"Ref":"ExportBucket"}, - "KendraIndexParameter": {"Ref":"KendraIndexParameter"} + "KendraIndex": {"Ref":"KendraIndex"} } } } diff --git a/templates/master/index.js b/templates/master/index.js index 9ae84264a..9850b43fa 100644 --- a/templates/master/index.js +++ b/templates/master/index.js @@ -168,7 +168,14 @@ module.exports={ "Type":"String", "Default":"TRUE" }, - "KendraIndexParameter":{ + "EnableKendraFAQ":{ + "Type":"String", + "AllowedValues":["FALSE","TRUE"], + "ConstraintDescription":"Allowed Values are FALSE or TRUE", + "Default":"FALSE", + "Description":"Defines whether Kendra FAQ integration is enabled" + }, + "KendraIndex":{ "Type":"String", "Default":"" } @@ -179,7 +186,8 @@ module.exports={ "Domain":{"Fn::Not":[{"Fn::Equals":[{"Ref":"ApprovedDomain"},"NONE"]}]}, "BuildExamples":{"Fn::Equals":[{"Ref":"BuildExamples"},"TRUE"]}, "CreateDomain":{"Fn::Equals":[{"Ref":"ElasticsearchName"},"EMPTY"]}, - "DontCreateDomain":{"Fn::Not":[{"Fn::Equals":[{"Ref":"ElasticsearchName"},"EMPTY"]}]} + "DontCreateDomain":{"Fn::Not":[{"Fn::Equals":[{"Ref":"ElasticsearchName"},"EMPTY"]}]}, + "KendraFAQ":{"Fn::Equals":[{"Ref":"EnableKendraFAQ"}, "TRUE"]} } } diff --git a/templates/public/index.js b/templates/public/index.js index 5cda0aec0..80f36a34d 100644 --- a/templates/public/index.js +++ b/templates/public/index.js @@ -30,13 +30,14 @@ module.exports=Promise.resolve(require('../master')).then(function(base){ "IntentFallback", "DefaultSettingsSSMParameterName", "CustomSettingsSSMParameterName", - "KendraIndexParameter" + "KendraIndex" ]) base.Parameters=_.pick(base.Parameters,[ "Email", "Username", - "KendraIndexParameter" + "KendraIndex", + "EnableKendraFAQ" ]) base.Conditions.Public={"Fn::Equals":[true,true]} base.Conditions.AdminSignUp={"Fn::Equals":[true,true]} From 35f189892f78f08d7be117553b91419cb7daaf3e Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 9 Jul 2020 15:32:10 +0000 Subject: [PATCH 017/137] debugging --- lambda/export/kendraSync.js | 7 - lambda/export/parseJSON.js | 1 - lambda/export/test/automateSyncTest.js | 8 +- lambda/export/test/syncEvent.json | 2 +- templates/dev/master.js | 2 +- templates/export/bucket.js | 27 ++-- templates/export/resources.js | 35 +++-- website/js/components/designer/index.vue | 3 + website/js/components/designer/synckendra.vue | 129 ++++++++++++++++++ website/js/lib/store/api/actions/sync.js | 52 +++++++ 10 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 website/js/components/designer/synckendra.vue create mode 100644 website/js/lib/store/api/actions/sync.js diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js index 886d1a971..612942911 100644 --- a/lambda/export/kendraSync.js +++ b/lambda/export/kendraSync.js @@ -5,18 +5,11 @@ aws.config.region=process.env.AWS_REGION var s3=new aws.S3() const AWSS3 = require('aws-sdk/clients/s3') -// var lambda=new aws.Lambda() -var stride=parseInt(process.env.STRIDE) var _=require('lodash') -// var start=require('./lib/start') -// var step=require('./lib/step') -// var join=require('./lib/join') -// var clean=require('./lib/clean') var parse=require('./parseJSON') var create=require('./createFAQ') exports.performSync=function(event,context,cb){ - console.log("step") console.log("Request",JSON.stringify(event,null,2)) var Bucket=event.Records[0].s3.bucket.name var Key=decodeURI(event.Records[0].s3.object.key) diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index 83a46374d..f34c5065e 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -29,7 +29,6 @@ function qnaJsonParser(params) { data.push(entry); }); }); - console.log(`data is ${data}`); return csvWriter .writeRecords(data) diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index a4a6777e9..92d0e651a 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -57,9 +57,9 @@ async function test_performSync() { const event = require('./syncEvent.json'); var context = undefined; var cb = undefined; - process.env.OUTPUT_S3_BUCKET = 'qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe' + process.env.OUTPUT_S3_BUCKET = 'qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j' process.env.KENDRA_INDEX = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; - process.env.KENDRA_ROLE = 'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-2-ExportStac-KendraSyncRole-1G3IEI1JF7L3S' + process.env.KENDRA_ROLE = 'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-3-ExportStac-KendraSyncRole-1RN4NKGMDFRNH' return kendraSync.performSync(event, context, cb); } @@ -71,12 +71,12 @@ describe('#test automate-sync()', () => { // it('test_create_faq', async function() { // let resp = await test_create_faq(); - // assert(resp, undefined); // TODO: fill in later + // assert(resp, 'Failed to create FAQ'); // }); it('test_perform_sync', async function() { let resp = await test_performSync(); - assert(resp, 'Synced'); // TODO: fill in later + assert(resp, 'Synced'); }); }); diff --git a/lambda/export/test/syncEvent.json b/lambda/export/test/syncEvent.json index 55ef1b5da..6abed757f 100644 --- a/lambda/export/test/syncEvent.json +++ b/lambda/export/test/syncEvent.json @@ -27,7 +27,7 @@ "arn": "arn:aws:s3:::example-bucket" }, "object": { - "key": "qna_FAQ.json", + "key": "data/qna.json", "size": 1024, "eTag": "8f9ad97afed386ca0ff9452ef2ca198a" } diff --git a/templates/dev/master.js b/templates/dev/master.js index 64132e684..fb0ebb160 100644 --- a/templates/dev/master.js +++ b/templates/dev/master.js @@ -10,7 +10,7 @@ module.exports=Promise.join( base.Parameters.BootstrapBucket.Default=output.Bucket base.Parameters.BootstrapPrefix.Default=output.Prefix base.Parameters.Email.Default=config.devEmail - // base.Parameters.KendraIndex=config.KendraIndex + base.Parameters.KendraIndex.Default=config.KendraIndex return base }) diff --git a/templates/export/bucket.js b/templates/export/bucket.js index 1dae5fec2..4617ceb6f 100644 --- a/templates/export/bucket.js +++ b/templates/export/bucket.js @@ -12,7 +12,15 @@ module.exports={ Name:"prefix", Value:"status" }]}} - }] + },{ + LambdaFunctionArn:{"Fn::GetAtt":["KendraSyncLambda","Arn"]}, + Events:["s3:ObjectCreated:*"], + Filter:{Key:{FilterRules:[{ + Name:"prefix", + Value:"data" + }]}} + } + ] } } }, @@ -26,23 +34,6 @@ module.exports={ "SourceArn":{"Fn::Sub":"arn:aws:s3:::${ExportBucket}"} } }, - "KendraSyncFromS3":{ - "Type": "Custom::S3Lambda", - "Properties": { - "ServiceToken": { "Ref" : "CFNLambda"}, - "Bucket":{"Ref":"ExportBucket"}, - NotificationConfiguration:{ - LambdaFunctionConfigurations:[{ - LambdaFunctionArn:{"Fn::GetAtt":["KendraSyncLambda","Arn"]}, - Events:["s3:ObjectCreated:*"], - Filter:{Key:{FilterRules:[{ - Name:"prefix", - Value:"kendradata" - }]}} - }] - } - } - }, "KendraSyncPermission":{ "Type": "AWS::Lambda::Permission", "Properties": { diff --git a/templates/export/resources.js b/templates/export/resources.js index 11d7caa9a..cb24c7dda 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -186,22 +186,29 @@ module.exports=Object.assign( "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource":[{"Fn::Sub":"arn:aws:s3:::${ExportBucket}*"}] - },{ - "Effect": "Allow", - "Action": [ - "" - ], - "Resource":[{"Ref":"KendraIndex"}] + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "kendra:CreateFaq", + "kendra:ListFaqs", + "s3:ListBucket", + "kendra:TagResource", + "kendra:DeleteFaq" + ], + "Resource": [ + {"Fn::Sub":"arn:aws:kendra:::index/${KendraIndex}"}, + {"Fn::Sub":"arn:aws:kendra:::index/${KendraIndex}/faq/*"}, + {"Fn::Sub":"arn:aws:s3:::${ExportBucket}"}, + {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"} + ] }] } } } -}) + } +) + diff --git a/website/js/components/designer/index.vue b/website/js/components/designer/index.vue index d3cc6047f..7a5f615a9 100644 --- a/website/js/components/designer/index.vue +++ b/website/js/components/designer/index.vue @@ -20,6 +20,8 @@ alexa v-list-tile build + v-list-tile + sync span questions(@filter="get(pagination)" v-if="active==='questions'") test(v-if="active==='test'") @@ -146,6 +148,7 @@ module.exports={ edit:require('./edit.vue').default, build:require('./rebuild.vue').default, alexa:require('./alexa.vue').default, + sync:require('./synckendra.vue').default }, computed:{ empty:function(){ diff --git a/website/js/components/designer/synckendra.vue b/website/js/components/designer/synckendra.vue new file mode 100644 index 000000000..9e9474024 --- /dev/null +++ b/website/js/components/designer/synckendra.vue @@ -0,0 +1,129 @@ + + + + + + diff --git a/website/js/lib/store/api/actions/sync.js b/website/js/lib/store/api/actions/sync.js new file mode 100644 index 000000000..b203553e7 --- /dev/null +++ b/website/js/lib/store/api/actions/sync.js @@ -0,0 +1,52 @@ +/* +Copyright 2017-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Amazon Software License (the "License"). You may not use this file +except in compliance with the License. A copy of the License is located at + +http://aws.amazon.com/asl/ + +or in the "license" file accompanying this file. This file is distributed on an "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the +License for the specific language governing permissions and limitations under the License. +*/ +var query=require('query-string').stringify +var _=require('lodash') +var Promise=require('bluebird') +var axios=require('axios') +var Url=require('url') +var sign=require('aws4').sign +var path=require('path') +var Mutex=require('async-mutex').Mutex +var aws=require('aws-sdk') +const mutex = new Mutex(); + +var reason=function(r){ + return (err)=>{ + console.log(err) + Promise.reject(r) + } +} +var aws=require('aws-sdk') + +var failed=false +module.kendrasync={ + startExport:async function(context,opts){ + var info=await context.dispatch('_request',{ + url:context.rootState.info._links.jobs.href, + method:'get' + }) + var result=await context.dispatch('_request',{ + url:`${info._links.exports.href}/${opts.name}`, + method:'put', + body:opts.filter ? {filter:`${opts.filter}.*`} : {} + }) + }, + +} + + + + + + From 0d0d8867e45b87b762b76fbe0ef23345f1f1ef98 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 9 Jul 2020 18:46:52 +0000 Subject: [PATCH 018/137] export works --- lambda/export/createFAQ.js | 2 + lambda/export/kendraSync.js | 2 - lambda/export/test/syncEvent.json | 70 +++++++++++++++---------------- templates/export/resources.js | 60 +++++++++++++++++++++++--- 4 files changed, 92 insertions(+), 42 deletions(-) diff --git a/lambda/export/createFAQ.js b/lambda/export/createFAQ.js index 707ba93e0..1fa842ae4 100644 --- a/lambda/export/createFAQ.js +++ b/lambda/export/createFAQ.js @@ -147,6 +147,8 @@ async function createFAQ(params) { IndexId: params.faq_index_id } var del_faq_response = await faqDeleter(kendraClient, delete_faq_params); + } else { + console.log("No old FAQ to delete"); } diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js index 612942911..5e66da4e8 100644 --- a/lambda/export/kendraSync.js +++ b/lambda/export/kendraSync.js @@ -38,8 +38,6 @@ exports.performSync=function(event,context,cb){ kendra_s3_access_role:process.env.KENDRA_ROLE, region:process.env.REGION } - console.log('s3 bucket is ' + createFAQparams.s3_bucket); - console.log('kendra role is ' + createFAQparams.kendra_s3_access_role); create.handler(createFAQparams) console.log('Completed CSV converting to FAQ'); }); diff --git a/lambda/export/test/syncEvent.json b/lambda/export/test/syncEvent.json index 6abed757f..865e9211f 100644 --- a/lambda/export/test/syncEvent.json +++ b/lambda/export/test/syncEvent.json @@ -1,37 +1,37 @@ { - "Records": [ - { - "eventVersion": "2.0", - "eventSource": "aws:s3", - "awsRegion": "us-east-1", - "eventTime": "1970-01-01T00:00:00.000Z", - "eventName": "ObjectCreated:Put", - "userIdentity": { - "principalId": "EXAMPLE" - }, - "requestParameters": { - "sourceIPAddress": "127.0.0.1" - }, - "responseElements": { - "x-amz-request-id": "EXAMPLE123456789", - "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" - }, - "s3": { - "s3SchemaVersion": "1.0", - "configurationId": "testConfigRule", - "bucket": { - "name": "qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe", - "ownerIdentity": { - "principalId": "EXAMPLE" - }, - "arn": "arn:aws:s3:::example-bucket" - }, - "object": { - "key": "data/qna.json", - "size": 1024, - "eTag": "8f9ad97afed386ca0ff9452ef2ca198a" - } - } + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "EXAMPLE" + }, + "requestParameters": { + "sourceIPAddress": "127.0.0.1" + }, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j", + "ownerIdentity": { + "principalId": "425742325899" + }, + "arn": "arn:aws:s3:::qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j" + }, + "object": { + "key": "qna_FAQ.json", + "size": 1024, + "eTag": "51265b7064ef5fa26a8b1ffdc83d8d3c" } - ] -} + } + } + ] +} \ No newline at end of file diff --git a/templates/export/resources.js b/templates/export/resources.js index cb24c7dda..bf2e90d08 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -139,7 +139,7 @@ module.exports=Object.assign( "Variables": { "KENDRA_INDEX":{"Ref":"KendraIndex"}, "OUTPUT_S3_BUCKET":{"Ref":"ExportBucket"}, - "KENDRA_ROLE":{"Fn::GetAtt": ["KendraSyncRole","Arn"]}, + "KENDRA_ROLE":{"Fn::GetAtt": ["KendraS3Role","Arn"]}, "REGION":{"Ref":"AWS::Region"} } }, @@ -196,13 +196,63 @@ module.exports=Object.assign( "kendra:ListFaqs", "s3:ListBucket", "kendra:TagResource", - "kendra:DeleteFaq" + "kendra:DeleteFaq", + "iam:passRole" ], "Resource": [ - {"Fn::Sub":"arn:aws:kendra:::index/${KendraIndex}"}, - {"Fn::Sub":"arn:aws:kendra:::index/${KendraIndex}/faq/*"}, + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}"}, + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}/faq/*"}, {"Fn::Sub":"arn:aws:s3:::${ExportBucket}"}, - {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"} + {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"}, + {"Fn::GetAtt": ["KendraS3Role","Arn"]} + ] + }] + } + } + }, + "KendraS3Role": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + },{ + "Effect": "Allow", + "Principal": { + "Service": "kendra.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Path": "/", + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + {"Ref":"KendraS3Policy"} + ] + } + }, + "KendraS3Policy": { + "Type": "AWS::IAM::ManagedPolicy", + "Properties": { + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "kendra:CreateFaq", + ], + "Resource": [ + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}"}, + {"Fn::Sub":"arn:aws:s3:::${ExportBucket}"}, + {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"}, ] }] } From 9aaa980e237c1d184cac4a4726f45c49e24a33ee Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 9 Jul 2020 18:48:45 +0000 Subject: [PATCH 019/137] updated test, but it is user specific --- lambda/export/test/automateSyncTest.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index 92d0e651a..3565d2b85 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -43,9 +43,9 @@ async function test_create_faq() { faq_index_id:'e1c23860-e5c8-4409-ae26-b05bd6ced00a', csv_path:parseJSONparams.output_path, csv_name:parseJSONparams.csv_name, - s3_bucket:'qna-dev-dev-dev-master-2-exportbucket-nwlyflasajwe', + s3_bucket:'qna-dev-dev-dev-master-4-exportbucket-o5r0tsjifuu9', s3_key:"kendra_csv" + "/" + parseJSONparams.csv_name, - kendra_s3_access_role:'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-2-ExportStac-KendraSyncRole-1G3IEI1JF7L3S', + kendra_s3_access_role:'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-4-ExportStack-KendraS3Role-1D5W35EQT8OCX', region:'us-east-1' } return create.handler(createFAQparams); @@ -57,9 +57,9 @@ async function test_performSync() { const event = require('./syncEvent.json'); var context = undefined; var cb = undefined; - process.env.OUTPUT_S3_BUCKET = 'qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j' + process.env.OUTPUT_S3_BUCKET = 'qna-dev-dev-dev-master-4-exportbucket-o5r0tsjifuu9' process.env.KENDRA_INDEX = 'e1c23860-e5c8-4409-ae26-b05bd6ced00a'; - process.env.KENDRA_ROLE = 'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-3-ExportStac-KendraSyncRole-1RN4NKGMDFRNH' + process.env.KENDRA_ROLE = 'arn:aws:iam::425742325899:role/QNA-dev-dev-dev-master-4-ExportStack-KendraS3Role-1D5W35EQT8OCX' return kendraSync.performSync(event, context, cb); } From 69187809eb7a35f23245e9ed4371af8e486735a0 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 9 Jul 2020 21:32:39 +0000 Subject: [PATCH 020/137] alternative query function for kendra engine --- lambda/proxy-es/lib/query.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lambda/proxy-es/lib/query.js b/lambda/proxy-es/lib/query.js index 171392b24..1d54fc14d 100755 --- a/lambda/proxy-es/lib/query.js +++ b/lambda/proxy-es/lib/query.js @@ -69,10 +69,19 @@ async function get_hit(req, res) { score_answer_field: _.get(req, '_settings.ES_SCORE_ANSWER_FIELD'), fuzziness: _.get(req, '_settings.ES_USE_FUZZY_MATCH'), }; + + // TODO: find the parameter which says EnableKendraFAQ instead + if (_.get(req, "_settings.ALT_SEARCH_KENDRA_INDEXES").length > 0) { + // TODO: if Kendra engine is enabled, what 'question' do we want? + var kendra_question = _.get(req, '_settings.KENDRA_QUERY_QUESTION', 'no_hits_alternative'); + query_params.question = kendra_question; + } var no_hits_question = _.get(req, '_settings.ES_NO_HITS_QUESTION', 'no_hits'); + var response = await run_query(req, query_params); console.log("Query response: ", JSON.stringify(response,null,2)); var hit = _.get(response, "hits.hits[0]._source"); + if (hit) { res['got_hits'] = 1; // response flag, used in logging / kibana } else { From 1d2f567bd03b700bd7e391e60b0453d98333e5a0 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 14 Jul 2020 14:13:56 +0000 Subject: [PATCH 021/137] adding kendraquery function, adding sync kendra button --- bin/config.js | 2 +- lambda/export/ExportJSON.js | 2 +- lambda/export/kendraSync.js | 2 +- lambda/export/parseJSON.js | 3 +- lambda/export/test/automateSyncTest.js | 33 +-- lambda/export/test/qna.txt | 10 + lambda/export/test/qna_FAQ.csv | 26 +- lambda/export/test/qna_stats.json | 54 ++++ lambda/export/test/syncEvent.json | 8 +- lambda/proxy-es/lib/kendraQuery.js | 249 ++++++++++++++++++ lambda/proxy-es/lib/query.js | 30 ++- templates/dev/master.js | 2 +- .../KendraFallback/KendraFallback.js | 3 +- templates/export/index.js | 2 +- templates/export/resources.js | 8 +- templates/master/default-settings.js | 2 + templates/master/elasticsearch/proxy.js | 2 +- templates/master/exportstack.js | 2 +- templates/master/index.js | 4 +- templates/master/lex-build/index.js | 2 +- templates/master/lex/fulfillment.js | 2 +- templates/master/proxy-es.js | 17 +- templates/master/routes/jobs/export-start.vm | 5 +- templates/master/schemaLambda.js | 2 +- templates/public/index.js | 4 +- website/js/components/designer/synckendra.vue | 49 ++-- website/js/lib/store/api/actions/export.js | 13 +- website/js/lib/store/api/actions/sync.js | 52 ---- workshops/reinvent2019/templates/public.json | 10 +- 29 files changed, 447 insertions(+), 153 deletions(-) create mode 100644 lambda/export/test/qna.txt create mode 100644 lambda/export/test/qna_stats.json create mode 100644 lambda/proxy-es/lib/kendraQuery.js delete mode 100644 website/js/lib/store/api/actions/sync.js diff --git a/bin/config.js b/bin/config.js index 03044927b..4ff90f309 100644 --- a/bin/config.js +++ b/bin/config.js @@ -19,7 +19,7 @@ module.exports={ "devEmail":"", "namespace":"dev", "stackNamePrefix":"QNA", - "KendraIndex":"" + "KendraFAQIndex":"" } if (require.main === module) { diff --git a/lambda/export/ExportJSON.js b/lambda/export/ExportJSON.js index 67e783e02..24d0267ad 100644 --- a/lambda/export/ExportJSON.js +++ b/lambda/export/ExportJSON.js @@ -33,7 +33,7 @@ async function run_export(params,reply) { "faq_index_id":params.faq_index_id, "id":params.id, "config":"kendra-status/"+params.id, - "tmp":"kendra-tmp/"+params.id, + "tmp":"tmp/"+params.id, "key":"kendra-data/"+params.id, "filter":"", "status":"Started" diff --git a/lambda/export/kendraSync.js b/lambda/export/kendraSync.js index 5e66da4e8..06308b492 100644 --- a/lambda/export/kendraSync.js +++ b/lambda/export/kendraSync.js @@ -27,7 +27,7 @@ exports.performSync=function(event,context,cb){ } parse.handler(parseJSONparams).then(()=>{ console.log("Parsed content JSON into CSV stored locally"); - + var createFAQparams = { faq_name:'qna-facts', faq_index_id:process.env.KENDRA_INDEX, diff --git a/lambda/export/parseJSON.js b/lambda/export/parseJSON.js index f34c5065e..d7df993f2 100644 --- a/lambda/export/parseJSON.js +++ b/lambda/export/parseJSON.js @@ -18,7 +18,8 @@ function qnaJsonParser(params) { }); const data = []; - + // var qna = `{"qna":[${params.content.toString().replace(/\n/g,',\n')}]}` + // params.content = JSON.parse(qna).qna; const q_list = params.content.qna; q_list.forEach(function(elem) { elem.q.forEach(function(ques) { diff --git a/lambda/export/test/automateSyncTest.js b/lambda/export/test/automateSyncTest.js index 3565d2b85..9d1dc792a 100644 --- a/lambda/export/test/automateSyncTest.js +++ b/lambda/export/test/automateSyncTest.js @@ -4,19 +4,21 @@ var assert = require('assert'); async function test_parser() { const parseJSON = require('../parseJSON.js'); var content = require('./qna_export.json'); + // var content = require('./qna.txt') + // var qna = `{"qna":[${content.toString().replace(/\n/g,',\n')}]}` + // console.log(JSON.parse(qna).qna); + var parseJSONparams = { csv_name:'qna_FAQ.csv', content:content, - output_path:'/tmp/qna_FAQ.csv', + output_path:'./test/qna_FAQ.csv', } - const csv_path = await parseJSON.handler(parseJSONparams); - + const resp = await parseJSON.handler(parseJSONparams); const fs = require('fs') try { - if (fs.existsSync(csv_path)) { - //file exists - return 'exists'; + if (fs.existsSync(parseJSONparams.output_path)) { + return true; } else { return false; } @@ -24,7 +26,6 @@ async function test_parser() { console.error(err) return false; } - // TODO: CHECK CONTENTS OF CSV ROWS TO VALIDATE FORMAT } @@ -32,7 +33,7 @@ async function test_parser() { // create FAQ test async function test_create_faq() { const create = require('../createFAQ.js'); - var content = require('./qna_export.json'); + var content = require('./qna_FAQ.json'); var parseJSONparams = { csv_name:'qna_FAQ.csv', content:content, @@ -64,19 +65,19 @@ async function test_performSync() { } describe('#test automate-sync()', () => { - // it('test_json_parser', async function() { - // let resp = await test_parser(); - // assert(resp, "CSV file does not exist!"); - // }); + it('test_json_parser', async function() { + let resp = await test_parser(); + assert.equal(resp, true); + }); // it('test_create_faq', async function() { // let resp = await test_create_faq(); // assert(resp, 'Failed to create FAQ'); // }); - it('test_perform_sync', async function() { - let resp = await test_performSync(); - assert(resp, 'Synced'); - }); + // it('test_perform_sync', async function() { + // let resp = await test_performSync(); + // assert(resp, 'Synced'); + // }); }); diff --git a/lambda/export/test/qna.txt b/lambda/export/test/qna.txt new file mode 100644 index 000000000..a38373658 --- /dev/null +++ b/lambda/export/test/qna.txt @@ -0,0 +1,10 @@ +{"qid":"KendraFallback","a":"The Kendra Fallback search was not able to identify any results","l":"QNA:EXTKendraFallback","type":"qna","q":["no_hits_alternative"]} +{"a":"I am the QnA bot, ask me a question and I will try my best to answer it.","qid":"Help","type":"qna","q":["help","help me","need help"]} +{"a":"Thank you for your feedback - we will try to improve this answer.","qid":"Feedback.001","args":["incorrect"],"l":"QNA:ExamplePYTHONLambdaFeedback","next":"","r":{"subTitle":"","title":"","url":"","text":"","imageUrl":"","buttons":[{"text":"","value":""}]},"t":"","alt":{"ssml":"","markdown":""},"type":"qna","q":["Thumbs down","Bad answer"]} +{"qid":"stats.2","a":"Correlation is a measure of the strength of a linear relationship.","type":"qna","q":["Define correlation"]} +{"qid":"stats.4","a":"P(A or B) = P(A) + P(B) - P(A and B)","type":"qna","q":["What is the general addition theory?"]} +{"qid":"stats.5","a":"The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.","type":"qna","q":["What is a normal distribution?"]} +{"qid":"stats.1","a":"There are two types of categorical variables: ordinal and nominal variables. Ordinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating. Nominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.","type":"qna","q":["What are the types of categorical variables?"]} +{"qid":"stats.3","a":"There are 2 main methods to calculate the expected value of a random variable. Firstly, you could use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens. Alternatively, you can use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.","type":"qna","q":["How do I calculate expectation?"]} +{"a":"Thank you for your positive feedback on this answer, your feedback helps us continuously improve.","qid":"Feedback.002","args":["correct"],"l":"QNA:ExamplePYTHONLambdaFeedback","type":"qna","q":["Thumbs up","Good answer"]} +{"a":"{{Settings.EMPTYMESSAGE}}","qid":"CustomNoMatches","type":"qna","q":["no_hits"]} \ No newline at end of file diff --git a/lambda/export/test/qna_FAQ.csv b/lambda/export/test/qna_FAQ.csv index 14d6b98e1..394b6c61d 100644 --- a/lambda/export/test/qna_FAQ.csv +++ b/lambda/export/test/qna_FAQ.csv @@ -1,11 +1,17 @@ Question,Answer,Document URL (optional) -no_hits,The Kendra Fallback search was not able to identify any results,"{""qid"":""KendraFallback"",""a"":""The Kendra Fallback search was not able to identify any results"",""l"":""QNA:EXTKendraFallback"",""type"":""qna"",""q"":[""no_hits""]}" -How do I calculate expectation?,"There are 2 main methods to calculate the expected value of a random variable. -1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens. -2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.","{""qid"":""stat.4"",""a"":""There are 2 main methods to calculate the expected value of a random variable.\n1) Use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens.\n2) Use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation."",""type"":""qna"",""q"":[""How do I calculate expectation?""]}" -What are the types of categorical variables?,"There are two types of categorical variables: ordinal and nominal variables. -Ordinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating. -Nominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.","{""qid"":""stat.5"",""a"":""There are two types of categorical variables: ordinal and nominal variables.\nOrdinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating.\nNominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name."",""type"":""qna"",""q"":[""What are the types of categorical variables?""]}" -What is the general addition theory?,P(A or B) = P(A) + P(B) - P(A and B),"{""qid"":""stat.1"",""a"":""P(A or B) = P(A) + P(B) - P(A and B)"",""type"":""qna"",""q"":[""What is the general addition theory?""]}" -Define correlation,Correlation is a measure of the strength of a linear relationship.,"{""qid"":""stat.2"",""a"":""Correlation is a measure of the strength of a linear relationship."",""type"":""qna"",""q"":[""Define correlation""]}" -What is a normal distribution?,"The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.","{""qid"":""stat.3"",""a"":""The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation."",""type"":""qna"",""q"":[""What is a normal distribution?""]}" +Will the Sun ever consume the earth?,"When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant.","{""next"":"""",""args"":[""""],""a"":""When all the Hydrogen has been burned, the Sun will continue for about 130 million more years. During this time, it is consuming Helium and will expand to engulf Mercury, Venus, and the Earth. It is now a red giant."",""r"":{""subTitle"":"""",""imageUrl"":"""",""title"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.4"",""type"":""qna"",""selected"":false,""q"":[""Will the Sun ever consume the earth?""]}" +How large is the sun?,"The Sun is 109 times wider than the Earth and 330,000 times as massive","{""a"":""The Sun is 109 times wider than the Earth and 330,000 times as massive"",""r"":{""subTitle"":""Image referenced from space-facts.com"",""imageUrl"":""https://space-facts.com/wp-content/uploads/sun-size.png"",""title"":""Image""},""qid"":""sun.6"",""type"":""qna"",""q"":[""How large is the sun?""]}" +How hot is it at the center of the Sun?,At its centre the Sun reaches temperatures of 15 million °C.,"{""args"":[""""],""next"":"""",""a"":""At its centre the Sun reaches temperatures of 15 million °C."",""r"":{""buttons"":[{""text"":""How far away is the sun?"",""value"":""How far away is the sun?""}],""subTitle"":""Image referenced from solarsystem.nasa.gov"",""imageUrl"":""https://solarsystem.nasa.gov/system/resources/detail_files/823_cover-1920_detail.jpg"",""title"":""Sun Image"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""# Sun Temp\n* It has a surface temperature of 5,500 degrees C\n* At its centre the Sun reaches temperatures of **15 million °C**."",""ssml"":""\nAt its centre the Sun reaches temperatures of 15 million °C\n""},""l"":"""",""qid"":""sun.3"",""type"":""qna"",""q"":[""How hot is it at the center of the Sun?""]}" +Tell me about the sun?,"Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!","{""args"":[""""],""next"":"""",""a"":""Our sun is 4.6 billion years old. Its considered a yellow dwarf with a diameter of 1,392,684 kilometers and a circumference of 4,370,005 kilometers. It has a mass that is equal to 333,060 earths and a surface temperature of 5,500 degrees celsius. Really Hot!"",""r"":{""buttons"":[{""text"":""How hot is the sun"",""value"":""How hot is the sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""## Some facts\n* Our sun is 4.6 billion years old\n* It is considered a yellow dwarf\n* It has a mass that is equal to 333,060 earths"",""ssml"":"" Our Sun is 4.6 billion years old. It is considered a yellow dwarf and has a mass that is equal to 333,060 earths.""},""l"":"""",""qid"":""sun.1"",""type"":""qna"",""q"":[""Tell me about the sun?""]}" +How old is the Sun?,Our Sun is 4.6 billion years old.,"{""args"":[""""],""next"":"""",""a"":""Our Sun is 4.6 billion years old."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.2"",""type"":""qna"",""q"":[""How old is the Sun?""]}" +How far away is the sun?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" +What distance is the sun from the earth?,"The sun is 149,598,262 km (1 AU) from earth.","{""args"":[""""],""next"":"""",""a"":""The sun is 149,598,262 km (1 AU) from earth."",""r"":{""buttons"":[{""text"":""How large is the sun?"",""value"":""How large is the sun?""}],""subTitle"":""Image referenced from blogspot.com"",""imageUrl"":""https://3.bp.blogspot.com/-EQIjUw2H3To/WZvsoY4IrlI/AAAAAAAABIs/-iXnUDigA1cK2iZ9dV6AFp3v-P8kD_eewCLcBGAs/s1600/earthorbit.gif"",""title"":""Map""},""t"":"""",""alt"":{""markdown"":""### Distance\n\nThe sun is 149,598,262 km or 1 astronomical unit from the earth"",""ssml"":""""},""l"":"""",""qid"":""sun.5"",""type"":""qna"",""selected"":false,""q"":[""How far away is the sun?"",""What distance is the sun from the earth?""]}" +What other locations on the internet have information about the sun?,Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/.,"{""args"":[""""],""next"":"""",""a"":""Here is one place on the web you can go: https://solarsystem.nasa.gov/solar-system/sun/overview/."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Options""},""t"":"""",""alt"":{""markdown"":""[Nasa](https://solarsystem.nasa.gov/solar-system/sun/overview/ \""Nasa's Sun Page\"") is also a great source of information on the Sun."",""ssml"":""""},""l"":"""",""qid"":""sun.7"",""type"":""qna"",""selected"":false,""q"":[""What other locations on the internet have information about the sun?""]}" +What are solar flares?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +what is a solar flare?,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +Define solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +I'd like to know more about solar flares.,"A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions.","{""args"":[""""],""next"":"""",""a"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions."",""r"":{""buttons"":[{""text"":"""",""value"":""""}],""subTitle"":"""",""imageUrl"":"""",""title"":"""",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":""A brief eruption of intense high-energy radiation from the sun's surface, associated with sunspots and causing electromagnetic disturbances on the earth, as with radio frequency communications and power line transmissions. [NASA Definition](https://www.nasa.gov/content/goddard/what-is-a-solar-flare 'Solar Flare')"",""ssml"":""A brief eruption of intense high-energy radiation from the sun's surface.""},""l"":"""",""qid"":""sun.9"",""type"":""qna"",""q"":[""What are solar flares?"",""what is a solar flare?"",""Define solar flares."",""I'd like to know more about solar flares.""]}" +How many planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" +Which planets orbit the Sun?,"Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto.","{""args"":[""""],""next"":"""",""a"":""Eight planets orbit the sun. Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, and Neptune. Sorry I'm required to leave out my favorite, Pluto."",""r"":{""buttons"":[{""text"":""What is a solar flare?"",""value"":""What is a solar flare""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.10"",""type"":""qna"",""q"":[""How many planets orbit the Sun?"",""Which planets orbit the Sun?""]}" +What is the diameter of the Sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" +How wide is the sun?,The Sun's diameter is 1.4 million kilometers.,"{""args"":[""""],""next"":"""",""a"":""The Sun's diameter is 1.4 million kilometers."",""r"":{""buttons"":[{""text"":""How many planets orbit the Sun?"",""value"":""How many planets orbit the Sun""}],""subTitle"":"""",""imageUrl"":"""",""title"":""Suggestions"",""text"":"""",""url"":""""},""t"":"""",""alt"":{""markdown"":"""",""ssml"":""""},""l"":"""",""qid"":""sun.11"",""type"":""qna"",""q"":[""What is the diameter of the Sun?"",""How wide is the sun?""]}" diff --git a/lambda/export/test/qna_stats.json b/lambda/export/test/qna_stats.json new file mode 100644 index 000000000..371e08f41 --- /dev/null +++ b/lambda/export/test/qna_stats.json @@ -0,0 +1,54 @@ +{ + "qna": [ + { + "a": "I am the QnA bot, ask me a question and I will try my best to answer it.", + "qid": "Help", + "type": "qna", + "q": [ + "help", + "help me", + "need help" + ] + }, + { + "qid": "stats.2", + "a": "Correlation is a measure of the strength of a linear relationship.", + "type": "qna", + "q": [ + "Define correlation" + ] + }, + { + "qid": "stats.4", + "a": "P(A or B) = P(A) + P(B) - P(A and B)", + "type": "qna", + "q": [ + "What is the general addition theory?" + ] + }, + { + "qid": "stats.5", + "a": "The normal distribution, also known as the Gaussian distribution, is a type of continuous probability distribution for a real-valued random variable. It is a perfectly symmetric distribution that extends to both infinities defined by two parameters, its mean and standard deviation.", + "type": "qna", + "q": [ + "What is a normal distribution?" + ] + }, + { + "qid": "stats.1", + "a": "There are two types of categorical variables: ordinal and nominal variables. Ordinal variables have natural, ordered categories and the distances between the categories is not known. They exist on scales, such as socioeconomic status, education level, or satisfaction rating. Nominal variables take on qualitative values representing different categories, and there is no intrinsic ordering of these categories, like gender or name.", + "type": "qna", + "q": [ + "What are the types of categorical variables?" + ] + }, + { + "qid": "stats.3", + "a": "There are 2 main methods to calculate the expected value of a random variable. Firstly, you could use the definition: the expectation of a random variable is the probability of an event multiplied by the amount of times the event happens. Alternatively, you can use expectation by conditioning: define the random variable in terms of another, and condition on it. Then find the conditional expectation in terms of the other variable, and finally find the expectation.", + "type": "qna", + "q": [ + "How do I calculate expectation?" + ] + } + ] +} \ No newline at end of file diff --git a/lambda/export/test/syncEvent.json b/lambda/export/test/syncEvent.json index 865e9211f..03586ce1e 100644 --- a/lambda/export/test/syncEvent.json +++ b/lambda/export/test/syncEvent.json @@ -20,16 +20,16 @@ "s3SchemaVersion": "1.0", "configurationId": "testConfigRule", "bucket": { - "name": "qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j", + "name": "qna-dev-dev-dev-master-4-exportbucket-o5r0tsjifuu9", "ownerIdentity": { "principalId": "425742325899" }, - "arn": "arn:aws:s3:::qna-dev-dev-dev-master-3-exportbucket-1nvhu6nwzea5j" + "arn": "arn:aws:s3:::qna-dev-dev-dev-master-4-exportbucket-o5r0tsjifuu9" }, "object": { - "key": "qna_FAQ.json", + "key": "kendra_csv/qna_FAQ.json", "size": 1024, - "eTag": "51265b7064ef5fa26a8b1ffdc83d8d3c" + "eTag": "8f9ad97afed386ca0ff9452ef2ca198a" } } } diff --git a/lambda/proxy-es/lib/kendraQuery.js b/lambda/proxy-es/lib/kendraQuery.js new file mode 100644 index 000000000..2a76aa375 --- /dev/null +++ b/lambda/proxy-es/lib/kendraQuery.js @@ -0,0 +1,249 @@ +/** + * optional environment variables - These are not used defined during setup of this function in QnABot but are + * useful for testing if defined. + * + * REGION - optional AWS region to target + * + */ + +const AWSKendra = require('aws-sdk/clients/kendra'); +let kendraFaqIndex = undefined; + +/** + * Function to query kendraClient and return results via Promise + * @param kendraClient + * @param params + * @param resArray + * @returns {*} + */ +function kendraRequester(kendraClient,params) { + return new Promise(function(resolve, reject) { + kendraClient.query(params, function(err, data) { + let indexId = params.IndexId; + if (err) { + console.log(err, err.stack); + reject('Error from Kendra query request:' + err); + } + else { + data.originalKendraIndexId = indexId; + console.log("Data from Kendra request:" + JSON.stringify(data, null, 2)); + resolve(data); + } + }); + }); +} + + + +/** + * Function to sort and merge overlapping intervals + * @param intervals + * @returns [*] + * Source: https://gist.github.com/vrachieru/5649bce26004d8a4682b + */ +function mergeIntervals(intervals) { + // test if there are at least 2 intervals + if(intervals.length <= 1) + return intervals; + + var stack = []; + var top = null; + + // sort the intervals based on their start values + intervals.sort(function(a, b) {return a[0] - b[0]}); + + // push the 1st interval into the stack + stack.push(intervals[0]); + + // start from the next interval and merge if needed + for (var i = 1; i < intervals.length; i++) { + // get the top element + top = stack[stack.length - 1]; + + // if the current interval doesn't overlap with the + // stack top element, push it to the stack + if (top.EndOffset < intervals[i].BeginOffset) { + stack.push(intervals[i]); + } + // otherwise update the end value of the top element + // if end of current interval is higher + else if (top.EndOffset < intervals[i].EndOffset) + { + top.EndOffset = intervals[i].EndOffset; + stack.pop(); + stack.push(top); + } + } + + return stack; +} + + + +/** Function that returns if a string has JSON structure + * @param str - input string + * @returns boolean true or false + */ +function hasJsonStructure(str) { + if (typeof str !== 'string') return false; + try { + const result = JSON.parse(str); + const type = Object.prototype.toString.call(result); + return type === '[object Object]' + || type === '[object Array]'; + } catch (err) { + return false; + } +} + + + +/** Function that processes kendra requests and handles response. Decides whether to handle SNS + * events or Lambda Hook events from QnABot. + * @param event - input event passed to the Lambda Handler + * @param context - input context passed to the Lambda Handler + * @returns {Promise<*>} - returns the response in event.res + */ +async function routeKendraRequest(event, context) { + + // remove any prior session attributes for kendra + if (event && event.res && event.res.session.kendraQueryId) delete event.res.session.kendraQueryId; + if (event && event.res && event.res.session.kendraIndexId) delete event.res.session.kendraIndexId; + if (event && event.res && event.res.session.kendraResultId) delete event.res.session.kendraResultId; + if (event && event.res && event.res.session.kendraResponsibleQid) delete event.res.session.kendraResponsibleQid; + + + var kendraClient = (process.env.REGION ? + new AWSKendra({apiVersion: '2019-02-03', region: process.env.REGION}) : + new AWSKendra({apiVersion: '2019-02-03'}) + ); + + // process query against Kendra for QnABot + kendraFaqIndex = process.env.KENDRA_FAQ_INDEX; + if (kendraFaqIndex === undefined) { + throw new Error('Undefined Kendra Index'); + } + const params = { + IndexId: kendraFaqIndex, + QueryText: event.req["_event"].inputTranscript + } + var res = await Promise.all([kendraRequester(kendraClient, params)]) + + // process kendra query responses and update answer content + + /* default message text - can be overridden using QnABot SSM Parameter Store Custom Property */ + let answerMessage = 'While I did not find an exact answer, these search results from Amazon Kendra might be helpful. '; + let answerMessageMd = '*While I did not find an exact answer, these search results from Amazon Kendra might be helpful.* \n '; + let faqanswerMessage = 'Answer from Amazon Kendra FAQ.'; + let faqanswerMessageMd = '*Answer from Amazon Kendra FAQ.* \n '; + // let markdownAnswer = ""; + // let helpfulLinksMsg = 'Source Link'; + // let extractedTextMsg = 'Discovered Text'; + // let moreResultsMsg = 'Additional Search Results'; + let maxDocumentCount = 2; + var seenTop = false; + + let foundAnswerCount = 0; + // let foundDocumentCount = 0; + let kendraQueryId; + let kendraIndexId; + let kendraResultId; + // let answerDocumentUris = new Set(); + // let helpfulDocumentsUris = new Set(); + + if (res && res.ResultItems.length > 0) { + // helpfulLinksMsg = event.req["_settings"]["ALT_SEARCH_HELPFUL_LINKS_MSG"] ? event.req["_settings"]["ALT_SEARCH_HELPFUL_LINKS_MSG"] : helpfulLinksMsg; + // extractedTextMsg = event.req["_settings"]["ALT_SEARCH_EXTRACTED_TEXT_MSG"] ? event.req["_settings"]["ALT_SEARCH_EXTRACTED_TEXT_MSG"] : extractedTextMsg; + maxDocumentCount = event.req["_settings"]["ALT_SEARCH_MAX_DOCUMENT_COUNT"] ? event.req["_settings"]["ALT_SEARCH_MAX_DOCUMENT_COUNT"] : maxDocumentCount; + // answerMessage = event.req["_settings"]["ALT_SEARCH_MESSAGE"] ? event.req["_settings"]["ALT_SEARCH_MESSAGE"] : answerMessage; + // answerMessageMd = event.req["_settings"]["ALT_SEARCH_MESSAGE_MD"] ? event.req["_settings"]["ALT_SEARCH_MESSAGE_MD"] : answerMessageMd; + + res.ResultItems.forEach(function (element, i) { + /* Note - only FAQ format will be provided back to the requester */ + if (element.Type === 'QUESTION_ANSWER' && foundAnswerCount === 0 && element.AdditionalAttributes && + element.AdditionalAttributes.length > 1) { + // There will be 2 elements - [0] - QuestionText, [1] - AnswerText + answerMessage = faqanswerMessage + '\n\n ' + element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); + + seenTop = true; // if the answer is in the FAQ, don't show document extracts + let answerTextMd = element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " "); + // iterates over the FAQ answer highlights in sorted order of BeginOffset, merges the overlapping intervals + var sorted_highlights = mergeIntervals(element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Highlights); + var j, elem; + for (j=0; j 0) { + event.res.message = answerMessage; + let ssmlMessage = `${answerMessage.substring(0,600).replace(/\r?\n|\r/g, " ")}`; + let lastIndex = ssmlMessage.lastIndexOf('.'); + if (lastIndex > 0) { + ssmlMessage = ssmlMessage.substring(0,lastIndex); + } + ssmlMessage = ` ${ssmlMessage} `; + event.res.session.appContext.altMessages.markdown = answerMessageMd; + event.res.session.appContext.altMessages.ssml = ssmlMessage; + if (event.req["_event"].outputDialogMode !== 'Text') { + event.res.message = ssmlMessage; + event.res.type = 'SSML'; + event.res.plainMessage = answerMessage; + } + } + // if (answerDocumentUris.size > 0) { + // event.res.session.appContext.altMessages.markdown += `\n\n ${helpfulLinksMsg}: `; + // answerDocumentUris.forEach(function (element) { + // event.res.session.appContext.altMessages.markdown += `[${element}](${element})`; + // }); + // } + + // let idx=0; + // if (seenTop == false){ + // helpfulDocumentsUris.forEach(function (element) { + // if (idx++ < maxDocumentCount) { + // event.res.session.appContext.altMessages.markdown += `\n\n`; + // event.res.session.appContext.altMessages.markdown += `***`; + // event.res.session.appContext.altMessages.markdown += `\n\n
`; + + // if (element.text && element.text.length > 0) { + // event.res.session.appContext.altMessages.markdown += `\n\n ${element.text}`; + // } + // event.res.session.appContext.altMessages.markdown += `\n\n ${helpfulLinksMsg}: [${element.uri}](${element.uri})`; + // } + // }); + // } + + if (kendraQueryId) { + event.res.session.kendraResponsibleQid = event.res.result.qid; + event.res.session.kendraQueryId = kendraQueryId; + event.res.session.kendraIndexId = kendraIndexId; + event.res.session.kendraResultId = kendraResultId; + } + console.log("RETURN: " + JSON.stringify(event,null,2)); + return event; +} + +exports.handler = async (event, context) => { + console.log("event: " + JSON.stringify(event, null, 2)); + console.log('context: ' + JSON.stringify(context, null, 2)); + return routeKendraRequest(event, context); +}; diff --git a/lambda/proxy-es/lib/query.js b/lambda/proxy-es/lib/query.js index 1d54fc14d..045cb2707 100755 --- a/lambda/proxy-es/lib/query.js +++ b/lambda/proxy-es/lib/query.js @@ -6,12 +6,24 @@ var request = require('./request'); var build_es_query = require('./esbodybuilder'); var handlebars = require('./handlebars'); var translate = require('./translate'); +var kendra = require('./kendraQuery'); + // use DEFAULT_SETTINGS_PARAM as random encryption key unique to this QnABot installation var key = _.get(process.env, "DEFAULT_SETTINGS_PARAM", "fdsjhf98fd98fjh9 du98fjfd 8ud8fjdf"); var encryptor = require('simple-encryptor')(key); async function run_query(req, query_params) { + // TODO: add EnableKendraFAQ to _settings + if (_.get(req, "_settings.ALT_SEARCH_KENDRA_INDEXES").length > 0) { + return await run_query_kendra(req, query_params); + } else { + console.log('running kendra'); + return await run_query_es(req, query_params); + } +} + +async function run_query_es(req, query_params) { var es_query = await build_es_query(query_params); var es_response = await request({ url: `https://${req._info.es.address}/${req._info.es.index}/_doc/_search?search_type=dfs_query_then_fetch`, @@ -21,6 +33,16 @@ async function run_query(req, query_params) { return es_response; } +async function run_query_kendra(req, query_params) { + // new function duplicating KendraFallback code + var context=undefined; + var event = { + "req":req, + "res":undefined + }; + return await kendra.handler(event, context); +} + function merge_next(hit1, hit2) { if (hit1 === undefined) { return hit2; @@ -69,15 +91,7 @@ async function get_hit(req, res) { score_answer_field: _.get(req, '_settings.ES_SCORE_ANSWER_FIELD'), fuzziness: _.get(req, '_settings.ES_USE_FUZZY_MATCH'), }; - - // TODO: find the parameter which says EnableKendraFAQ instead - if (_.get(req, "_settings.ALT_SEARCH_KENDRA_INDEXES").length > 0) { - // TODO: if Kendra engine is enabled, what 'question' do we want? - var kendra_question = _.get(req, '_settings.KENDRA_QUERY_QUESTION', 'no_hits_alternative'); - query_params.question = kendra_question; - } var no_hits_question = _.get(req, '_settings.ES_NO_HITS_QUESTION', 'no_hits'); - var response = await run_query(req, query_params); console.log("Query response: ", JSON.stringify(response,null,2)); var hit = _.get(response, "hits.hits[0]._source"); diff --git a/templates/dev/master.js b/templates/dev/master.js index fb0ebb160..958de3ff7 100644 --- a/templates/dev/master.js +++ b/templates/dev/master.js @@ -10,7 +10,7 @@ module.exports=Promise.join( base.Parameters.BootstrapBucket.Default=output.Bucket base.Parameters.BootstrapPrefix.Default=output.Prefix base.Parameters.Email.Default=config.devEmail - base.Parameters.KendraIndex.Default=config.KendraIndex + base.Parameters.KendraFAQIndex.Default=config.KendraFAQIndex return base }) diff --git a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js index 04b021484..1b3071266 100644 --- a/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js +++ b/templates/examples/extensions/js_lambda_hooks/KendraFallback/KendraFallback.js @@ -142,7 +142,7 @@ async function routeKendraRequest(event, context) { } } if (kendraIndexes === undefined) { - throw new Error('Undefined Kendra Indexe'); + throw new Error('Undefined Kendra Indexes'); } // This function can handle configuration with an array of kendraIndexes. @@ -276,7 +276,6 @@ async function routeKendraRequest(event, context) { docInfo.uri = element.DocumentURI; helpfulDocumentsUris.add(docInfo); foundAnswerCount++; - foundDocumentCount++; // TODO: foundDocumentCount is not used elsewhere... } }); } diff --git a/templates/export/index.js b/templates/export/index.js index 44e2002ac..84b9d6f0f 100644 --- a/templates/export/index.js +++ b/templates/export/index.js @@ -20,6 +20,6 @@ module.exports={ "EsEndpoint": {"Type":"String"}, "EsProxyLambda": {"Type":"String"}, "ExportBucket": {"Type":"String"}, - "KendraIndex": {"Type":"String"} + "KendraFAQIndex": {"Type":"String"} } } \ No newline at end of file diff --git a/templates/export/resources.js b/templates/export/resources.js index bf2e90d08..806dd73cd 100644 --- a/templates/export/resources.js +++ b/templates/export/resources.js @@ -137,7 +137,7 @@ module.exports=Object.assign( }, "Environment": { "Variables": { - "KENDRA_INDEX":{"Ref":"KendraIndex"}, + "KENDRA_INDEX":{"Ref":"KendraFAQIndex"}, "OUTPUT_S3_BUCKET":{"Ref":"ExportBucket"}, "KENDRA_ROLE":{"Fn::GetAtt": ["KendraS3Role","Arn"]}, "REGION":{"Ref":"AWS::Region"} @@ -200,8 +200,8 @@ module.exports=Object.assign( "iam:passRole" ], "Resource": [ - {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}"}, - {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}/faq/*"}, + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraFAQIndex}"}, + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraFAQIndex}/faq/*"}, {"Fn::Sub":"arn:aws:s3:::${ExportBucket}"}, {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"}, {"Fn::GetAtt": ["KendraS3Role","Arn"]} @@ -250,7 +250,7 @@ module.exports=Object.assign( "kendra:CreateFaq", ], "Resource": [ - {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraIndex}"}, + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraFAQIndex}"}, {"Fn::Sub":"arn:aws:s3:::${ExportBucket}"}, {"Fn::Sub":"arn:aws:s3:::${ExportBucket}/*"}, ] diff --git a/templates/master/default-settings.js b/templates/master/default-settings.js index 688a4cbb0..2d22b5a1f 100644 --- a/templates/master/default-settings.js +++ b/templates/master/default-settings.js @@ -19,6 +19,8 @@ var default_settings = { ENABLE_MULTI_LANGUAGE_SUPPORT: "false", //User can override and set to true to Enable Multilanguage support MINIMUM_CONFIDENCE_SCORE: 0.6, //User can override this value to set the minimum confidence they accept using CustomQnABotSettings ALT_SEARCH_KENDRA_INDEXES: [], // Add Kendra index to array to enable Amazon Kendra as a fallback source of answers + ENABLE_KENDRA_FAQ: "false", + KENDRA_FAQ_INDEX: "", // Kendra Index specific for FAQ for if Kendra FAQ sync is enabled ELICIT_RESPONSE_MAX_RETRIES: 3, // Number of times an elicitResponse LexBot can be called before giving up when the Bot returns Failed ELICIT_RESPONSE_RETRY_MESSAGE: "Please try again?", // Default retry message when working with LexBot ELICIT_RESPONSE_BOT_FAILURE_MESSAGE: "Your response was not understood. Please start again.", // Message used when maximum number of retries is exceeded diff --git a/templates/master/elasticsearch/proxy.js b/templates/master/elasticsearch/proxy.js index 573fbe4b1..3c1e05332 100644 --- a/templates/master/elasticsearch/proxy.js +++ b/templates/master/elasticsearch/proxy.js @@ -58,7 +58,7 @@ module.exports={ "ServiceToken": { "Fn::GetAtt" : ["ESCFNProxyLambda", "Arn"] }, "create":{ index:{"Fn::Sub":"${Var.QnaIndex}"}, - endpoint:{"Fn::GetAtt":["ESVar","ESAddress"]}, + endpoint:{"Fn::GetAtt":["ESVar","ESAddress", "KendraFAQIndex"]}, body:{"Fn::Sub":JSON.stringify({ settings:require('./index_settings.js'), mappings:require('./index_mappings.js'), diff --git a/templates/master/exportstack.js b/templates/master/exportstack.js index e54ed66da..1a74b0074 100644 --- a/templates/master/exportstack.js +++ b/templates/master/exportstack.js @@ -12,7 +12,7 @@ module.exports={ "EsEndpoint": {"Fn::GetAtt": ["ESVar", "ESAddress"]}, "EsProxyLambda": {"Fn::GetAtt":["ESProxyLambda","Arn"]}, "ExportBucket": {"Ref":"ExportBucket"}, - "KendraIndex": {"Ref":"KendraIndex"} + "KendraFAQIndex": {"Ref":"KendraFAQIndex"} } } } diff --git a/templates/master/index.js b/templates/master/index.js index 9850b43fa..b04603ba2 100644 --- a/templates/master/index.js +++ b/templates/master/index.js @@ -175,7 +175,7 @@ module.exports={ "Default":"FALSE", "Description":"Defines whether Kendra FAQ integration is enabled" }, - "KendraIndex":{ + "KendraFAQIndex":{ "Type":"String", "Default":"" } @@ -187,7 +187,7 @@ module.exports={ "BuildExamples":{"Fn::Equals":[{"Ref":"BuildExamples"},"TRUE"]}, "CreateDomain":{"Fn::Equals":[{"Ref":"ElasticsearchName"},"EMPTY"]}, "DontCreateDomain":{"Fn::Not":[{"Fn::Equals":[{"Ref":"ElasticsearchName"},"EMPTY"]}]}, - "KendraFAQ":{"Fn::Equals":[{"Ref":"EnableKendraFAQ"}, "TRUE"]} + "KendraFAQIndex":{"Fn::Equals":[{"Ref":"EnableKendraFAQ"}, "TRUE"]} } } diff --git a/templates/master/lex-build/index.js b/templates/master/lex-build/index.js index a6a46dadc..a049b4897 100644 --- a/templates/master/lex-build/index.js +++ b/templates/master/lex-build/index.js @@ -100,7 +100,7 @@ module.exports={ "Path": "/", "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - {"Ref":"EsPolicy"}, + {"Ref":"QueryPolicy"}, "arn:aws:iam::aws:policy/AmazonLexFullAccess" ] } diff --git a/templates/master/lex/fulfillment.js b/templates/master/lex/fulfillment.js index 9f720a948..64f5aee9a 100644 --- a/templates/master/lex/fulfillment.js +++ b/templates/master/lex/fulfillment.js @@ -127,7 +127,7 @@ module.exports = { "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws:iam::aws:policy/TranslateReadOnly", - { "Ref": "EsPolicy" } + { "Ref": "QueryPolicy" } ], "Policies": [ { diff --git a/templates/master/proxy-es.js b/templates/master/proxy-es.js index 221a3ed65..1bba9f1ab 100644 --- a/templates/master/proxy-es.js +++ b/templates/master/proxy-es.js @@ -146,6 +146,8 @@ module.exports={ "Variables": { DEFAULT_SETTINGS_PARAM:{"Ref":"DefaultQnABotSettings"}, CUSTOM_SETTINGS_PARAM:{"Ref":"CustomQnABotSettings"}, + KENDRA_FAQ:{"Fn::GetAtt":["Var","EnableKendraFAQ"]}, + KENDRA_FAQ_INDEX:{"Fn::GetAtt":["Var","KendraFAQIndex"]} } }, "Handler": "index.query", @@ -206,7 +208,7 @@ module.exports={ "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws:iam::aws:policy/TranslateReadOnly", - {"Ref":"EsPolicy"}, + {"Ref":"QueryPolicy"}, "arn:aws:iam::aws:policy/AmazonLexFullAccess" ], "Policies": [ @@ -287,7 +289,7 @@ module.exports={ ] } }, - "EsPolicy": { + "QueryPolicy": { "Type": "AWS::IAM::ManagedPolicy", "Properties": { "PolicyDocument": { @@ -299,8 +301,15 @@ module.exports={ "es:*" ], "Resource":["*"] - }, - { + },{ + "Effect": "Allow", + "Action": [ + "kendra:Query" + ], + "Resource":[ + {"Fn::Sub":"arn:aws:kendra:${AWS::Region}:${AWS::AccountId}:index/${KendraFAQIndex}"}, + ] + },{ "Effect": "Allow", "Action": ["s3:Get*"], "Resource":[ diff --git a/templates/master/routes/jobs/export-start.vm b/templates/master/routes/jobs/export-start.vm index eb9c82663..e95d9d4cb 100644 --- a/templates/master/routes/jobs/export-start.vm +++ b/templates/master/routes/jobs/export-start.vm @@ -6,8 +6,7 @@ "id":"$input.params('proxy')", "config":"status/$input.params('proxy')", "tmp":"tmp/$input.params('proxy')", - "key":"data/$input.params('proxy')", + "key":"$inputRoot.get('prefix')data/$input.params('proxy')", "filter":"$inputRoot.get('filter')", "status":"Started" -} - +} \ No newline at end of file diff --git a/templates/master/schemaLambda.js b/templates/master/schemaLambda.js index fbcacde0e..9bab5860d 100644 --- a/templates/master/schemaLambda.js +++ b/templates/master/schemaLambda.js @@ -47,7 +47,7 @@ module.exports={ "Path": "/", "ManagedPolicyArns": [ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - {"Ref":"EsPolicy"} + {"Ref":"QueryPolicy"} ] } } diff --git a/templates/public/index.js b/templates/public/index.js index 80f36a34d..1cd4e6b57 100644 --- a/templates/public/index.js +++ b/templates/public/index.js @@ -30,13 +30,13 @@ module.exports=Promise.resolve(require('../master')).then(function(base){ "IntentFallback", "DefaultSettingsSSMParameterName", "CustomSettingsSSMParameterName", - "KendraIndex" + "KendraFAQIndex" ]) base.Parameters=_.pick(base.Parameters,[ "Email", "Username", - "KendraIndex", + "KendraFAQIndex", "EnableKendraFAQ" ]) base.Conditions.Public={"Fn::Equals":[true,true]} diff --git a/website/js/components/designer/synckendra.vue b/website/js/components/designer/synckendra.vue index 9e9474024..38c779130 100644 --- a/website/js/components/designer/synckendra.vue +++ b/website/js/components/designer/synckendra.vue @@ -1,7 +1,7 @@