From fa447941a85a73cf6b1212f5d48d486328a41916 Mon Sep 17 00:00:00 2001 From: Carlos Cruz Date: Tue, 27 Jun 2023 10:49:04 +0100 Subject: [PATCH] feat(app): add cors and correct parse of literature id (#8) --- app.js | 20 +++++++++-- controllers/publication.js | 2 +- controllers/publicationSummary.js | 6 ++-- package-lock.json | 55 +++++++++++++++++++++++++++++++ package.json | 6 ++-- routes/literature.js | 20 ++++++----- services/cacheService.js | 46 ++++++++++++++++++++++++++ utils/index.js | 3 ++ 8 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 services/cacheService.js diff --git a/app.js b/app.js index 41fcb12..23b9672 100644 --- a/app.js +++ b/app.js @@ -1,16 +1,32 @@ import express from "express"; +import cors from "cors"; + import logger from "./utils/logger.js"; import httpLogger from "./middlewares/httpLogger.js"; import literatureRouter from "./routes/literature.js"; import healthRouter from "./routes/health.js"; -import { normalizePort } from "./utils/index.js"; +import { normalizePort, isProduction, isDevelopment } from "./utils/index.js"; -var port = normalizePort(process.env.PORT || "8080"); +const port = normalizePort(process.env.PORT || "8080"); +const originRegExp = /^(.*\.)?opentargets\.(org|xwz)$/; const app = express(); + app.use(httpLogger); app.use(express.json()); +if (isDevelopment) { + app.use(cors()); +} + +if (isProduction) { + app.use( + cors({ + origin: [originRegExp], + }) + ); +} + app.use("/literature", literatureRouter); app.use("/health", healthRouter); diff --git a/controllers/publication.js b/controllers/publication.js index 34ff7e5..08c618d 100644 --- a/controllers/publication.js +++ b/controllers/publication.js @@ -2,7 +2,7 @@ import axios from "axios"; import { XMLParser } from "fast-xml-parser"; function URLPublicationFullText({ id }) { - const baseUrl = `https://www.ebi.ac.uk/europepmc/webservices/rest/PMC${id}/fullTextXML`; + const baseUrl = `https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`; const requestOptions = { method: "GET", headers: { diff --git a/controllers/publicationSummary.js b/controllers/publicationSummary.js index 38101f2..4499bc6 100644 --- a/controllers/publicationSummary.js +++ b/controllers/publicationSummary.js @@ -46,7 +46,7 @@ export const getPublicationSummary = async ({ targetSymbol, diseaseName, pmcId, - wbTracer, + wbTracer = null, }) => { const prompt = createPrompt({ targetSymbol, diseaseName }); @@ -58,11 +58,9 @@ export const getPublicationSummary = async ({ }); const docs = await textSplitter.createDocuments([text]); - logger.info(JSON.stringify({ wordCount, docsLength: docs.length })); - const chain = loadQAMapReduceChain(model); - logger.info("reauest to gpt"); + logger.info("request to openai"); if (wbTracer !== null) { wandb.log({ targetSymbol: targetSymbol, diff --git a/package-lock.json b/package-lock.json index 15b969a..82c0c9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@wandb/sdk": "^0.5.1", "axios": "^1.4.0", "cookie-parser": "~1.4.4", + "cors": "^2.8.5", "debug": "~2.6.9", "dotenv": "^16.0.3", "express": "~4.16.1", @@ -18,6 +19,7 @@ "http-errors": "~1.6.3", "langchain": "^0.0.78", "morgan": "~1.9.1", + "node-cache": "^5.1.2", "node-fetch": "^3.3.1", "pug": "2.0.0-beta11", "winston": "^3.8.2", @@ -454,6 +456,14 @@ "wordwrap": "0.0.2" } }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -581,6 +591,18 @@ "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", "hasInstallScript": true }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/cross-fetch": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.6.tgz", @@ -1657,6 +1679,17 @@ "node": ">= 0.6" } }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -3005,6 +3038,11 @@ "wordwrap": "0.0.2" } }, + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" + }, "color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -3112,6 +3150,15 @@ "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==" }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "cross-fetch": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.6.tgz", @@ -3806,6 +3853,14 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, + "node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "requires": { + "clone": "2.x" + } + }, "node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", diff --git a/package.json b/package.json index c0b6e2d..fcd592c 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,15 @@ "private": true, "type": "module", "scripts": { - "start": "node app.js", + "start": "NODE_ENV=development node app.js", "start:prod": "NODE_ENV=production node app.js", - "dev": "nodemon app.js" + "dev": "NODE_ENV=development nodemon app.js" }, "dependencies": { "@wandb/sdk": "^0.5.1", "axios": "^1.4.0", "cookie-parser": "~1.4.4", + "cors": "^2.8.5", "debug": "~2.6.9", "dotenv": "^16.0.3", "express": "~4.16.1", @@ -19,6 +20,7 @@ "http-errors": "~1.6.3", "langchain": "^0.0.78", "morgan": "~1.9.1", + "node-cache": "^5.1.2", "node-fetch": "^3.3.1", "pug": "2.0.0-beta11", "winston": "^3.8.2", diff --git a/routes/literature.js b/routes/literature.js index 649137c..91e61a7 100644 --- a/routes/literature.js +++ b/routes/literature.js @@ -1,11 +1,13 @@ import express from "express"; import { WandbTracer } from "@wandb/sdk/integrations/langchain"; +import * as dotenv from "dotenv"; + import { getPublicationPlainText } from "../controllers/publication.js"; import { getPublicationSummary, streamTest, } from "../controllers/publicationSummary.js"; -import * as dotenv from "dotenv"; +import { isDevelopment } from "../utils/index.js"; import logger from "../utils/logger.js"; dotenv.config(); @@ -46,13 +48,15 @@ router.post("/publication/summary/", async (req, res) => { const { pmcId, targetSymbol, diseaseName } = req.body.payload; const prettyDiseaseName = diseaseName.replace(/\s/g, "_"); - const wbIdWithRandom = `${pmcId}_${targetSymbol}_${prettyDiseaseName}_${Math.floor( - Math.random() * 1000 - )}`; - const wbTracer = await WandbTracer.init( - { project: "ot-explain", id: wbIdWithRandom }, - false - ); + const queryId = `${pmcId}_${targetSymbol}_${prettyDiseaseName}`; + const wbIdWithRandom = `${queryId}_${Math.floor(Math.random() * 1000)}`; + let wbTracer = null; + if (isDevelopment) { + wbTracer = await WandbTracer.init( + { project: "ot-explain", id: wbIdWithRandom }, + false + ); + } logger.info(`Request on pub summary`); diff --git a/services/cacheService.js b/services/cacheService.js new file mode 100644 index 0000000..82a4852 --- /dev/null +++ b/services/cacheService.js @@ -0,0 +1,46 @@ +import NodeCache from "node-cache"; + +class Cache { + constructor(ttlSeconds) { + this.cache = new NodeCache({ + stdTTL: ttlSeconds, + checkperiod: ttlSeconds * 0.2, + useClones: false, + }); + } + + get(key, storeFunction) { + const value = this.cache.get(key); + if (value) { + return Promise.resolve(value); + } + + return storeFunction().then((result) => { + this.cache.set(key, result); + return result; + }); + } + + del(keys) { + this.cache.del(keys); + } + + delStartWith(startStr = "") { + if (!startStr) { + return; + } + + const keys = this.cache.keys(); + for (const key of keys) { + if (key.indexOf(startStr) === 0) { + this.del(key); + } + } + } + + flush() { + this.cache.flushAll(); + } +} + +export default Cache; diff --git a/utils/index.js b/utils/index.js index 0c56649..24226b2 100644 --- a/utils/index.js +++ b/utils/index.js @@ -10,3 +10,6 @@ export function normalizePort(val) { } return false; } + +export const isDevelopment = process.env.NODE_ENV === "development"; +export const isProduction = process.env.NODE_ENV === "production";