From 8250a2baccf20eaca7fa59d6cad963c77407a15d Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Tue, 8 Oct 2024 14:40:05 +0100 Subject: [PATCH] New server side endpoint for getting (all) mutations, which provides data in chunks (#617) * mutation chunks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * exclude helm charts from prettier * adjust prettier exclusion * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .prettierignore | 1 + .../taxonium-backend/templates/updater.yaml | 2 +- taxonium_backend/server.js | 43 ++++++++++- .../src/hooks/useServerBackend.jsx | 73 ++++++++++++++++--- 4 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 .prettierignore diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..a020d7d1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +helm_charts/** diff --git a/helm_charts/taxonium-backend/templates/updater.yaml b/helm_charts/taxonium-backend/templates/updater.yaml index ecc4ec41..6b8fb3ec 100644 --- a/helm_charts/taxonium-backend/templates/updater.yaml +++ b/helm_charts/taxonium-backend/templates/updater.yaml @@ -42,4 +42,4 @@ subjects: roleRef: kind: Role name: deployment-restarter - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/taxonium_backend/server.js b/taxonium_backend/server.js index c336bade..3d64636b 100644 --- a/taxonium_backend/server.js +++ b/taxonium_backend/server.js @@ -13,6 +13,7 @@ var pako = require("pako"); const URL = require("url").URL; const ReadableWebToNodeStream = require("readable-web-to-node-stream"); const { execSync } = require("child_process"); +const { Readable } = require("stream"); var importing; var filtering; var exporting; @@ -206,7 +207,6 @@ app.get("/config", function (req, res) { (processedData.overallMinY + processedData.overallMaxY) / 2; config.initial_zoom = -2; config.genes = processedData.genes; - config.mutations = processedData.mutations; config = { ...config, ...processedData.overwrite_config }; config.rootMutations = processedData.rootMutations; config.rootId = processedData.rootId; @@ -214,6 +214,47 @@ app.get("/config", function (req, res) { res.send(config); }); +app.get("/mutations/", function (req, res) { + // Set headers for SSE + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + // Function to send SSE + function sendSSE(data) { + res.write(`data: ${data}\n\n`); + } + + // Send mutations in chunks of 1000 + const chunkSize = 10000; + let index = 0; + + function sendNextChunk() { + const chunk = processedData.mutations.slice(index, index + chunkSize); + if (chunk.length > 0) { + sendSSE(JSON.stringify(chunk)); + index += chunkSize; + // Schedule the next chunk + setImmediate(sendNextChunk); + } else { + // All mutations sent, end the stream + sendSSE("END"); + res.end(); + } + } + + // Start sending chunks + sendNextChunk(); + + // Handle client disconnect + req.on("close", () => { + // No need to destroy a stream, just stop the process + index = processedData.mutations.length; // This will stop sendNextChunk on next iteration + }); +}); + app.get("/nodes/", function (req, res) { const start_time = Date.now(); let min_y = diff --git a/taxonium_component/src/hooks/useServerBackend.jsx b/taxonium_component/src/hooks/useServerBackend.jsx index dcd6f8b1..e149db62 100644 --- a/taxonium_component/src/hooks/useServerBackend.jsx +++ b/taxonium_component/src/hooks/useServerBackend.jsx @@ -121,20 +121,69 @@ function useServerBackend(backend_url, sid, url_on_fail) { }, [backend_url, sid] ); - const getConfig = useCallback( (setResult) => { - let url = backend_url + "/config/?sid=" + sid; - axios.get(url).then(function (response) { - console.log("got config", response.data); - if (response.data.error) { - window.alert( - response.data.error + (url_on_fail ? "\nRedirecting you." : "") - ); - window.location.href = url_on_fail; - } - setResult(response.data); - }); + const url = `${backend_url}/config/?sid=${sid}`; + + // Fetch initial config + axios + .get(url) + .then((response) => { + console.log("got config", response.data); + if (response.data.error) { + window.alert( + response.data.error + (url_on_fail ? "\nRedirecting you." : "") + ); + window.location.href = url_on_fail; + return; + } + + const config = response.data; + config.mutations = config.mutations ? config.mutations : []; + + // Stream mutations + const mutationsUrl = `${backend_url}/mutations/?sid=${sid}`; + const eventSource = new EventSource(mutationsUrl); + + eventSource.onmessage = (event) => { + if (event.data === "END") { + console.log("Finished receiving mutations"); + eventSource.close(); + setResult(config); + return; + } + + try { + const mutationsChunk = JSON.parse(event.data); + if (Array.isArray(mutationsChunk)) { + config.mutations.push(...mutationsChunk); + setResult({ ...config }); + console.log( + `Received chunk of ${mutationsChunk.length} mutations` + ); + } else { + console.error("Received non-array chunk:", mutationsChunk); + } + } catch (error) { + console.error("Error parsing mutations chunk:", error); + } + }; + + eventSource.onerror = (error) => { + console.error("EventSource failed:", error); + eventSource.close(); + }; + + // Set initial config + setResult(config); + }) + .catch((error) => { + console.error("Error fetching config:", error); + if (url_on_fail) { + window.alert("Failed to fetch config. Redirecting you."); + window.location.href = url_on_fail; + } + }); }, [backend_url, sid, url_on_fail] );