diff --git a/server/app/query/servers.tsx b/server/app/query/servers.tsx index b35d3e4..40494ef 100644 --- a/server/app/query/servers.tsx +++ b/server/app/query/servers.tsx @@ -10,6 +10,7 @@ export enum RemoteServerNames { export interface ServerLog { remoteServer: RemoteServer; logLine: string; + timestamp: number; } export enum Status { @@ -86,6 +87,10 @@ export class RemoteServer { throw new Error("Not Implemented"); } + logURL(id: string): URL { + return new URL(`/start/${id}/log-file`, this.baseURL); + } + logsWebSocketURL(id: string): URL { const webSocketURL = new URL(`/ws/logs/${id}`, this.baseURL); webSocketURL.protocol = "wss"; @@ -122,13 +127,45 @@ export class RemoteServer { ): WebSocket { const ws = this.logsSocket(id); ws.onmessage = (event) => { - const newLog: ServerLog = { - remoteServer: this, - logLine: event.data, - }; - - // only retain last 1000 logs - setLogs((prevLogs) => [...prevLogs.slice(-1000), newLog]); + let newLog: ServerLog; + try { + const logValue = JSON.parse(event.data); + newLog = { + remoteServer: this, + logLine: logValue.record.message, + timestamp: logValue.record.time.timestamp, + }; + } catch (e) { + newLog = { + remoteServer: this, + logLine: event.data, + timestamp: Date.now(), + }; + } + + // only retain last 10,000 logs + const maxNumLogs = 10000; + setLogs((prevLogs) => { + if ( + prevLogs.length === 0 || + newLog.timestamp >= prevLogs[prevLogs.length - 1].timestamp + ) { + // most the time, we put the new log at the end of the array + return [...prevLogs.slice(-maxNumLogs), newLog]; + } else { + // if the timestamp is out of order, e.g., less than the + // end of the array, we put it in the right location + const lastPreviousLogIndex = prevLogs.findLastIndex( + (log) => log.timestamp < newLog.timestamp, + ); + + return [ + ...prevLogs.slice(-maxNumLogs, lastPreviousLogIndex + 1), + newLog, + ...prevLogs.slice(lastPreviousLogIndex - 1), + ]; + } + }); }; ws.onclose = (event) => { console.log( diff --git a/server/app/query/view/[id]/components.tsx b/server/app/query/view/[id]/components.tsx index 187cac9..c9e7c72 100644 --- a/server/app/query/view/[id]/components.tsx +++ b/server/app/query/view/[id]/components.tsx @@ -88,31 +88,45 @@ export function LogViewer({ return (
- {logs.map((log, index) => ( +
+ {logs.map((log, index) => { + const date = new Date(log.timestamp * 1000); + return ( +
+
+ {date.toISOString()} | {log.remoteServer.remoteServerNameStr}: +
+
+ {log.logLine} +
+
+ ); + })}
- {log.logLine} + {">_"}
- ))} -
- {">_"} -
+
); diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 6541b5f..c634270 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -11,6 +11,7 @@ import { Status, ServerLog, RemoteServer, + RemoteServerNames, RemoteServersType, IPARemoteServers, //hack until the queryId is stored in a DB StatusByRemoteServer, @@ -30,6 +31,18 @@ export default function QueryPage({ params }: { params: { id: string } }) { const [query, setQuery] = useState(null); const [logs, setLogs] = useState([]); + const [selectedRemoteServerLogs, setSelectedRemoteServerLogs] = useState< + string[] + >( + Object.keys(RemoteServerNames).filter((item) => { + return isNaN(Number(item)); + }), + ); + + const displayedLogs = logs.filter((item) => + selectedRemoteServerLogs.includes(item.remoteServer.remoteServerNameStr), + ); + const [statusByRemoteServer, setStatusByRemoteServer] = useState(initialStatusByRemoteServer); const [statsByRemoteServer, setStatsByRemoteServer] = @@ -45,10 +58,29 @@ export default function QueryPage({ params }: { params: { id: string } }) { setStatsHidden(!statsHidden); } + const queryParams = Object.entries( JSON.parse((query?.params as string) || "{}"), ); + function handleCheckbox(e: React.ChangeEvent) { + const remoteServer = e.target.id; + + if (e.target.checked) { + setSelectedRemoteServerLogs((prevSelectedRemoteServers) => [ + ...prevSelectedRemoteServers, + remoteServer, + ]); + } else { + setSelectedRemoteServerLogs((prevSelectedRemoteServers) => + prevSelectedRemoteServers.filter( + (prevSelectedRemoteServer) => + prevSelectedRemoteServer !== remoteServer, + ), + ); + } + } + const kill = async (remoteServers: RemoteServersType) => { const query: Query = await getQuery(params.id); @@ -67,8 +99,10 @@ export default function QueryPage({ params }: { params: { id: string } }) { (async () => { const query: Query = await getQuery(params.id); setQuery(query); - let webSockets: WebSocket[] = []; + // useEffect() gets called twice locally + // so this prevents the logs from being shown twice + setLogs([]); for (const remoteServer of Object.values(IPARemoteServers)) { const loggingWs = remoteServer.openLogSocket(query.uuid, setLogs); const statusWs = remoteServer.openStatusSocket( @@ -247,7 +281,58 @@ export default function QueryPage({ params }: { params: { id: string } }) { - {!logsHidden && } + {!logsHidden && ( + <> +
+
+
    + {Object.values(IPARemoteServers).map( + (remoteServer: RemoteServer) => { + return ( + <> +
  • + +
    +
    + + {remoteServer.remoteServerNameStr}- + {query?.uuid} + .log + +
    +
    + {query && ( + + )} +
  • + + ); + }, + )} +
+
+
+ + + )} ); diff --git a/sidecar/app/query/base.py b/sidecar/app/query/base.py index 41d5b5e..f3b1332 100644 --- a/sidecar/app/query/base.py +++ b/sidecar/app/query/base.py @@ -43,7 +43,7 @@ def __post_init__(self): self._status_dir.mkdir(exist_ok=True) self._logger_id = logger.add( self.log_file_path, - format="{extra[role]}: {message}", + serialize=True, filter=lambda record: record["extra"].get("task") == self.query_id, enqueue=True, ) diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py index 5d32fff..4ee7b2e 100644 --- a/sidecar/app/routes/start.py +++ b/sidecar/app/routes/start.py @@ -1,7 +1,10 @@ +import json +from datetime import datetime from pathlib import Path from typing import Annotated -from fastapi import APIRouter, BackgroundTasks, Form +from fastapi import APIRouter, BackgroundTasks, Form, HTTPException +from fastapi.responses import StreamingResponse from ..local_paths import Paths from ..query.base import Query @@ -87,6 +90,38 @@ def get_ipa_helper_status( return {"status": query.status.name} +@router.get("/{query_id}/log-file") +def get_ipa_helper_log_file( + query_id: str, +): + query = Query.get_from_query_id(query_id) + if query is None: + return HTTPException(status_code=404, detail="Query not found") + + def iterfile(): + with open(query.log_file_path, "rb") as f: + for line in f: + try: + data = json.loads(line) + d = datetime.fromtimestamp( + float(data["record"]["time"]["timestamp"]) + ) + message = data["record"]["message"] + yield f"{d.isoformat()} - {message}\n" + except (json.JSONDecodeError, KeyError): + yield line + + return StreamingResponse( + iterfile(), + headers={ + "Content-Disposition": ( + f'attachment; filename="{query_id}-{settings.role.name.title()}.log"' + ) + }, + media_type="text/plain", + ) + + @router.post("/ipa-query/{query_id}") def start_ipa_test_query( query_id: str,