diff --git a/server/app/query/servers.tsx b/server/app/query/servers.tsx
index b35d3e4..40494ef 100644
--- a/server/app/query/servers.tsx
+++ b/server/app/query/servers.tsx
@@ -10,6 +10,7 @@ export enum RemoteServerNames {
export interface ServerLog {
remoteServer: RemoteServer;
logLine: string;
+ timestamp: number;
}
export enum Status {
@@ -86,6 +87,10 @@ export class RemoteServer {
throw new Error("Not Implemented");
}
+ logURL(id: string): URL {
+ return new URL(`/start/${id}/log-file`, this.baseURL);
+ }
+
logsWebSocketURL(id: string): URL {
const webSocketURL = new URL(`/ws/logs/${id}`, this.baseURL);
webSocketURL.protocol = "wss";
@@ -122,13 +127,45 @@ export class RemoteServer {
): WebSocket {
const ws = this.logsSocket(id);
ws.onmessage = (event) => {
- const newLog: ServerLog = {
- remoteServer: this,
- logLine: event.data,
- };
-
- // only retain last 1000 logs
- setLogs((prevLogs) => [...prevLogs.slice(-1000), newLog]);
+ let newLog: ServerLog;
+ try {
+ const logValue = JSON.parse(event.data);
+ newLog = {
+ remoteServer: this,
+ logLine: logValue.record.message,
+ timestamp: logValue.record.time.timestamp,
+ };
+ } catch (e) {
+ newLog = {
+ remoteServer: this,
+ logLine: event.data,
+ timestamp: Date.now(),
+ };
+ }
+
+ // only retain last 10,000 logs
+ const maxNumLogs = 10000;
+ setLogs((prevLogs) => {
+ if (
+ prevLogs.length === 0 ||
+ newLog.timestamp >= prevLogs[prevLogs.length - 1].timestamp
+ ) {
+ // most the time, we put the new log at the end of the array
+ return [...prevLogs.slice(-maxNumLogs), newLog];
+ } else {
+ // if the timestamp is out of order, e.g., less than the
+ // end of the array, we put it in the right location
+ const lastPreviousLogIndex = prevLogs.findLastIndex(
+ (log) => log.timestamp < newLog.timestamp,
+ );
+
+ return [
+ ...prevLogs.slice(-maxNumLogs, lastPreviousLogIndex + 1),
+ newLog,
+ ...prevLogs.slice(lastPreviousLogIndex - 1),
+ ];
+ }
+ });
};
ws.onclose = (event) => {
console.log(
diff --git a/server/app/query/view/[id]/components.tsx b/server/app/query/view/[id]/components.tsx
index 187cac9..c9e7c72 100644
--- a/server/app/query/view/[id]/components.tsx
+++ b/server/app/query/view/[id]/components.tsx
@@ -88,31 +88,45 @@ export function LogViewer({
return (
- {logs.map((log, index) => (
+
+ {logs.map((log, index) => {
+ const date = new Date(log.timestamp * 1000);
+ return (
+
+
-
+ {date.toISOString()} | {log.remoteServer.remoteServerNameStr}:
+
+ -
+ {log.logLine}
+
+
+ );
+ })}
- {log.logLine}
+ {">_"}
- ))}
-
- {">_"}
-
+
);
diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx
index 6541b5f..c634270 100644
--- a/server/app/query/view/[id]/page.tsx
+++ b/server/app/query/view/[id]/page.tsx
@@ -11,6 +11,7 @@ import {
Status,
ServerLog,
RemoteServer,
+ RemoteServerNames,
RemoteServersType,
IPARemoteServers, //hack until the queryId is stored in a DB
StatusByRemoteServer,
@@ -30,6 +31,18 @@ export default function QueryPage({ params }: { params: { id: string } }) {
const [query, setQuery] = useState(null);
const [logs, setLogs] = useState([]);
+ const [selectedRemoteServerLogs, setSelectedRemoteServerLogs] = useState<
+ string[]
+ >(
+ Object.keys(RemoteServerNames).filter((item) => {
+ return isNaN(Number(item));
+ }),
+ );
+
+ const displayedLogs = logs.filter((item) =>
+ selectedRemoteServerLogs.includes(item.remoteServer.remoteServerNameStr),
+ );
+
const [statusByRemoteServer, setStatusByRemoteServer] =
useState(initialStatusByRemoteServer);
const [statsByRemoteServer, setStatsByRemoteServer] =
@@ -45,10 +58,29 @@ export default function QueryPage({ params }: { params: { id: string } }) {
setStatsHidden(!statsHidden);
}
+
const queryParams = Object.entries(
JSON.parse((query?.params as string) || "{}"),
);
+ function handleCheckbox(e: React.ChangeEvent) {
+ const remoteServer = e.target.id;
+
+ if (e.target.checked) {
+ setSelectedRemoteServerLogs((prevSelectedRemoteServers) => [
+ ...prevSelectedRemoteServers,
+ remoteServer,
+ ]);
+ } else {
+ setSelectedRemoteServerLogs((prevSelectedRemoteServers) =>
+ prevSelectedRemoteServers.filter(
+ (prevSelectedRemoteServer) =>
+ prevSelectedRemoteServer !== remoteServer,
+ ),
+ );
+ }
+ }
+
const kill = async (remoteServers: RemoteServersType) => {
const query: Query = await getQuery(params.id);
@@ -67,8 +99,10 @@ export default function QueryPage({ params }: { params: { id: string } }) {
(async () => {
const query: Query = await getQuery(params.id);
setQuery(query);
-
let webSockets: WebSocket[] = [];
+ // useEffect() gets called twice locally
+ // so this prevents the logs from being shown twice
+ setLogs([]);
for (const remoteServer of Object.values(IPARemoteServers)) {
const loggingWs = remoteServer.openLogSocket(query.uuid, setLogs);
const statusWs = remoteServer.openStatusSocket(
@@ -247,7 +281,58 @@ export default function QueryPage({ params }: { params: { id: string } }) {
- {!logsHidden && }
+ {!logsHidden && (
+ <>
+
+
+ >
+ )}
>
);
diff --git a/sidecar/app/query/base.py b/sidecar/app/query/base.py
index 41d5b5e..f3b1332 100644
--- a/sidecar/app/query/base.py
+++ b/sidecar/app/query/base.py
@@ -43,7 +43,7 @@ def __post_init__(self):
self._status_dir.mkdir(exist_ok=True)
self._logger_id = logger.add(
self.log_file_path,
- format="{extra[role]}: {message}",
+ serialize=True,
filter=lambda record: record["extra"].get("task") == self.query_id,
enqueue=True,
)
diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py
index 5d32fff..4ee7b2e 100644
--- a/sidecar/app/routes/start.py
+++ b/sidecar/app/routes/start.py
@@ -1,7 +1,10 @@
+import json
+from datetime import datetime
from pathlib import Path
from typing import Annotated
-from fastapi import APIRouter, BackgroundTasks, Form
+from fastapi import APIRouter, BackgroundTasks, Form, HTTPException
+from fastapi.responses import StreamingResponse
from ..local_paths import Paths
from ..query.base import Query
@@ -87,6 +90,38 @@ def get_ipa_helper_status(
return {"status": query.status.name}
+@router.get("/{query_id}/log-file")
+def get_ipa_helper_log_file(
+ query_id: str,
+):
+ query = Query.get_from_query_id(query_id)
+ if query is None:
+ return HTTPException(status_code=404, detail="Query not found")
+
+ def iterfile():
+ with open(query.log_file_path, "rb") as f:
+ for line in f:
+ try:
+ data = json.loads(line)
+ d = datetime.fromtimestamp(
+ float(data["record"]["time"]["timestamp"])
+ )
+ message = data["record"]["message"]
+ yield f"{d.isoformat()} - {message}\n"
+ except (json.JSONDecodeError, KeyError):
+ yield line
+
+ return StreamingResponse(
+ iterfile(),
+ headers={
+ "Content-Disposition": (
+ f'attachment; filename="{query_id}-{settings.role.name.title()}.log"'
+ )
+ },
+ media_type="text/plain",
+ )
+
+
@router.post("/ipa-query/{query_id}")
def start_ipa_test_query(
query_id: str,