Analyze only a slice of file (#5)

* analyze only a subset of big files

Author: vish9812

README.md

@@ -43,7 +43,7 @@ For details, refer to its [README.md](https://github.com/vish9812/analog/blob/ma
 
 ## Prerequisite
 
-- Ensure you have Python 3 installed on your system then you can simply execute the `analog script` to run the app.
+- Ensure you have Python 3 [installed](https://www.python.org/downloads/) on your system then you can simply execute the Python server command to run the app.
 - Otherwise host the `index.html` manually on any server of your preference.
 
 ## Getting Started
@@ -56,19 +56,39 @@ Follow these steps to run the app:
 
 3. Open a terminal and navigate to the app's directory.
 
-4. Execute the `analog script` to start the app.
+4. Execute the following `python` command to start the app:
 
-- For Non-Windows, execute the script `./analog.sh`.
-- For Windows, use powershell:
-  - _Unblock_(one-time operation) the powershell file `analog.ps1` to be allowed to execute on the system: `Unblock-File .\analog.ps1`
-  - Execute the script `.\analog.ps1`
+- For Linux/Mac: `python3 -m http.server 20002`
+- For Windows: `python -m http.server 20002`
 
 5. Open your web browser and visit the following URL: `localhost:20002`
 
 You are now ready to use the Analog and take advantage of its powerful log analysis features.
 
 Enjoy analyzing your log data!
 
+## FAQ
+
+### Expected log format
+
+Following are the 3 must have keys in the logs:
+
+- level
+- timestamp
+- msg
+
+Example Format for JSON logs:
+
+```
+{"timestamp":"2023-10-16 10:13:16.710 +11:00","level":"debug","msg":"Received HTTP request","dynamicKey1":"value 1","dynamicKey2":"value 2"}
+```
+
+Example Format for plain-text logs:
+
+```
+debug [2023-10-16 10:13:16.710 +11:00] Received HTTP request dynamicKey1="value 1" dynamicKey2=value 2
+```
+
 ## License
 
 This app is released under the [MIT License](https://github.com/vish9812/analog/blob/main/LICENSE).

analog.ps1

@@ -13,7 +13,7 @@
     "test": "vitest --silent",
     "test-logs": "vitest",
     "coverage": "vitest run --coverage --silent",
-    "build": "vite build && cp analog.sh analog && cp analog.ps1 analog && bun build ./src/cmd/index.ts --compile --outfile ./analog/analog && zip -r analog.zip analog",
+    "build": "vite build && bun build ./src/cmd/index.ts --compile --outfile ./analog/analog && zip -r analog.zip analog",
     "preview": "vite preview"
   },
   "license": "MIT",

analog.sh

@@ -115,7 +115,8 @@ describe("init", () => {
     }
 
     const logData = new LogData();
-    logData.init(file as any, logsIterator);
+    logData.initFileInfo(file as any);
+    logData.init(logsIterator);
 
     expect(logData.fileInfo.name, "file name").toEqual(file.name);
     expect(logData.fileInfo.size, "file size").toEqual(file.size);

package.json

@@ -69,9 +69,7 @@ class LogData {
     error: "error",
   };
 
-  init(file: File, logsGeneratorFn: () => LogsGenerator) {
-    this.initFileInfo(file);
-
+  init(logsGeneratorFn: () => LogsGenerator) {
     const summaryMap: SummaryMap = {
       httpCodes: new Map<string, GroupedMsg>(),
       jobs: new Map<string, GroupedMsg>(),
@@ -83,7 +81,6 @@ class LogData {
     let count = 0;
     for (const log of logsGeneratorFn()) {
       if (log == null) {
-        console.warn("non-supported log format.");
         continue;
       }
 
@@ -107,7 +104,7 @@ class LogData {
     );
   }
 
-  private initFileInfo(file: File) {
+  initFileInfo(file: File) {
     this.fileInfo = {
       name: file.name,
       size: file.size,

src/models/logData.test.ts

@@ -8,6 +8,7 @@ import {
   ListItemIcon,
   ListItemText,
   Stack,
+  TextField,
   styled,
 } from "@suid/material";
 import useViewModel from "./useViewModel";
@@ -41,13 +42,31 @@ function Normalize() {
     processingFile,
     handleAnalyzeClick,
     handleFileUpload,
+    setTimeRange,
   } = useViewModel();
 
   const newFileText = () =>
     logDatas().length === 1 ? "Compare With" : "New File";
 
   return (
     <Grid container spacing={2}>
+      <Grid item xs={12}>
+        <p>Following are the 3 must have keys in the logs:</p>
+        <pre>{`
+- level
+- timestamp
+- msg
+      `}</pre>
+        <p>Example Format for JSON logs:</p>
+        <pre>{`
+{"timestamp":"2023-10-16 10:13:16.710 +11:00","level":"debug","msg":"Received HTTP request","dynamicKey1":"value 1","dynamicKey2":"value 2"}
+        `}</pre>
+
+        <p>Example Format for plain-text logs:</p>
+        <pre>{`
+debug [2023-10-16 10:13:16.710 +11:00] Received HTTP request dynamicKey1="value 1" dynamicKey2=value 2
+        `}</pre>
+      </Grid>
       <Grid item xs={12}>
         <Stack
           direction="row"
@@ -66,7 +85,11 @@ function Normalize() {
             <HiddenInput
               type="file"
               multiple={false}
-              onChange={(e) => handleFileUpload(e.target.files!, new LogData())}
+              onChange={(e) => {
+                handleFileUpload(e.target.files, new LogData());
+                // To allow uploading the same file again and triggering the "onChange" event for the same file
+                e.target.value = "";
+              }}
             />
           </Button>
           <Show when={processingFile()}>
@@ -86,21 +109,45 @@ function Normalize() {
         </Stack>
       </Grid>
       <Show when={!!logDatas().length}>
-        <Grid item xs={3}>
+        <Grid item xs={12} textAlign="center">
+          You can <em>optionally</em> filter the files by time
+          <br />- In case of big files( &gt; 70MB) for faster processing of data
+          <br />- To compare different time slices of the same or different
+          files
+        </Grid>
+        <Grid item xs={12}>
           <List subheader="Files">
             <For each={logDatas()}>
-              {(logData) => {
+              {(logData, idx) => {
                 return (
                   <ListItem disablePadding>
-                    <ListItemButton>
-                      <ListItemIcon>
-                        <InsertDriveFileIcon />
-                      </ListItemIcon>
-                      <ListItemText
-                        primary={logData.fileInfo.name}
-                        secondary={prettyBytes(logData.fileInfo.size)}
-                      />
-                    </ListItemButton>
+                    <Grid container spacing={2}>
+                      <Grid item xs={3}>
+                        <ListItemButton>
+                          <ListItemIcon>
+                            <InsertDriveFileIcon />
+                          </ListItemIcon>
+                          <ListItemText
+                            primary={logData.fileInfo.name}
+                            secondary={prettyBytes(logData.fileInfo.size)}
+                          />
+                        </ListItemButton>
+                      </Grid>
+                      <Grid item xs={3}>
+                        <TextField
+                          fullWidth={true}
+                          label="Start Time"
+                          onChange={(_, val) => setTimeRange(idx(), "min", val)}
+                        />
+                      </Grid>
+                      <Grid item xs={3}>
+                        <TextField
+                          fullWidth={true}
+                          label="End Time"
+                          onChange={(_, val) => setTimeRange(idx(), "max", val)}
+                        />
+                      </Grid>
+                    </Grid>
                   </ListItem>
                 );
               }}

src/models/logData.ts

@@ -24,8 +24,11 @@ describe("useViewModel", () => {
     createRoot((dispose) => {
       const setPage = vi.fn();
 
-      useViewModel().handleAnalyzeClick(setPage);
+      const vm = useViewModel();
+      vm.handleAnalyzeClick(setPage);
 
+      expect(vm.processingFile(), "processingFile").toEqual(false);
+      expect(vm.analyzeDisabled(), "analyzeDisabled").toEqual(false);
       expect(setPage, "setPage").toHaveBeenCalledWith(Pages.analyze);
 
       dispose();
@@ -47,20 +50,15 @@ describe("useViewModel", () => {
       "$filesLen file uploaded",
       async ({ files, newFileDisabled, filesLen }) => {
         const logData = new LogData();
-        const spyInit = vi.spyOn(normalizer, "init").mockResolvedValue();
-        const spyAddLogData = vi.spyOn(comparer, "addLogData");
 
         await vm.handleFileUpload(files as any, logData);
 
         expect(vm.analyzeDisabled(), "analyzeDisabled").toEqual(false);
-        expect(vm.processingFile(), "processingFile").toEqual(false);
         expect(vm.newFileDisabled(), "newFileDisabled").toEqual(
           newFileDisabled
         );
         expect(vm.logDatas(), "logDatas").toBeTruthy();
-        expect(vm.logDatas().length, "logDatas().length").toEqual(filesLen);
-        expect(spyInit, "spyInit").toHaveBeenCalledWith(logData, files[0]);
-        expect(spyAddLogData, "spyAddLogData").toHaveBeenCalledWith(logData);
+        expect(vm.logDatas().length, "length").toEqual(filesLen);
       }
     );
 

src/pages/normalize/index.tsx

@@ -1,32 +1,63 @@
 import comparer from "@al/services/comparer";
-import LogData from "@al/models/logData";
+import LogData, { type JSONLog } from "@al/models/logData";
 import { Setter, createSignal } from "solid-js";
 import { PagesValues, Pages } from "../usePage";
 import normalizer from "@al/services/normalizer";
+import { createStore } from "solid-js/store";
+
+// Temporarily store files
+// Empty the array once processed to free the memory
+let logFiles: File[] = [];
+
+interface TimeRange {
+  min: string;
+  max: string;
+}
 
 function useViewModel() {
   const [analyzeDisabled, setAnalyzeDisabled] = createSignal(true);
   const [processingFile, setProcessingFile] = createSignal(false);
+  const [timeRange, setTimeRange] = createStore<TimeRange[]>([]);
   const [logDatas, setLogDatas] = createSignal<LogData[]>([]);
   const newFileDisabled = () => logDatas().length > 1;
 
-  const handleAnalyzeClick = (setPage: Setter<PagesValues>) => {
+  async function handleAnalyzeClick(setPage: Setter<PagesValues>) {
+    setProcessingFile(true);
+    setAnalyzeDisabled(true);
+
+    for (let i = 0; i < logFiles.length; i++) {
+      const logData = logDatas()[i];
+      await normalizer.init(logData, logFiles[i], getFilterFn(timeRange[i]));
+      comparer.addLogData(logData);
+    }
+
+    setAnalyzeDisabled(false);
+    setProcessingFile(false);
+
+    logFiles = [];
+
     setPage(Pages.analyze);
-  };
+  }
 
-  const handleFileUpload = async (files: FileList, logData: LogData) => {
+  function handleFileUpload(files: FileList | null, logData: LogData) {
     if (!files || !files.length) return;
 
-    setProcessingFile(true);
-    setAnalyzeDisabled(true);
+    const file = files[0];
+    logData.initFileInfo(file);
 
-    await normalizer.init(logData, files[0]);
-    comparer.addLogData(logData);
+    logFiles.push(file);
     setLogDatas((prev) => [...prev, logData]);
-
+    setTimeRange(timeRange.length, { min: "", max: "" });
     setAnalyzeDisabled(false);
-    setProcessingFile(false);
-  };
+  }
+
+  function getFilterFn(timeRange: TimeRange) {
+    return ({ timestamp }: JSONLog) =>
+      !!(
+        (timeRange.min && timestamp < timeRange.min) ||
+        (timeRange.max && timestamp > timeRange.max)
+      );
+  }
 
   return {
     logDatas,
@@ -35,6 +66,7 @@ function useViewModel() {
     processingFile,
     handleAnalyzeClick,
     handleFileUpload,
+    setTimeRange,
   };
 }