Merge pull request #2 from AceCentre/create-overlay-to-measure

Create overlay to measure
AceCentre · Nov 5, 2024 · d3dbfe4 · d3dbfe4
2 parents e3e8990 + 82d73a7
commit d3dbfe4
Show file tree

Hide file tree

Showing 7 changed files with 785 additions and 58 deletions.
diff --git a/nodejs/sender-monitor/main.js b/nodejs/sender-monitor/main.js
@@ -1,11 +1,12 @@
-const { app, Tray, Menu, shell, clipboard, BrowserWindow, ipcMain } = require("electron");
+const { app, Tray, Menu, shell, clipboard, BrowserWindow, ipcMain, screen} = require("electron");
 const path = require("path");
 const fs = require("fs");
 const webrtc = require("./webrtc");
 const QRCode = require("qrcode");
 const { Monitor } = require("node-screenshots");
 const { performOCR } = require("./ocr");
 const { Translator } = require("google-translate-api-x");
+const sharp = require("sharp");
 
 let tray;
 let qrWindow;
@@ -18,6 +19,7 @@ let clipboardMonitorInterval;
 let ocrMonitorInterval;
 const maxRetries = 10;
 const retryInterval = 3000;
+let overlayWindow = null; 
 
 const logFilePath = path.join(app.getPath("userData"), "log.txt");
 
@@ -38,6 +40,45 @@ ipcMain.on("close-qr-window", () => {
     }
 });
 
+function createOverlayWindow() {
+  overlayWindow = new BrowserWindow({
+    fullscreen: true,
+    frame: false,
+    transparent: true,
+    alwaysOnTop: true,
+    resizable: false,
+    webPreferences: {
+      contextIsolation: true,
+      preload: path.join(__dirname, "overlay-preload.js"),
+    },
+  });
+
+  overlayWindow.loadURL(`file://${__dirname}/overlay.html`);
+  overlayWindow.on("closed", () => {
+    overlayWindow = null;
+    console.log("Overlay window closed");
+  });
+}
+
+// Updated `set-capture-area` handler
+ipcMain.on("set-capture-area", (event, bounds) => {
+  console.log("Received capture area bounds from overlay:", bounds);
+  if (!bounds || !bounds.width || !bounds.height) {
+    console.error("Invalid capture area bounds received:", bounds);
+    return;
+  }
+
+  config.captureArea = bounds;
+  fs.writeFileSync(configFilePath, JSON.stringify(config, null, 2), "utf-8");
+  console.log(`Updated capture area to new values: ${JSON.stringify(bounds)}`);
+
+  // Close the overlay window after capturing bounds
+  if (overlayWindow) {
+    overlayWindow.close();
+  }
+});
+
+
 async function createQRWindow(qrDataUrl) {
     if (qrWindow) {
         qrWindow.close();
@@ -82,7 +123,39 @@ function reloadConfig() {
         config = JSON.parse(fs.readFileSync(configFilePath, "utf-8"));
         console.log("Configuration reloaded.");
         logMessage("Configuration reloaded from config.json.");
-        // You may want to reinitialize any settings here that depend on the config
+
+        // Stop any existing monitoring intervals
+        if (clipboardMonitorInterval) {
+            clearInterval(clipboardMonitorInterval);
+            clipboardMonitorInterval = null;
+        }
+        if (ocrMonitorInterval) {
+            clearInterval(ocrMonitorInterval);
+            ocrMonitorInterval = null;
+        }
+
+        // Start the appropriate monitoring based on the reloaded config
+        if (config.monitorMode === "clipboard") {
+            clipboardMonitorInterval = setInterval(() => {
+                const currentText = clipboard.readText();
+                processAndSendText(currentText);
+            }, config.captureInterval);
+            console.log("Switched to clipboard monitoring mode.");
+            logMessage("Switched to clipboard monitoring mode.");
+        } else if (config.monitorMode === "ocr") {
+            ocrMonitorInterval = setInterval(async () => {
+                const recognizedText = await captureAndProcessScreen();
+                processAndSendText(recognizedText);
+            }, config.captureInterval);
+            console.log("Switched to OCR monitoring mode.");
+            logMessage("Switched to OCR monitoring mode.");
+        }
+
+        // Update translation settings based on the reloaded config
+        const isTranslationEnabled = config.translation?.enabled || false;
+        console.log(`Translation enabled: ${isTranslationEnabled}`);
+        logMessage(`Translation enabled: ${isTranslationEnabled}`);
+
     } catch (error) {
         console.error("Failed to reload config:", error);
         logMessage("Failed to reload config.");
@@ -91,65 +164,105 @@ function reloadConfig() {
 
 // Capture and OCR function with return of recognized text
 async function captureAndProcessScreen() {
-    const { x, y, width, height } = config.captureArea;
-    const useEdgeForOCR = config.useEdgeForOCR;
+  const { x, y, width, height } = config.captureArea;
+  const useEdgeForOCR = config.useEdgeForOCR;
+  const fullImagePath = path.join(app.getPath("temp"), "full-image.png");
+  const croppedImagePath = path.join(app.getPath("temp"), "cropped-image.png");
+
+  try {
+    console.log("Starting screen capture and cropping process...");
+
+    const monitor = Monitor.all().find(m => m.isPrimary);
+    if (!monitor) {
+      console.error("No primary monitor found.");
+      return;
+    }
 
-    const filePath = path.join(app.getPath("temp"), "ocr-capture.png");
+    const fullImage = await monitor.captureImage();
+    if (!fullImage) {
+      console.error("Image capture failed.");
+      return;
+    }
 
-    try {
-        const monitor = Monitor.all().find(m => m.isPrimary);
-        if (!monitor) {
-            console.error("No primary monitor found.");
-            return "";
-        }
+    // Convert the full captured image to a PNG buffer
+    const fullImageBuffer = await fullImage.toPng().catch(err => {
+        console.error("Failed to create fullImageBuffer:", err);
+        throw err; 
+    });
 
-        // Capture the full image
-        const fullImage = await monitor.captureImage();
-        if (!fullImage) {
-            console.error("Image capture failed.");
-            return "";
-        }
-
-        // Crop the image to the specified capture area
-        const croppedImage = await fullImage.crop(x, y, width, height);
-        if (!croppedImage) {
-            console.error("Image cropping failed.");
-            return "";
-        }
+    // Save the full image
+    await fs.promises.writeFile(fullImagePath, fullImageBuffer);
+    console.log("Full screenshot saved at", fullImagePath);
 
-        // Write the cropped image to the temp path
-        const imageBuffer = await croppedImage.toPng();
-        fs.writeFileSync(filePath, imageBuffer);
+    if (!fs.existsSync(fullImagePath)) {
+      console.error("Full screenshot file does not exist at", fullImagePath);
+      return;
+    }
 
-        // Confirm file existence
-        if (!fs.existsSync(filePath)) {
-            console.error("File does not exist at", filePath);
-            return ""; // Exit early if file creation failed
-        }
+    // Get image dimensions using sharp to verify metadata
+    const imageMetadata = await sharp(fullImagePath).metadata();
+    console.log("Full image dimensions:", imageMetadata);
 
-        console.log("Image successfully saved at", filePath);
+    // Get the primary display's scaling factor
+    const primaryDisplay = screen.getPrimaryDisplay();
+    const scaleFactor = primaryDisplay.scaleFactor;
+    console.log("Primary display scale factor:", scaleFactor);
 
-        // Call performOCR with the useEdgeForOCR flag
-        const recognizedText = await performOCR(filePath, useEdgeForOCR);
+    // Adjust bounds to account for scaling
+    const scaledX = Math.round(x * scaleFactor);
+    const scaledY = Math.round(y * scaleFactor);
+    const scaledWidth = Math.round(width * scaleFactor);
+    const scaledHeight = Math.round(height * scaleFactor);
 
-        // Log recognized text
-        console.log("OCR completed, recognized text:", recognizedText);
+    console.log(`Scaled crop bounds: x=${scaledX}, y=${scaledY}, width=${scaledWidth}, height=${scaledHeight}`);
 
-        return recognizedText || ""; // Return the OCR text or an empty string
+    // Adjust bounds to ensure they are within the image dimensions
+    const adjustedX = Math.min(Math.max(scaledX, 0), imageMetadata.width);
+    const adjustedY = Math.min(Math.max(scaledY, 0), imageMetadata.height);
+    const adjustedWidth = Math.min(scaledWidth, imageMetadata.width - adjustedX);
+    const adjustedHeight = Math.min(scaledHeight, imageMetadata.height - adjustedY);
 
-    } catch (error) {
-        console.error("Screen capture or OCR failed:", error);
-        logMessage("Error during screen capture or OCR");
-        return "";
-    } finally {
-        // Clean up the temp file after everything is done
-        if (fs.existsSync(filePath)) {
-            fs.unlinkSync(filePath);
-            console.log("Temporary OCR file cleaned up:", filePath);
-        }
+    console.log(`Adjusted crop bounds for the image: x=${adjustedX}, y=${adjustedY}, width=${adjustedWidth}, height=${adjustedHeight}`);
+
+    // Now proceed to crop the image using sharp with the adjusted dimensions
+    await sharp(fullImagePath)
+      .extract({ left: adjustedX, top: adjustedY, width: adjustedWidth, height: adjustedHeight })
+      .toFile(croppedImagePath);
+
+    console.log("Cropped image successfully saved at", croppedImagePath);
+
+    if (!fs.existsSync(croppedImagePath)) {
+      console.error("Cropped image file does not exist at", croppedImagePath);
+      return;
+    }
+
+    const recognizedText = await performOCR(croppedImagePath, useEdgeForOCR);
+
+    if (recognizedText === undefined) {
+      console.log("OCR result was undefined. Opening image directory for inspection...");
+    } else {
+      console.log("OCR completed, recognized text:", recognizedText);
     }
+
+    return recognizedText || "";
+
+  } catch (error) {
+    console.error("Error occurred during capture or cropping process:", error);
+  } finally {
+    // Clean up temporary files if they exist
+    if (fs.existsSync(croppedImagePath)) {
+      fs.unlinkSync(croppedImagePath);
+      console.log("Temporary OCR file cleaned up:", croppedImagePath);
+    }
+
+    if (fs.existsSync(fullImagePath)) {
+      fs.unlinkSync(fullImagePath);
+      console.log("Temporary OCR file cleaned up of screen:", fullImagePath);
+    }
+  }
 }
 
+
 const translator = new Translator({
     from: config.translation?.sourceLang || "en",
     to: config.translation?.targetLang || "es",
@@ -267,6 +380,10 @@ function updateTrayMenu() {
             click: () => shell.openPath(configFilePath)
                 .catch(err => console.error("Failed to open config file:", err))
         },
+        {
+            label: "Define OCR Area", // New option to define OCR area
+            click: createOverlayWindow,
+        },
         {
             label: "Reload Config",
             click: reloadConfig,

diff --git a/nodejs/sender-monitor/ocr.js b/nodejs/sender-monitor/ocr.js
@@ -20,18 +20,30 @@ const ocrImageEdge = edge.func({
         {
             public async Task<object> Invoke(dynamic input)
             {
-                string filePath = (string)input;
-                var storageFile = await StorageFile.GetFileFromPathAsync(filePath);
-
-                using (IRandomAccessStream stream = await storageFile.OpenAsync(FileAccessMode.Read))
+                try
                 {
-                    var decoder = await BitmapDecoder.CreateAsync(stream);
-                    var bitmap = await decoder.GetSoftwareBitmapAsync();
+                    string filePath = (string)input;
+                    var storageFile = await StorageFile.GetFileFromPathAsync(filePath);
+
+                    using (IRandomAccessStream stream = await storageFile.OpenAsync(FileAccessMode.Read))
+                    {
+                        var decoder = await BitmapDecoder.CreateAsync(stream);
+                        var bitmap = await decoder.GetSoftwareBitmapAsync();
+
+                        if (bitmap == null) return "Bitmap loading failed";
+
+                        var ocrEngine = OcrEngine.TryCreateFromUserProfileLanguages();
+                        if (ocrEngine == null) return "OCR Engine creation failed";
 
-                    var ocrEngine = OcrEngine.TryCreateFromUserProfileLanguages();
-                    var ocrResult = await ocrEngine.RecognizeAsync(bitmap);
+                        var ocrResult = await ocrEngine.RecognizeAsync(bitmap);
+                        if (ocrResult == null || ocrResult.Text == null) return "OCR result is empty or null";
 
-                    return ocrResult.Text;
+                        return ocrResult.Text;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    return "Error: " + ex.Message;
                 }
             }
         }
@@ -47,11 +59,13 @@ const ocrImageEdge = edge.func({
 
 // Main OCR function that switches based on config
 async function performOCR(filePath, useEdgeForOCR) {
+    console.log("performOCR called with useEdgeForOCR:", useEdgeForOCR);
+
     if (useEdgeForOCR) {
         try {
             const text = await ocrImageEdge(filePath);
             console.log("Recognized text (Windows.Media.Ocr):", text);
-            return text;
+            return text || ""; // Return text or an empty string if text is undefined
         } catch (error) {
             console.error("Windows.Media.Ocr failed:", error);
             return null;

diff --git a/nodejs/sender-monitor/overlay-preload.js b/nodejs/sender-monitor/overlay-preload.js
@@ -0,0 +1,42 @@
+const { ipcRenderer } = require('electron');
+
+let isSelecting = false;
+let startX, startY, endX, endY;
+
+// This listens for the mouse events to capture the coordinates
+window.addEventListener('mousedown', (event) => {
+  isSelecting = true;
+  startX = event.clientX;
+  startY = event.clientY;
+  console.log('Selection started at:', { x: startX, y: startY });
+});
+
+window.addEventListener('mousemove', (event) => {
+  if (isSelecting) {
+    endX = event.clientX;
+    endY = event.clientY;
+    console.log('Selection moving:', { endX, endY });
+  }
+});
+
+window.addEventListener('mouseup', (event) => {
+  if (isSelecting) {
+    isSelecting = false;
+    endX = event.clientX;
+    endY = event.clientY;
+    console.log('Selection ended at:', { x: endX, y: endY });
+
+    // Calculate the bounds
+    const bounds = {
+      x: Math.min(startX, endX),
+      y: Math.min(startY, endY),
+      width: Math.abs(endX - startX),
+      height: Math.abs(endY - startY)
+    };
+
+    console.log('Bounds calculated:', bounds);
+
+    // Send bounds to the main process
+    ipcRenderer.send('set-capture-area', bounds);
+  }
+});