Merge pull request #41 from tnc-ca-geo/memory-issue-fix

Memory issue fix

README.md

@@ -289,6 +289,14 @@ $ cd animl-base
 $ npm install
 ```
 
+3. Create a directories for `~/images/queue/` and `~/images/archive/`
+
+```shell
+$ mkdir /home/animl/images
+$ mkdir /home/animl/images/queue
+$ mkdir /home/animl/images/archive
+```
+
 3. Add a .env file to the project's root directory with the following items. Note, AWS creds can be found in the [TNC Cameratrap network passwords document](https://tnc.app.box.com/file/762650708780). For access, contact [email protected]:
 
 ```
@@ -301,8 +309,12 @@ AWS_SECRET_ACCESS_KEY = [secret access key]
 AWS_REGION = us-west-2
 
 # Image directory to watch
-IMG_DIR = '/home/animl/data/<base name>/cameras/'
-# IMG_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows
+WATCH_DIR = '/home/animl/data/<base name>/cameras/'
+# WATCH_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows
+
+# Directories for queued and archived images
+QUEUE_DIR = '/home/animl/images/queue'
+ARCHIVE_DIR = '/home/animl/images/archive'
 
 # Log file to watch
 LOG_FILE = '/home/animl/data/<base name>/log.txt'
@@ -414,9 +426,17 @@ $ mbasectl -i
 
 For adding new cameras, repeaters, and managing deployed devices, use the Multibase Server edition local web application, which can be found at `localhost:8888` from within the computer when Mulibase is running. You can remotely access it by remote-desktoping into the computer via AnyDesk/VCN and launchubg the local web app in a browser window if you're trying to manage the devices remotely. More detailed documentation on using the Buckeye MultiBase SE application can be found [here](https://tnc.app.box.com/file/794348600237?s=3x3e0onul82mxawahpo3qeffmzomm4uq).
 
-> NOTE: If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and
+> [!IMPORTANT]  
+> Because animl-base moves images out of the directory that Multibase SE expects them to be in (see [explaination below](https://github.com/tnc-ca-geo/animl-base?tab=local-image-file-storage-and-archive) for more detail), it will appear in the Multibase SE webapp as though there the network has never recieved any images. We reccommend using https://animl.camera for all image review, but if you need to access the image files locally, a backup of the most recent images can be found at `~/images/archive/`.
+
+> [!TIP]
+> If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and
 > clicking the "admin" button under the base entry), try "restoring the network" (hamburger menu -> Restore Network). This will search for and locate any devices that were have already been registered to the base.
 
+### Local image file storage and archive
+
+Typically, Buckeye's MultiBase SE program stores all image files in the `~/data/data/<base name>/cameras/` directory. However, largely due to [memory issues](https://github.com/tnc-ca-geo/animl-base/issues/20) with watching an ever-growing directory of images, once animl-base detects a new image file it moves it to a "queue" directory at `~/images/queue/`, and after it's been successfully updated it gets moved to `~/images/archive/` to serve as a local backup of the image files.
+
 ### Sixfab power managment web app (Raspberry Pi only)
 
 For remotely monitoring the RPi's status and configuring power, connectivity, and temperature alerts, you can access [https://power.sixfab.com](https://power.sixfab.com) from any computer. Credentials are in the Camera trap network [password document](https://tnc.app.box.com/file/762650708780).

src/app.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const fs = require('fs');
 const chokidar = require('chokidar');
 const Queue = require('./utils/queue');
 const Worker = require('./utils/worker');
@@ -32,11 +33,25 @@ function validateFile(filePath) {
   return true;
 }
 
-function handleNewFile(filePath, queue, metricsLogger) {
+async function handleNewFile(filePath, queue, metricsLogger) {
   console.log(`New file detected: ${filePath}`);
-  if (validateFile(filePath)) {
-    queue.add(filePath);
-    metricsLogger.handleNewImage(filePath);
+  if (!validateFile(filePath)) return;
+  try {
+    // move file to queue directory
+    const destPath = path.join(
+      config.queueDir,
+      filePath.split('/').slice(6).join('/')
+    );
+    const destDir = path.dirname(destPath);
+    if (!fs.existsSync(destDir)) {
+      fs.mkdirSync(destDir, { recursive: true });
+    }
+    fs.renameSync(filePath, destPath);
+    console.log(`Moved file from ${filePath} to ${destPath}`);
+    await metricsLogger.handleNewImage(destPath);
+    queue.add(destPath);
+  } catch (err) {
+    console.log('Error handling new file: ', err);
   }
 }
 
@@ -57,10 +72,10 @@ async function start() {
   await queue.init();
 
   // Initialize directory watcher
-  const imgWatcher = chokidar.watch(config.imgDir, config.watcher);
+  const imgWatcher = chokidar.watch(config.watchDir, config.watcher);
   imgWatcher
     .on('ready', async () => {
-      console.log(`Watching for changes to ${config.imgDir}`);
+      console.log(`Watching for changes to ${config.watchDir}`);
 
       // NOTE: just for testing
       // const filesWatchedOnStart = imgWatcher.getWatched();

src/config/index.js

@@ -6,7 +6,9 @@ dotenv.config();
 module.exports = {
   platform: process.platform,
   baseName: process.env.BASE_NAME,
-  imgDir: process.env.IMG_DIR,
+  watchDir: process.env.WATCH_DIR,
+  queueDir: process.env.QUEUE_DIR,
+  archiveDir: process.env.ARCHIVE_DIR,
   dbFile: 'db.json',
   logFile: process.env.LOG_FILE,
   aws: {
@@ -17,6 +19,7 @@ module.exports = {
     ignored: /(^|[\/\\])\../, // ignore dotfiles
     ignoreInitial: true, // ignore files in the directory on start
     persistent: true,
+    awaitWriteFinish: true,
   },
   supportedFileTypes: ['.jpg', '.png'],
   pollInterval: 5000,

src/utils/worker.js

@@ -1,3 +1,5 @@
+const path = require('path');
+const fs = require('fs');
 const Backoff = require('backo');
 const S3Service = require('./s3Service');
 
@@ -34,7 +36,23 @@ class Worker {
       await this.s3.upload(img.path);
       console.log('Upload success');
       await this.queue.remove(img.path);
-      await this.imgWatcher.unwatch(img.path); // TODO: test whether this works
+
+      // move file from queue directory to archive directory
+      console.log('Moving file to archive directory');
+      const destPath = img.path.replace('queue', 'archive');
+      const destDir = path.dirname(destPath);
+      if (!fs.existsSync(destDir)) {
+        fs.mkdirSync(destDir, { recursive: true });
+      }
+      fs.renameSync(img.path, destPath);
+
+      // Just for testing...
+      const filesWatched = this.imgWatcher.getWatched();
+      Object.keys(filesWatched).forEach((dir) => {
+        console.log(
+          `Number of files being watched in ${dir} : ${filesWatched[dir].length}`
+        );
+      });
 
       // Reset backoff and poll again
       this.backoff.reset();