From d0bd2eece9ce18c2c17c5bd0ea9aa113b5e5f6dd Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 11:25:03 -0700 Subject: [PATCH 1/8] Add image queue and archive paths to config --- README.md | 16 ++++++++++++++-- src/app.js | 4 ++-- src/config/index.js | 4 +++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bc0368b..903ae64 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,14 @@ $ cd animl-base $ npm install ``` +3. Create a directories for `~/images/queue/` and `~/images/archive/` + +```shell +$ mkdir /home/animl/images +$ mkdir /home/animl/images/queue +$ mkdir /home/animl/images/archive +``` + 3. Add a .env file to the project's root directory with the following items. Note, AWS creds can be found in the [TNC Cameratrap network passwords document](https://tnc.app.box.com/file/762650708780). For access, contact nathaniel.rindlaub@tnc.org: ``` @@ -301,8 +309,12 @@ AWS_SECRET_ACCESS_KEY = [secret access key] AWS_REGION = us-west-2 # Image directory to watch -IMG_DIR = '/home/animl/data//cameras/' -# IMG_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows +WATCH_DIR = '/home/animl/data//cameras/' +# WATCH_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows + +# Directories for queued and archived images +QUEUE_DIR = '/home/animl/images/queue' +ARCHIVE_DIR = '/home/animl/images/archive' # Log file to watch LOG_FILE = '/home/animl/data//log.txt' diff --git a/src/app.js b/src/app.js index 2d2103c..8a4a926 100644 --- a/src/app.js +++ b/src/app.js @@ -57,10 +57,10 @@ async function start() { await queue.init(); // Initialize directory watcher - const imgWatcher = chokidar.watch(config.imgDir, config.watcher); + const imgWatcher = chokidar.watch(config.watchDir, config.watcher); imgWatcher .on('ready', async () => { - console.log(`Watching for changes to ${config.imgDir}`); + console.log(`Watching for changes to ${config.watchDir}`); // NOTE: just for testing // const filesWatchedOnStart = imgWatcher.getWatched(); diff --git a/src/config/index.js b/src/config/index.js index 11ec7e3..a3ac1c0 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -6,7 +6,9 @@ dotenv.config(); module.exports = { platform: process.platform, baseName: process.env.BASE_NAME, - imgDir: process.env.IMG_DIR, + watchDir: process.env.WATCH_DIR, + queueDir: process.env.QUEUE_DIR, + archiveDir: process.env.ARCHIVE_DIR, dbFile: 'db.json', logFile: process.env.LOG_FILE, aws: { From 79d2600401ab26eba0d9939a3f7b992679a4101d Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 11:26:44 -0700 Subject: [PATCH 2/8] Wait for image to finish being written before triggering 'add' event --- src/config/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/index.js b/src/config/index.js index a3ac1c0..77cf169 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -19,6 +19,7 @@ module.exports = { ignored: /(^|[\/\\])\../, // ignore dotfiles ignoreInitial: true, // ignore files in the directory on start persistent: true, + awaitWriteFinish: true, }, supportedFileTypes: ['.jpg', '.png'], pollInterval: 5000, From 137c21c645c7adb8c62c868cc7eb8c4f72f1fe53 Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 11:45:55 -0700 Subject: [PATCH 3/8] Move images from watched dir to queue dir when added --- src/app.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/app.js b/src/app.js index 8a4a926..18595d3 100644 --- a/src/app.js +++ b/src/app.js @@ -1,4 +1,5 @@ const path = require('path'); +const fs = require('fs').promises; const chokidar = require('chokidar'); const Queue = require('./utils/queue'); const Worker = require('./utils/worker'); @@ -32,11 +33,21 @@ function validateFile(filePath) { return true; } -function handleNewFile(filePath, queue, metricsLogger) { +async function handleNewFile(filePath, queue, metricsLogger) { console.log(`New file detected: ${filePath}`); - if (validateFile(filePath)) { - queue.add(filePath); - metricsLogger.handleNewImage(filePath); + if (!validateFile(filePath)) return; + try { + // move file to queue directory + const destPath = path.join( + config.queueDir, + filePath.split('/').slice(4).join('/') + ); + await fs.rename(filePath, destPath); + console.log(`Moved file from ${source} to ${destPath}`); + await metricsLogger.handleNewImage(destPath); + queue.add(destPath); + } catch (err) { + console.log('Error handling new file: ', err); } } From 6cd7d71212d94de9d3bc063d90e43cb469ac7ba1 Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 12:05:52 -0700 Subject: [PATCH 4/8] Create queue dirs if they don't exist --- src/app.js | 4 ++++ src/utils/worker.js | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/app.js b/src/app.js index 18595d3..84692ec 100644 --- a/src/app.js +++ b/src/app.js @@ -42,6 +42,10 @@ async function handleNewFile(filePath, queue, metricsLogger) { config.queueDir, filePath.split('/').slice(4).join('/') ); + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } await fs.rename(filePath, destPath); console.log(`Moved file from ${source} to ${destPath}`); await metricsLogger.handleNewImage(destPath); diff --git a/src/utils/worker.js b/src/utils/worker.js index ffa4bc7..2dcf32d 100644 --- a/src/utils/worker.js +++ b/src/utils/worker.js @@ -36,6 +36,14 @@ class Worker { await this.queue.remove(img.path); await this.imgWatcher.unwatch(img.path); // TODO: test whether this works + // Just for testing... + const filesWatched = this.imgWatcher.getWatched(); + Object.keys(filesWatched).forEach((dir) => { + console.log( + `Number of files being watched in ${dir} : ${filesWatched[dir].length}` + ); + }); + // Reset backoff and poll again this.backoff.reset(); this.poll(); From 4c31a0d86530f805c8993fafdbb4ea0aae376854 Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 14:06:10 -0700 Subject: [PATCH 5/8] Use sync fs module --- src/app.js | 8 ++++---- src/utils/worker.js | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/app.js b/src/app.js index 84692ec..86bcc6b 100644 --- a/src/app.js +++ b/src/app.js @@ -1,5 +1,5 @@ const path = require('path'); -const fs = require('fs').promises; +const fs = require('fs'); const chokidar = require('chokidar'); const Queue = require('./utils/queue'); const Worker = require('./utils/worker'); @@ -40,14 +40,14 @@ async function handleNewFile(filePath, queue, metricsLogger) { // move file to queue directory const destPath = path.join( config.queueDir, - filePath.split('/').slice(4).join('/') + filePath.split('/').slice(6).join('/') ); const destDir = path.dirname(destPath); if (!fs.existsSync(destDir)) { fs.mkdirSync(destDir, { recursive: true }); } - await fs.rename(filePath, destPath); - console.log(`Moved file from ${source} to ${destPath}`); + fs.renameSync(filePath, destPath); + console.log(`Moved file from ${filePath} to ${destPath}`); await metricsLogger.handleNewImage(destPath); queue.add(destPath); } catch (err) { diff --git a/src/utils/worker.js b/src/utils/worker.js index 2dcf32d..5d2c4d3 100644 --- a/src/utils/worker.js +++ b/src/utils/worker.js @@ -34,7 +34,6 @@ class Worker { await this.s3.upload(img.path); console.log('Upload success'); await this.queue.remove(img.path); - await this.imgWatcher.unwatch(img.path); // TODO: test whether this works // Just for testing... const filesWatched = this.imgWatcher.getWatched(); From 2ddeb582661d26165882d2689859837bdebf5293 Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 14:19:47 -0700 Subject: [PATCH 6/8] Move file to archive dir after upload --- src/utils/worker.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/utils/worker.js b/src/utils/worker.js index 5d2c4d3..281900a 100644 --- a/src/utils/worker.js +++ b/src/utils/worker.js @@ -35,6 +35,15 @@ class Worker { console.log('Upload success'); await this.queue.remove(img.path); + // move file from queue directory to archive directory + console.log('Moving file to archive directory'); + const destPath = img.path.replace('queue', 'archive'); + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + fs.renameSync(img.path, destPath); + // Just for testing... const filesWatched = this.imgWatcher.getWatched(); Object.keys(filesWatched).forEach((dir) => { From 5608848fa4fe3383f854c2aa8d155025fc282152 Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 14:23:03 -0700 Subject: [PATCH 7/8] Require path and fs --- src/utils/worker.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/worker.js b/src/utils/worker.js index 281900a..4c9480d 100644 --- a/src/utils/worker.js +++ b/src/utils/worker.js @@ -1,3 +1,5 @@ +const path = require('path'); +const fs = require('fs'); const Backoff = require('backo'); const S3Service = require('./s3Service'); From 43f4fcc08b58f158cc30a982edfd1c7966aabc9e Mon Sep 17 00:00:00 2001 From: Nathaniel Rindlaub Date: Fri, 25 Oct 2024 14:41:46 -0700 Subject: [PATCH 8/8] Update README --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 903ae64..872cf51 100644 --- a/README.md +++ b/README.md @@ -426,9 +426,17 @@ $ mbasectl -i For adding new cameras, repeaters, and managing deployed devices, use the Multibase Server edition local web application, which can be found at `localhost:8888` from within the computer when Mulibase is running. You can remotely access it by remote-desktoping into the computer via AnyDesk/VCN and launchubg the local web app in a browser window if you're trying to manage the devices remotely. More detailed documentation on using the Buckeye MultiBase SE application can be found [here](https://tnc.app.box.com/file/794348600237?s=3x3e0onul82mxawahpo3qeffmzomm4uq). -> NOTE: If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and +> [!IMPORTANT] +> Because animl-base moves images out of the directory that Multibase SE expects them to be in (see [explaination below](https://github.com/tnc-ca-geo/animl-base?tab=local-image-file-storage-and-archive) for more detail), it will appear in the Multibase SE webapp as though there the network has never recieved any images. We reccommend using https://animl.camera for all image review, but if you need to access the image files locally, a backup of the most recent images can be found at `~/images/archive/`. + +> [!TIP] +> If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and > clicking the "admin" button under the base entry), try "restoring the network" (hamburger menu -> Restore Network). This will search for and locate any devices that were have already been registered to the base. +### Local image file storage and archive + +Typically, Buckeye's MultiBase SE program stores all image files in the `~/data/data//cameras/` directory. However, largely due to [memory issues](https://github.com/tnc-ca-geo/animl-base/issues/20) with watching an ever-growing directory of images, once animl-base detects a new image file it moves it to a "queue" directory at `~/images/queue/`, and after it's been successfully updated it gets moved to `~/images/archive/` to serve as a local backup of the image files. + ### Sixfab power managment web app (Raspberry Pi only) For remotely monitoring the RPi's status and configuring power, connectivity, and temperature alerts, you can access [https://power.sixfab.com](https://power.sixfab.com) from any computer. Credentials are in the Camera trap network [password document](https://tnc.app.box.com/file/762650708780).