Process Samba spool ASAP, in order to prevent loosing the spooled tas…

…ks (stored in tmpfs) on reboot

Fixes #224

greyhole.cron.d

@@ -1,5 +1,6 @@
 # cifs client workaround
 # Ref: http://blog.dhampir.no/content/cifs-vfs-no-response-for-cmd-n-mid
-@reboot  	root	/sbin/modprobe cifs enable_oplocks=n; if [ -x /proc/fs/cifs/OplockEnabled ]; then echo 0 > /proc/fs/cifs/OplockEnabled; fi
-@reboot  	root	/usr/bin/greyhole --create-mem-spool >/dev/null
-@reboot  	root	/usr/bin/greyhole --boot-init
+@reboot    root  /sbin/modprobe cifs enable_oplocks=n; if [ -x /proc/fs/cifs/OplockEnabled ]; then echo 0 > /proc/fs/cifs/OplockEnabled; fi
+@reboot    root  /usr/bin/greyhole --create-mem-spool >/dev/null
+@reboot    root  /usr/bin/greyhole --boot-init
+* * * * *  root  /usr/bin/greyhole --process-spool --keepalive

greyhole.systemd

@@ -8,6 +8,7 @@ Type=simple
 ExecStartPre=/usr/bin/greyhole --test-config
 ExecStart=/usr/bin/greyhole --daemon
 ExecReload=/usr/bin/kill -HUP $MAINPID
+ExecStopPost=/usr/bin/greyhole --process-spool
 Restart=always
 RestartSec=5
 

includes/CLI/CommandLineHelper.php

@@ -40,6 +40,7 @@
 require_once('includes/CLI/LogsCliRunner.php');
 require_once('includes/CLI/MD5WorkerCliRunner.php');
 require_once('includes/CLI/PauseCliRunner.php');
+require_once('includes/CLI/ProcessSpoolCliRunner.php');
 require_once('includes/CLI/RemoveShareCliRunner.php');
 require_once('includes/CLI/ResumeCliRunner.php');
 require_once('includes/CLI/ReplaceCliRunner.php');
@@ -89,6 +90,7 @@ function __construct() {
             new CliCommandDefinition('create-mem-spool', '',    null,          'CreateMemSpoolRunner',    null),
             new CliCommandDefinition('test-config',      '',    null,          'TestCliRunner',           null),
             new CliCommandDefinition('boot-init',        '',    null,          'BootInitCliRunner',       null),
+            new CliCommandDefinition('process-spool',    '',    null,          'ProcessSpoolCliRunner',   null),
         );
 
         $this->cliOptionsDefinitions = array(

includes/CLI/ProcessSpoolCliRunner.php

@@ -0,0 +1,40 @@
+<?php
+/*
+Copyright 2020 Guillaume Boudreau
+
+This file is part of Greyhole.
+
+Greyhole is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Greyhole is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Greyhole.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+require_once('includes/CLI/AbstractCliRunner.php');
+
+class ProcessSpoolCliRunner extends AbstractCliRunner {
+    public function run() {
+        global $argv;
+        $start = time();
+        while (time() - $start < 55) {
+            SambaSpool::parse_samba_spool();
+            if (!array_contains($argv, '--keepalive')) {
+                return;
+            }
+            if (time() - $start >= 55) {
+                break;
+            }
+            sleep(5);
+        }
+    }
+}
+
+?>

includes/DB.php

@@ -142,6 +142,27 @@ public static function lastInsertedId() {
         return $lastInsertedId;
     }
 
+    public static function acquireLock($name, $timeout = NULL) {
+        $locked = static::getFirstValue("SELECT GET_LOCK(:name, :timeout)", ['name' => $name, 'timeout' => $timeout]);
+        if ($locked) {
+            return TRUE;
+        }
+        return FALSE;
+    }
+
+    public static function releaseLock($name) {
+        $released = static::getFirstValue("SELECT RELEASE_LOCK(:name)", ['name' => $name]);
+        if (!$released && static::isLocked($name)) {
+            return FALSE;
+        }
+        return TRUE;
+    }
+
+    public static function isLocked($name) {
+        $is_lock_free = static::getFirstValue("SELECT IS_FREE_LOCK(:name)", $name);
+        return !$is_lock_free;
+    }
+
     public static function error() {
         return self::$options->error;
     }

includes/SambaSpool.php

@@ -48,18 +48,8 @@ public static function parse_samba_spool() {
             chmod('/var/spool/greyhole/mem', 0777);
         }
 
-        // If we have enough queued tasks (90% of $max_queued_tasks), let's not parse the log at this time, and get some work done.
-        // Once we fall below that, we'll queue up to at most $max_queued_tasks new tasks, then get back to work.
-        // This will effectively 'batch' large file operations to make sure the DB doesn't become a problem because of the number of rows,
-        //   and this will allow the end-user to see real activity, other that new rows in greyhole.tasks...
-        $num_rows = DBSpool::get_num_tasks();
-        $max_queued_tasks = Config::get(CONFIG_MAX_QUEUED_TASKS);
-        $queued_tasks_limit = round($max_queued_tasks * 0.9);
-        if ($num_rows >= $queued_tasks_limit) {
-            Log::restorePreviousAction();
-            if (time() % 10 == 0) {
-                Log::debug("  More than $queued_tasks_limit tasks queued... Won't queue any more at this time.");
-            }
+        if (!DB::acquireLock(ACTION_READ_SAMBA_POOL, 5)) {
+            // Another thread is already processing the Samba Spool; we don't want multiple threads doing this in parallel!
             return;
         }
 
@@ -70,8 +60,7 @@ public static function parse_samba_spool() {
         while (TRUE) {
             $files = array();
             $last_filename = FALSE;
-            $space_left_in_queue = $max_queued_tasks - $num_rows - $new_tasks;
-            exec('find -L /var/spool/greyhole -type f -printf "%T@ %p\n" | sort -n 2> /dev/null | head -' . $space_left_in_queue, $files);
+            exec('find -L /var/spool/greyhole -type f -printf "%T@ %p\n" | sort -n 2> /dev/null | head -n 10000', $files);
             if (count($files) == 0) {
                 break;
             }
@@ -170,19 +159,10 @@ public static function parse_samba_spool() {
                     /** @noinspection PhpUndefinedVariableInspection */
                     $db_spool->insert($act, $share, @$fullpath, @$fullpath_target, @$fd);
                 }
-
-                // If we have enough queued tasks ($max_queued_tasks), let's stop parsing the log, and get some work done.
-                if ($num_rows+$new_tasks >= $max_queued_tasks) {
-                    Log::debug("  We now have more than $max_queued_tasks tasks queued... Will stop parsing for now.");
-                    break;
-                }
             }
             if ($last_filename) {
                 unlink($last_filename);
             }
-            if ($num_rows+$new_tasks >= $max_queued_tasks) {
-                break;
-            }
         }
 
         // Close & fwrite logs are only processed when no more duplicates are found, so we'll execute this now that we're done parsing all spooled files.
@@ -197,6 +177,8 @@ public static function parse_samba_spool() {
             Log::debug("Found $new_tasks new tasks in spool.");
         }
 
+        DB::releaseLock(ACTION_READ_SAMBA_POOL);
+
         Log::restorePreviousAction();
     }
 

initd_script.sh

@@ -120,6 +120,7 @@ stop () {
 		echo
 	fi
 	[ $RETVAL -eq 0 ] && rm -f $PIDFILE
+	/usr/bin/greyhole --process-spool >/dev/null
 	return $ret
 }
 

upstart.conf

@@ -24,4 +24,5 @@ end script
 
 post-stop script
    rm -f /var/run/greyhole.pid
+   /usr/bin/greyhole --process-spool >/dev/null
 end script