Assetbatcher can now be run on multiple cores by adding the `-jobs [n…

…um]` and `-processors [count]` args. Recommended is to set processors to the amount of cores, and jobs to about twice that.

toolkit/assetbatcher/assetbatcherapp.cc

@@ -17,6 +17,7 @@
 #include "flat/physics/material.h"
 #include "jobs2/jobs2.h"
 #include "system/nebulasettings.h"
+#include "toolkit/toolkit-common/text.h"
 
 #ifdef WIN32
 #include "io/win32/win32consolehandler.h"
@@ -111,6 +112,31 @@ AssetBatcherApp::Close()
     DistributedToolkitApp::Close();
 }
 
+//------------------------------------------------------------------------------
+/**
+*/
+void
+AssetBatcherApp::Run()
+{
+    Timing::Timer totalTime;
+    totalTime.Start();
+
+    DistributedTools::DistributedToolkitApp::Run();
+
+    totalTime.Stop();
+    if (this->IsMaster() || !this->listfileArg.IsValid())
+    {
+        this->logger.Print("\n");
+        this->logger.Print("--- Batching took ");
+        int minutes = Math::floor(totalTime.GetTime() / 60.0f);
+        float seconds = Math::fmod(totalTime.GetTime(), 60.0f);
+        Util::String time = Util::String::FromInt(minutes) + "m " + Util::String::FromFloat(seconds) + "s\n";
+        this->logger.Print(
+            ToolkitUtil::Text::Text(time).Color(ToolkitUtil::TextColor::Green).Style(ToolkitUtil::FontMode::Bold).AsCharPtr()
+        );
+    }
+}
+
 //------------------------------------------------------------------------------
 /**
 */
@@ -225,6 +251,9 @@ AssetBatcherApp::DoWork()
     if (this->listfileArg.IsValid())
     {
         Array<String> fileList = CreateFileList();
+        IO::AssignRegistry::Instance()->SetAssign(IO::Assign("src", "proj:work"));
+        exporter->UpdateSource();
+        exporter->SetProgressMinMax(0, fileList.Size() * PRECISION);
         exporter->ExportList(fileList);
     }
     else
@@ -316,14 +345,17 @@ void
 AssetBatcherApp::ShowHelp()
 {
     n_printf("Nebula asset batcher.\n"
-        "(C) 2012-2020 Individual contributors, see AUTHORS file.\n");
-    n_printf("-help         --display this help\n"
-             "-force        --ignores time stamps\n"
-             "-asset        --asset name, implies source argument\n"
-             "-source       --select asset source from projectinfo, default all\n"
-             "-work         --batch a non-registered work folder into the project\n"
-             "-mode         --batch only a type of resource, can be: fbx, model, surface, texture, physics, gltf, audio\n"
-             "-project      --projectinfo override\n");
+        "(C) 2012-2024 Individual contributors, see AUTHORS file.\n");
+    n_printf("-force       -- ignores time stamps\n"
+             "-asset       -- asset name, implies source argument\n"
+             "-source      -- select asset source from projectinfo, default all\n"
+             "-work        -- batch a non-registered work folder into the project\n"
+             "-mode        -- batch only a type of resource, can be: fbx, model, surface, texture, physics, gltf, audio\n"
+             "-project     -- projectinfo override\n"
+    );
+
+    String additionalHelp = GetArgumentDescriptionString();
+    n_printf(additionalHelp.AsCharPtr());
 }
 
 //------------------------------------------------------------------------------
@@ -367,7 +399,14 @@ AssetBatcherApp::CreateFileList()
         for (int directoryIndex = 0; directoryIndex < directories.Size(); directoryIndex++)
         {
             String category = workDir + "/" + directories[directoryIndex];
-            res.Append(category);
+
+            Array<String> filesInFolder = IoServer::Instance()->ListFiles(category, "*");
+            filesInFolder.AppendArray(IoServer::Instance()->ListDirectories(category, "*"));
+            for (auto& f : filesInFolder)
+            {
+                f = category + "/" + f;
+                res.Append(f);
+            }
         }
 
         // update progressbar in batchexporter

toolkit/assetbatcher/assetbatcherapp.h

@@ -32,6 +32,8 @@ class AssetBatcherApp : public DistributedTools::DistributedToolkitApp
     /// close the application
     void Close();
     /// runs the application
+    void Run() override;
+    /// runs the application
     void DoWork();
 protected:
     /// creates file list for job-driven exporting

toolkit/toolkit-common/distributedtools/distributedjobs/distributedappjob.cc

@@ -61,7 +61,7 @@ DistributedAppJob::RunJob()
     for (fileIndex = 0; fileIndex < this->fileList.Size(); fileIndex++)
     {
         writer->WriteString(this->fileList[fileIndex]);   
-        writer->WriteString("\r\n");
+        writer->WriteString("\n");
     }
     IndexT dirIndex;
     for (dirIndex = 0; dirIndex < this->dirList.Size(); dirIndex++)
@@ -72,7 +72,7 @@ DistributedAppJob::RunJob()
             String path;
             path.Format("%s/%s", this->dirList[dirIndex].AsCharPtr(), dirFiles[fileIndex].AsCharPtr());
             writer->WriteString(path);
-            writer->WriteString("\r\n");
+            writer->WriteString("\n");
         }
     }
 

toolkit/toolkit-common/distributedtools/distributedtoolkitapp.cc

@@ -455,7 +455,7 @@ DistributedToolkitApp::CreateAndProcessJobs()
         argString.Append(this->GetAdditionalArguments());
 
         job->SetArguments(argString);
-        job->SetSVNPath(this->projectInfo.GetPathAttr("SVNToolPath"));
+        //job->SetSVNPath(this->projectInfo.GetPathAttr("SVNToolPath"));
         job->SetProjectDirectory(this->projectdirArg);
        // job->SetRequiredRevision(revision);
         job->SetIdentifier(String::FromInt(jobIndex));

toolkit/toolkit-common/distributedtools/distributedtoolkitapp.h

@@ -53,6 +53,8 @@ class DistributedToolkitApp : public ToolkitUtil::ToolkitApp
     bool IsDistributed();
     /// determine if application is running in slave mode
     bool IsSlave();
+    /// determine if application is running in master mode
+    bool IsMaster();
 
 protected:
     /// split task of this application into jobs and process them
@@ -128,6 +130,16 @@ DistributedToolkitApp::IsSlave()
     return this->slaveArg;
 }
 
+//------------------------------------------------------------------------------
+/**
+    determine if application is in master mode   
+*/
+inline bool
+DistributedToolkitApp::IsMaster()
+{
+    return this->masterMode;
+}
+
 //------------------------------------------------------------------------------
 /**
     determine if application is distributed