Schema based configuration

.gitignore

@@ -53,3 +53,6 @@ calgrind.*
 *.hdf
 wct-deps.dot
 
+
+.direnv
+.envrc

apps/inc/WireCellApps/Main.h

@@ -91,11 +91,18 @@ namespace WireCell {
         /// Call to finalize any terminal components.
         void finalize();
 
+        /// Return the compiled version string
+        std::string version() const;
+
        private:
         ConfigManager m_cfgmgr;
         std::vector<std::string> m_plugins, m_apps, m_cfgfiles, m_load_path;
         Persist::externalvars_t m_extvars, m_extcode, m_tlavars, m_tlacode;
         Log::logptr_t l;
+
+        // Limit number of threads.  0 means set no limit.  This is
+        // only relevant if we are built with TBB support
+        int m_threads{0};
     };
 
 }  // namespace WireCell

apps/src/Main.cxx

@@ -3,13 +3,15 @@
 
 #include "WireCellApps/Main.h"
 
+#include "WireCellUtil/Version.h"
 #include "WireCellUtil/NamedFactory.h"
 #include "WireCellUtil/String.h"
 #include "WireCellUtil/Point.h"
 
 #include "WireCellIface/IConfigurable.h"
 #include "WireCellIface/ITerminal.h"
 #include "WireCellIface/IApplication.h"
+#include "WireCellIface/INamed.h"
 
 #include <boost/program_options.hpp>
 #include <boost/algorithm/string.hpp>
@@ -25,6 +27,10 @@
 #include <fftw3.h>
 #endif
 
+#if HAVE_TBB_LIB
+#include <tbb/global_control.h>
+#endif
+
 using namespace WireCell;
 using namespace std;
 namespace po = boost::program_options;
@@ -79,7 +85,16 @@ int Main::cmdline(int argc, char* argv[])
          "specify a Jsonnet top level arguments variable=<code>")
 
         ("path,P", po::value<vector<string> >(),
-         "add to JSON/Jsonnet search path");
+         "add to JSON/Jsonnet search path")
+#if HAVE_TBB_LIB
+        ("threads,t", po::value<int>(),
+         "limit number of threads used")
+#endif
+
+        ("version,v", 
+         "print the compiled version to stdout")
+
+        ;
     // clang-format on
 
     po::variables_map opts;
@@ -171,13 +186,28 @@ int Main::cmdline(int argc, char* argv[])
         }
     }
 
+#ifdef HAVE_TBB_LIB
+    if (opts.count("threads")) {
+        m_threads = opts["threads"].as<int>();
+    }
+#endif
+
+    if (opts.count("version")) {
+        std::cout << version() << std::endl;
+    }
+
     // Maybe make this cmdline configurable.  For now, set all
     // backends the same.
     Log::set_pattern("[%H:%M:%S.%03e] %L [%^%=8n%$] %v");
 
     return 0;
 }
 
+std::string Main::version() const
+{
+    return WireCell::version;
+}
+
 void Main::add_plugin(const std::string& libname) { m_plugins.push_back(libname); }
 
 void Main::add_app(const std::string& tn) { m_apps.push_back(tn); }
@@ -209,6 +239,9 @@ void Main::add_path(const std::string& dirname) { m_load_path.push_back(dirname)
 
 void Main::initialize()
 {
+    // Here we got thought the boot-up sequence steps.
+
+    // Load configuration files
     for (auto filename : m_cfgfiles) {
         l->info("loading config file {} ...", filename);
         Persist::Parser p(m_load_path, m_extvars, m_extcode, m_tlavars, m_tlacode);
@@ -217,7 +250,7 @@ void Main::initialize()
         l->info("...done");
     }
 
-    // Find if we have our special configuration entry
+    // Find if we have our own special configuration entry
     int ind = m_cfgmgr.index("wire-cell");
     Configuration main_cfg = m_cfgmgr.pop(ind);
     if (!main_cfg.isNull()) {
@@ -243,11 +276,9 @@ void Main::initialize()
         pm.add(pname, lname);
     }
 
-    // Apply any user configuration.  This is a two step.  First, just
-    // assure all the components referenced in the configuration
-    // sequence can be instantiated.  Then, find them again and
-    // actually configure them.  This way, any problems fails fast.
+    // Apply any component configuration sequence.
 
+    // Instantiation
     for (auto c : m_cfgmgr.all()) {
         if (c.isNull()) {
             continue;  // allow and ignore any totally empty configurations
@@ -265,6 +296,23 @@ void Main::initialize()
         l->info("constructing app: \"{}\"", c);
         Factory::lookup_tn<IApplication>(c);
     }
+
+    // Give any named components their name.
+    for (auto c : m_cfgmgr.all()) {
+        if (c.isNull()) {
+            continue;  // allow and ignore any totally empty configurations
+        }
+        string type = get<string>(c, "type");
+        string name = get<string>(c, "name");
+        auto namobj = Factory::find_maybe<INamed>(type, name);  // doesn't throw.
+        if (!namobj) {
+            continue;
+        }
+        namobj->set_name(name);
+    }
+
+    // Finally, ask any configurables for their default, merge with
+    // user config and give back.
     for (auto c : m_cfgmgr.all()) {
         if (c.isNull()) {
             continue;  // allow and ignore any totally empty configurations
@@ -295,12 +343,28 @@ void Main::operator()()
         auto a = Factory::find<IApplication>(type, name);  // throws
         app_objs.push_back(a);
     }
+    l->debug("executing {} apps, thread limit {}:",
+             m_apps.size(), m_threads);
+
+#if HAVE_TBB_LIB
+    std::unique_ptr<tbb::global_control> gc;
+    if (m_threads) {
+        gc = std::make_unique<tbb::global_control>(
+            tbb::global_control::max_allowed_parallelism,
+            m_threads);
+    }
+    l->debug("executing {} apps, thread limit {}:",
+             m_apps.size(), m_threads);
+#else
     l->debug("executing {} apps:", m_apps.size());
+#endif
+
     for (size_t ind = 0; ind < m_apps.size(); ++ind) {
         auto aobj = app_objs[ind];
         l->debug("executing app: \"{}\"", m_apps[ind]);
         aobj->execute();  // throws
     }
+
 }
 
 void Main::finalize()

apps/wscript_build

@@ -1,3 +1,8 @@
-bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS')
+if 'HAVE_TBB' in bld.env:
+    bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS TBB')
+else:
+    bld.smplpkg('WireCellApps', use='WireCellIface FFTWTHREADS')
+
+
 #            app_use='WireCellUtil JSONNET',
 #            test_use='WireCellUtil DYNAMO BOOST JSONCPP JSONNET')

aux/inc/WireCellAux/ClusterHelpers.h

@@ -0,0 +1,33 @@
+// Provide some helper functions for working with ICluster
+
+#ifndef WIRECELLAUX_CLUSTERJSONIFY
+#define WIRECELLAUX_CLUSTERJSONIFY
+
+#include "WireCellUtil/Configuration.h"
+#include "WireCellIface/IFrame.h"
+#include "WireCellIface/ISlice.h"
+#include "WireCellIface/ICluster.h"
+
+#include <string>
+
+namespace WireCell::Aux {
+    /// Return JSON representation of the cluster.
+    Json::Value jsonify(const ICluster::pointer& cluster,
+                         double drift_speed);
+
+    /// Return name for the cluster in a canonical form suitable for
+    /// use as a file name.  It only includes info related directly to
+    /// the cluster (the ident).
+    std::string name(const ICluster::pointer& cluster);
+
+    /// Return the slices in the cluster.
+    ISlice::vector find_slices(const ICluster::pointer& cluster);
+
+    /// Return the frame of the first slice in the cluster.  Note, in
+    /// principle, clusters can span frames.
+    IFrame::pointer find_frame(const ICluster::pointer& cluster);
+
+}
+
+#endif
+

aux/inc/WireCellAux/Configurable.h

@@ -0,0 +1,64 @@
+/** A base class providing auto-configuration.
+
+    This works in conjunction with a config struct which is produced
+    via shema codegen.
+
+    To use
+
+    1) Inherit your MyConfigurable class from
+    Configurable<Cfg::MyConfigurable> instead of directly
+    IConfigurable.
+
+    2) If migrating from old, remove configure() and
+    default_configure() methods.
+
+    3) If you configure() method held non-configuration related
+    initialization, it may move into configured().
+
+    4) Use the protected m_cfg object to access the configuration
+    struct including any "service" type components "used" by yours.
+
+ */
+
+
+#ifndef WIRECELLAUX_CONFIGURABLE
+#define WIRECELLAUX_CONFIGURABLE
+
+#include "WireCellUtil/nljs2jcpp.hpp" // remove when ditch JsonCPP
+
+#include "WireCellIface/IConfigurable.h"
+
+namespace WireCell::Aux {
+
+    template<typename CfgStruct>
+    class Configurable : public IConfigurable {
+      public:
+        using cfg_type = CfgStruct;
+
+        // Subclass may implement in order to get post-config entry.
+        virtual void configured() { }
+
+        // Subclass need not but MAY overide IF it also calls.
+        virtual void configure(const WireCell::Configuration& jcfg)
+        {
+            nljs_t nljs = jcfg;
+            m_cfg = nljs.get<cfg_type>();
+            configured();
+        }
+
+        // Subclass need not but MAY overide IF it also calls.
+        virtual WireCell::Configuration default_configuration() const
+        {
+            const nljs_t nljs = m_cfg;
+            return nljs.get<Json::Value>();
+        }
+
+      protected:
+        // Subclass should use m_cfg directly.
+        cfg_type m_cfg;
+    };
+
+}
+
+
+#endif

aux/inc/WireCellAux/DepoTools.h

@@ -0,0 +1,29 @@
+/**
+   Some helper functions operating on IDepo
+ */
+#ifndef WIRECELLAUX_DEPOTOOLS
+#define WIRECELLAUX_DEPOTOOLS
+
+#include "WireCellIface/IDepo.h"
+#include "WireCellUtil/Array.h"
+
+namespace WireCell::Aux {
+
+     /** Fill data and info arrays from IDepoSet.
+
+         The "data" array will be shaped (ndepos,7) and each
+         7-tuple holds: (time, charge, x, y, z, long, tran).
+
+         The "info" array is integer and shaped (nedpos, 4).  Each
+         4-tuple holds: (id, pdg, gen, child).
+
+         If "priors" is true, then run the depos through flatten() in
+         order to save them and give non-zero gen and child.  O.w.,
+         only those depos in pass directly in the vector are saved.
+     */
+     void fill(Array::array_xxf& data, Array::array_xxi& info, 
+               const IDepo::vector& depos, bool priors = true);
+
+}
+
+#endif

aux/inc/WireCellAux/FrameTools.h

@@ -6,12 +6,19 @@
 
 #include "WireCellIface/IFrame.h"
 #include "WireCellUtil/Array.h"
+#include "WireCellUtil/Logging.h"
 
 namespace WireCell {
-    namespace aux {
+    namespace Aux {
 
-        // Print some info to cerr about frame
-        void dump_frame(WireCell::IFrame::pointer frame);
+        /// Return a canonical name for the frame.
+        std::string name(const WireCell::IFrame::pointer& frame);
+
+        /// Print some info to cerr about frame.  If log is null, use stderr.
+        void dump_frame(WireCell::IFrame::pointer frame, Log::logptr_t log = nullptr);
+
+        /// Return brief info string summarizing frame and trace tag.
+        std::string taginfo(const WireCell::IFrame::pointer& frame);
 
         /// Return a vector of traces which have no trace tags.  Here,
         /// any frame tags are ignored.  Returned vector of traces has
@@ -106,7 +113,7 @@ namespace WireCell {
         /// outside of any individual trace.
         IFrame::pointer sum(std::vector<IFrame::pointer> frames, int ident);
 
-    }  // namespace aux
+    }  // namespace Aux
 }  // namespace WireCell
 
 #endif