diff --git a/Config.cc b/Config.cc index 201b28e9..5f9d46b6 100644 --- a/Config.cc +++ b/Config.cc @@ -99,11 +99,13 @@ namespace Config bool backwardFit = false; bool includePCA = false; - bool json_patch_dump_before = false; - bool json_patch_dump_after = false; - bool json_patch_verbose = false; + bool json_dump_before = false; + bool json_dump_after = false; + bool json_verbose = false; std::vector json_patch_filenames; + std::vector json_load_filenames; std::string json_save_iters_fname_fmt; + bool json_save_iters_include_iter_info_preamble = false; void RecalculateDependentConstants() { diff --git a/Config.h b/Config.h index 67e91adb..dfff5543 100644 --- a/Config.h +++ b/Config.h @@ -399,11 +399,13 @@ namespace Config // ================================================================ - extern bool json_patch_dump_before; - extern bool json_patch_dump_after; - extern bool json_patch_verbose; + extern bool json_dump_before; + extern bool json_dump_after; + extern bool json_verbose; extern std::vector json_patch_filenames; + extern std::vector json_load_filenames; extern std::string json_save_iters_fname_fmt; + extern bool json_save_iters_include_iter_info_preamble; // ================================================================ diff --git a/Track.cc b/Track.cc index 4ead16b4..17b9f10d 100644 --- a/Track.cc +++ b/Track.cc @@ -100,6 +100,25 @@ bool TrackBase::hasSillyValues(bool dump, bool fix, const char* pref) return is_silly; } +const char* TrackBase::algoint_to_cstr(int algo) +{ + static const char* names[] = { "undefAlgorithm", "ctf", "duplicateMerge", + "cosmics", "initialStep", "lowPtTripletStep", "pixelPairStep", "detachedTripletStep", + "mixedTripletStep", "pixelLessStep", "tobTecStep", "jetCoreRegionalStep", "conversionStep", + "muonSeededStepInOut", "muonSeededStepOutIn", "outInEcalSeededConv", "inOutEcalSeededConv", + "nuclInter", "standAloneMuon", "globalMuon", "cosmicStandAloneMuon", "cosmicGlobalMuon", + "highPtTripletStep", "lowPtQuadStep", "detachedQuadStep", "reservedForUpgrades1", + "reservedForUpgrades2", "bTagGhostTracks", "beamhalo", "gsf", "hltPixel", "hltIter0", + "hltIter1", "hltIter2", "hltIter3", "hltIter4", "hltIterX", "hiRegitMuInitialStep", + "hiRegitMuLowPtTripletStep", "hiRegitMuPixelPairStep", "hiRegitMuDetachedTripletStep", + "hiRegitMuMixedTripletStep", "hiRegitMuPixelLessStep", "hiRegitMuTobTecStep", + "hiRegitMuMuonSeededStepInOut", "hiRegitMuMuonSeededStepOutIn", "algoSize" }; + + if (algo < 0 || algo >= (int) TrackAlgorithm::algoSize) return names[0]; + return names[algo]; +} + + //============================================================================== // Track //============================================================================== diff --git a/Track.h b/Track.h index e84be4c0..990dbf46 100644 --- a/Track.h +++ b/Track.h @@ -341,6 +341,8 @@ class TrackBase // bool isStopped() const { return status_.stopped; } // void setStopped() { status_.stopped = true; } + static const char* algoint_to_cstr(int algo); + // ------------------------------------------------------------------------ protected: diff --git a/mkFit/IterationConfig.cc b/mkFit/IterationConfig.cc index 65a0b3e6..a5572bea 100644 --- a/mkFit/IterationConfig.cc +++ b/mkFit/IterationConfig.cc @@ -1,5 +1,6 @@ #include "IterationConfig.h" #include "Config.h" +#include "Track.h" // for TrackBase::algoint_to_cstr, sigh #include "nlohmann/json.hpp" @@ -8,11 +9,18 @@ #include #include +// Redefine to also support ordered_json ... we want to keep variable order in JSON save files. +#define ITCONF_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } \ + inline void to_json(nlohmann::ordered_json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::ordered_json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + namespace mkfit { // Begin AUTO code, some members commented out. -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationLayerConfig, +ITCONF_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationLayerConfig, /* float */ m_select_min_dphi, /* float */ m_select_max_dphi, /* float */ m_select_min_dq, @@ -31,7 +39,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationLayerConfig, /* float */ c_c2_2 ) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationParams, +ITCONF_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationParams, /* int */ nlayers_per_seed, /* int */ maxCandsPerSeed, /* int */ maxHolesPerCand, @@ -50,7 +58,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationParams, /* float */ c_dzmax_el ) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationConfig, +ITCONF_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationConfig, // /* int */ m_iteration_index, /* int */ m_track_algorithm, /* mkfit::IterationParams */ m_params, @@ -63,7 +71,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationConfig, // /* function */ m_partition_seeds ) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationsInfo, +ITCONF_DEFINE_TYPE_NON_INTRUSIVE(mkfit::IterationsInfo, /* vector */ m_iterations ) @@ -243,12 +251,18 @@ int ConfigJsonPatcher::replace(const nlohmann::json &j) std::string s("/"); s += key; nlohmann::json::json_pointer jp(s); - if (m_verbose) + if (m_current->at(jp) != value) { - std::cout << " " << get_abs_path() << s << ": " << m_current->at(jp) << " -> " << value << "\n"; + if (m_verbose) + std::cout << " " << get_abs_path() << s << ": " << m_current->at(jp) << " -> " << value << "\n"; + + m_current->at(jp) = value; + ++n_replaced; } - m_current->at(jp) = value; - ++n_replaced; + // else { + // if (m_verbose) + // std::cout << " " << get_abs_path() << s << ": " << m_current->at(jp) << " -> unchanged\n"; + // } } else { @@ -382,27 +396,20 @@ namespace } } -ConfigJsonPatcher::PatchReport -ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector &fnames) +void ConfigJson_Patch_Files(IterationsInfo &its_info, const std::vector &fnames, + ConfigJsonPatcher::PatchReport *report) { - using nlohmann::json; - - ConfigJsonPatcher cjp(Config::json_patch_verbose); + ConfigJsonPatcher cjp(Config::json_verbose); cjp.Load(its_info); - if (Config::json_patch_dump_before) - { - std::cout << cjp.dump(3) << "\n"; - } - - ConfigJsonPatcher::PatchReport report; + ConfigJsonPatcher::PatchReport rep; for (auto &fname : fnames) { std::ifstream ifs; open_ifstream(ifs, fname, __func__); - if (Config::json_patch_verbose) + if (Config::json_verbose) { printf("%s begin reading from file %s.\n", __func__, fname.c_str()); } @@ -410,11 +417,11 @@ ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector & int n_read = 0, n_tot_replaced = 0; while (skipws_ifstream(ifs)) { - json j; + nlohmann::json j; ifs >> j; ++n_read; - if (Config::json_patch_verbose) + if (Config::json_verbose) { std::cout << " Read JSON entity " << n_read << " -- applying patch:\n"; // std::cout << j.dump(3) << "\n"; @@ -422,7 +429,7 @@ ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector & int n_replaced = cjp.replace(j); - if (Config::json_patch_verbose) + if (Config::json_verbose) { std::cout << " Replaced " << n_replaced << " entries.\n"; } @@ -430,7 +437,7 @@ ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector & n_tot_replaced += n_replaced; } - if (Config::json_patch_verbose) + if (Config::json_verbose) { printf("%s read %d JSON entities from file %s, replaced %d parameters.\n", __func__, n_read, fname.c_str(), n_tot_replaced); @@ -438,57 +445,130 @@ ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector & ifs.close(); - report.inc_counts(1, n_read, n_tot_replaced); + rep.inc_counts(1, n_read, n_tot_replaced); } - if (Config::json_patch_dump_after) + if (rep.n_replacements > 0) { - std::cout << cjp.dump(3) << "\n"; + cjp.Save(its_info); } - if (report.n_replacements > 0) + if (report) report->inc_counts(rep); +} + +IterationConfig& ConfigJson_Load_File(IterationsInfo &its_info, const std::string &fname, + ConfigJsonPatcher::PatchReport *report) +{ + ConfigJsonPatcher::PatchReport rep; + + std::ifstream ifs; + open_ifstream(ifs, fname, __func__); + + if (Config::json_verbose) { - cjp.Save(its_info); + printf("%s begin reading from file %s.\n", __func__, fname.c_str()); } - return report; -} + if ( ! skipws_ifstream(ifs)) throw std::runtime_error("empty file"); + + nlohmann::json j; + ifs >> j; + int track_algo = j["m_track_algorithm"]; + + int iii = -1; + for (int i = 0; i < its_info.size(); ++i) + { + if (its_info[i].m_track_algorithm == track_algo) { + iii = i; + break; + } + } + if (iii == -1) throw std::runtime_error("matching IterationConfig not found"); + + if (Config::json_verbose) + { + std::cout << " Read JSON entity, Iteration index is " << iii << " -- applying patch:\n"; + } + ConfigJsonPatcher cjp(Config::json_verbose); + cjp.Load(its_info[iii]); + + int n_replaced = cjp.replace(j); + + cjp.cd_top(); + + if (Config::json_verbose) + { + printf("%s read 1 JSON entity from file %s, replaced %d parameters.\n", + __func__, fname.c_str(), n_replaced); + } + + ifs.close(); + + rep.inc_counts(1, 1, n_replaced); + + if (rep.n_replacements > 0) + { + cjp.Save(its_info[iii]); + } + + if (report) report->inc_counts(rep); + + return its_info[iii]; +} // ============================================================================ // Save each IterationConfig into a separate json file // ============================================================================ -void ConfigJson_Save_Iterations(IterationsInfo &its_info, const std::string &fname_fmt) +void ConfigJson_Save_Iterations(IterationsInfo &its_info, const std::string &fname_fmt, + bool include_iter_info_preamble) { - assert(fname_fmt.find("%d") != std::string::npos && "JSON save filename-format must include a %d substring"); + bool has_pct_d = fname_fmt.find("%d") != std::string::npos; + bool has_pct_s = fname_fmt.find("%s") != std::string::npos; + + assert( (has_pct_d || has_pct_s) && "JSON save filename-format must include a %d or %s substring"); + assert(!(has_pct_d && has_pct_s) && "JSON save filename-format must include only one of %d or %s substrings"); for (int ii = 0; ii < its_info.size(); ++ii) { const IterationConfig &itconf = its_info[ii]; char fname[1024]; - snprintf(fname, 1024, fname_fmt.c_str(), ii); + if (has_pct_d) + snprintf(fname, 1024, fname_fmt.c_str(), ii); + else + snprintf(fname, 1024, fname_fmt.c_str(), TrackBase::algoint_to_cstr(itconf.m_track_algorithm)); std::ofstream ofs; open_ofstream(ofs, fname, __func__); - ofs << "{ \"m_iterations/" << ii << "\": "; - - ofs << std::setprecision(5); + if (include_iter_info_preamble) + { + ofs << "{ \"m_iterations/" << ii << "\": "; + } - nlohmann::json j; + nlohmann::ordered_json j; to_json(j, itconf); ofs << std::setw(1); ofs << j; - ofs << " }\n"; + if (include_iter_info_preamble) + { + ofs << " }"; + } + ofs << "\n"; ofs.close(); } } +void ConfigJson_Dump(IterationsInfo &its_info) +{ + nlohmann::ordered_json j = its_info; + std::cout << j.dump(3) << "\n"; +} // ============================================================================ // Tests for ConfigJson stuff diff --git a/mkFit/IterationConfig.h b/mkFit/IterationConfig.h index aa76ed4c..15d50718 100644 --- a/mkFit/IterationConfig.h +++ b/mkFit/IterationConfig.h @@ -275,6 +275,13 @@ class ConfigJsonPatcher n_json_entities += e; n_replacements += r; } + void inc_counts(const PatchReport& pr) + { + n_files += pr.n_files; + n_json_entities += pr.n_json_entities; + n_replacements += pr.n_replacements; + } + void reset() { n_files = n_json_entities = n_replacements = 0; } }; private: @@ -317,10 +324,26 @@ class ConfigJsonPatcher std::string dump(int indent=2); }; -ConfigJsonPatcher::PatchReport -ConfigJson_Patch_File(IterationsInfo &its_info, const std::vector &fnames); - -void ConfigJson_Save_Iterations(IterationsInfo &its_info, const std::string &fname_fmt); +// Patch IterationsInfo from a vector of files. +// Assumes patch files include iteration-info preambles, i.e., they +// were saved with include_iter_info_preamble=true. +// If report is non-null counts are added to existing object. +void ConfigJson_Patch_Files(IterationsInfo &its_info, const std::vector &fnames, + ConfigJsonPatcher::PatchReport *report=nullptr); + +// Load a single iteration from JSON file. +// Searches for a match between m_algorithm in its_info and in JSON file to decide +// which IterationConfig it will load over. +// Assumes JSON file has been saved WITHOUT iteration-info preamble. +// Returns reference to the selected IterationConfig. +// If report is non-null counts are added to existing object. +IterationConfig& ConfigJson_Load_File(IterationsInfo &its_info, const std::string &fname, + ConfigJsonPatcher::PatchReport *report=nullptr); + +void ConfigJson_Save_Iterations(IterationsInfo &its_info, const std::string &fname_fmt, + bool include_iter_info_preamble); + +void ConfigJson_Dump(IterationsInfo &its_info); void ConfigJson_Test_Direct(IterationConfig &it_cfg); void ConfigJson_Test_Patcher(IterationConfig &it_cfg); diff --git a/mkFit/mkFit.cc b/mkFit/mkFit.cc index d237e411..0a504963 100644 --- a/mkFit/mkFit.cc +++ b/mkFit/mkFit.cc @@ -48,18 +48,37 @@ void initGeom() TrackerInfo::ExecTrackerInfoCreatorPlugin(Config::geomPlugin, Config::TrkInfo, Config::ItrInfo); - if ( ! Config::json_patch_filenames.empty()) + if (Config::json_dump_before) ConfigJson_Dump(Config::ItrInfo); + + if ( ! Config::json_load_filenames.empty()) { - auto report = ConfigJson_Patch_File(Config::ItrInfo, Config::json_patch_filenames); + ConfigJsonPatcher::PatchReport report; + + for (auto &fn : Config::json_load_filenames) + { + ConfigJson_Load_File(Config::ItrInfo, fn, &report); + } - printf("mkFit.cc/%s() read %d JSON entities from %d files, replaced %d parameters.\n", + printf("mkFit.cc/%s--JSON-Load read %d JSON entities from %d files, replaced %d parameters.\n", __func__, report.n_json_entities, report.n_files, report.n_replacements); + } + + if ( ! Config::json_patch_filenames.empty()) + { + ConfigJsonPatcher::PatchReport report; + ConfigJson_Patch_Files(Config::ItrInfo, Config::json_patch_filenames, &report); + + printf("mkFit.cc/%s--JOSN-Patch read %d JSON entities from %d files, replaced %d parameters.\n", + __func__, report.n_json_entities, report.n_files, report.n_replacements); } + if (Config::json_dump_after) ConfigJson_Dump(Config::ItrInfo); + if ( ! Config::json_save_iters_fname_fmt.empty()) { - ConfigJson_Save_Iterations(Config::ItrInfo, Config::json_save_iters_fname_fmt); + ConfigJson_Save_Iterations(Config::ItrInfo, Config::json_save_iters_fname_fmt, + Config::json_save_iters_include_iter_info_preamble); } // Test functions for ConfigJsonPatcher @@ -602,13 +621,18 @@ int main(int argc, const char *argv[]) " must enable: --cmssw-pureseeds --backward-fit-pca\n" "\n----------------------------------------------------------------------------------------------------------\n\n" "JSON config patcher options:\n\n" - " --json-patch patch iteration config from given JSON file (def: do not patch)\n" + " --json-load load single IterationConfig from given JSON file (def: do not load)\n" + " can be specified multiple times for several files\n" + " --json-patch patch IterationsInfo from given JSON file (def: do not patch)\n" " can be specified multiple times for several files\n" " --json-save-iterations save per iteration json files\n" - " %%d in fname-fmt gets replaced with iteration number\n" - " --json-patch-verbose print each patch assignment as it is being made (def: %s)\n" - " --json-patch-dump-before print iteration config before patching (def: %s)\n" - " --json-patch-dump-after print iteration config after patching (def: %s)\n" + " %%d in fname-fmt gets replaced with iteration index\n" + " %%s in fname-fmt gets replaced with iteration algorithm name\n" + " exactly one of %%d and %%s must be specified\n" + " --json-save-iterations-include-iter-info-preamble (def: %s)\n" + " --json-verbose print each patch assignment as it is being made (def: %s)\n" + " --json-dump-before print iteration config before patching (def: %s)\n" + " --json-dump-after print iteration config after patching (def: %s)\n" "\n----------------------------------------------------------------------------------------------------------\n\n" , argv[0], @@ -679,9 +703,10 @@ int main(int argc, const char *argv[]) getOpt(trkParamBased, g_match_opts).c_str(), getOpt(trkParamBased, g_match_opts).c_str(), getOpt(labelBased, g_match_opts).c_str(), getOpt(labelBased, g_match_opts).c_str(), - b2a(Config::json_patch_verbose), - b2a(Config::json_patch_dump_before), - b2a(Config::json_patch_dump_after) + b2a(Config::json_verbose), + b2a(Config::json_save_iters_include_iter_info_preamble), + b2a(Config::json_dump_before), + b2a(Config::json_dump_after) ); printf("List of options for string based inputs \n"); @@ -996,6 +1021,11 @@ int main(int argc, const char *argv[]) Config::cmsswMatchingFW = labelBased; Config::cmsswMatchingBK = labelBased; } + else if (*i == "--json-load") + { + next_arg_or_die(mArgs, i); + Config::json_load_filenames.push_back(*i); + } else if (*i == "--json-patch") { next_arg_or_die(mArgs, i); @@ -1006,17 +1036,21 @@ int main(int argc, const char *argv[]) next_arg_or_die(mArgs, i); Config::json_save_iters_fname_fmt = *i; } - else if (*i == "--json-patch-verbose") + else if (*i == "--json-save-iterations-include-iter-info-preamble") + { + Config::json_save_iters_include_iter_info_preamble = true; + } + else if (*i == "--json-verbose") { - Config::json_patch_verbose = true; + Config::json_verbose = true; } - else if (*i == "--json-patch-dump-before") + else if (*i == "--json-dump-before") { - Config::json_patch_dump_before = true; + Config::json_dump_before = true; } - else if (*i == "--json-patch-dump-after") + else if (*i == "--json-dump-after") { - Config::json_patch_dump_after = true; + Config::json_dump_after = true; } else {