From 309e1245016630259efa75fcf4aa249b8028497b Mon Sep 17 00:00:00 2001 From: David Boehme Date: Tue, 30 Jan 2024 12:32:32 -0800 Subject: [PATCH] calql: Add leaf() preprocessor kernel (#525) * Add preprocessor leaf() kernel * leaf kernel can use path attributes * Always apply preprocessor before filter * Add documentation for leaf kernel --- doc/sphinx/calql.rst | 1 + .../common/CaliperMetadataAccessInterface.h | 2 + src/common/CaliperMetadataAccessInterface.cpp | 24 ++++++- src/reader/Preprocessor.cpp | 71 +++++++++++++++++-- src/reader/QueryProcessor.cpp | 4 +- src/reader/test/test_preprocessor.cpp | 32 ++++++++- src/tools/cali-query/cali-query.cpp | 4 +- 7 files changed, 127 insertions(+), 11 deletions(-) diff --git a/doc/sphinx/calql.rst b/doc/sphinx/calql.rst index eb7a9a652..3e90a929c 100644 --- a/doc/sphinx/calql.rst +++ b/doc/sphinx/calql.rst @@ -29,6 +29,7 @@ This table contains a quick reference of all CalQL statements: =truncate(,) # computes = - mod(, S) =first(,, ...) # is the first of , , ... found in the input record =sum(,,...) # computes sum of , , ... in the input record + =leaf() # Takes the leaf of the region hierarchy ... IF # apply only if input record meets condition SELECT # Select attributes and define aggregations (i.e., select columns) diff --git a/include/caliper/common/CaliperMetadataAccessInterface.h b/include/caliper/common/CaliperMetadataAccessInterface.h index 4a5bcda79..d57316041 100644 --- a/include/caliper/common/CaliperMetadataAccessInterface.h +++ b/include/caliper/common/CaliperMetadataAccessInterface.h @@ -65,4 +65,6 @@ class CaliperMetadataAccessInterface get_globals() = 0; }; +Entry get_path_entry(const CaliperMetadataAccessInterface& db, const Entry& e); + } diff --git a/src/common/CaliperMetadataAccessInterface.cpp b/src/common/CaliperMetadataAccessInterface.cpp index f7d47172a..978afd664 100644 --- a/src/common/CaliperMetadataAccessInterface.cpp +++ b/src/common/CaliperMetadataAccessInterface.cpp @@ -32,4 +32,26 @@ CaliperMetadataAccessInterface::find_attributes_with_prop(int prop) const ret.push_back(attr); return ret; -} \ No newline at end of file +} + +namespace cali +{ + +Entry +get_path_entry(const CaliperMetadataAccessInterface& db, const Entry& e) +{ + Entry ret; + + if (e.is_reference()) { + for (Node* node = e.node(); node; node = node->parent()) + if (db.get_attribute(node->attribute()).is_nested()) { + ret = Entry(node); + break; + } + } + + return ret; +} + +} + diff --git a/src/reader/Preprocessor.cpp b/src/reader/Preprocessor.cpp index c9a52fd51..d4716ea13 100644 --- a/src/reader/Preprocessor.cpp +++ b/src/reader/Preprocessor.cpp @@ -40,7 +40,6 @@ Variant get_value(const CaliperMetadataAccessInterface& db, const std::string& a return Variant(); } - class Kernel { public: @@ -273,7 +272,7 @@ class SumKernel : public Kernel if (v_tgt.empty()) continue; - + v_sum += v_tgt; } @@ -293,12 +292,72 @@ class SumKernel : public Kernel }; +class LeafKernel : public Kernel +{ + bool m_use_path; + std::string m_res_attr_name; + Attribute m_res_attr; + std::string m_tgt_attr_name; + Attribute m_tgt_attr; + +public: + + LeafKernel(const std::string& def) + : m_use_path(true), + m_res_attr_name(def), + m_res_attr(Attribute::invalid), + m_tgt_attr(Attribute::invalid) + { } + + LeafKernel(const std::string& def, const std::string& tgt) + : m_use_path(false), + m_res_attr_name(def), + m_res_attr(Attribute::invalid), + m_tgt_attr_name(tgt), + m_tgt_attr(Attribute::invalid) + { } + + void process(CaliperMetadataAccessInterface& db, EntryList& rec) { + if (m_res_attr == Attribute::invalid) { + cali_attr_type type = CALI_TYPE_STRING; + int prop = CALI_ATTR_SKIP_EVENTS | CALI_ATTR_ASVALUE; + + if (!m_use_path) { + m_tgt_attr = db.get_attribute(m_tgt_attr_name); + if (m_tgt_attr == Attribute::invalid) + return; + type = m_tgt_attr.type(); + prop |= m_tgt_attr.properties(); + prop &= ~CALI_ATTR_NESTED; + } + + m_res_attr = db.create_attribute(m_res_attr_name, type, prop); + } + + for (const Entry& e : rec) { + Entry e_target = m_use_path ? get_path_entry(db, e) : e.get(m_tgt_attr); + if (!e_target.empty() && m_res_attr.type() == e_target.value().type()) { + rec.push_back(Entry(m_res_attr, e_target.value())); + return; + } + } + } + + static Kernel* create(const std::string& def, const std::vector& args) { + if (args.empty()) + return new LeafKernel(def); + + return new LeafKernel(def, args.front()); + } +}; + enum KernelID { ScaledRatio, Scale, Truncate, First, - Sum + Sum, + Leaf }; const char* sratio_args[] = { "numerator", "denominator", "scale" }; @@ -315,6 +374,7 @@ const QuerySpec::FunctionSignature kernel_signatures[] = { { KernelID::Truncate, "truncate", 1, 2, scale_args }, { KernelID::First, "first", 1, 8, first_args }, { KernelID::Sum, "sum", 1, 8, first_args }, + { KernelID::Leaf, "leaf", 0, 1, scale_args }, QuerySpec::FunctionSignatureTerminator }; @@ -326,10 +386,11 @@ const KernelCreateFn kernel_create_fn[] = { ScaleKernel::create, TruncateKernel::create, FirstKernel::create, - SumKernel::create + SumKernel::create, + LeafKernel::create }; -constexpr int MAX_KERNEL_ID = 4; +constexpr int MAX_KERNEL_ID = 5; } diff --git a/src/reader/QueryProcessor.cpp b/src/reader/QueryProcessor.cpp index 013d668e0..6ce37094d 100644 --- a/src/reader/QueryProcessor.cpp +++ b/src/reader/QueryProcessor.cpp @@ -23,8 +23,8 @@ struct QueryProcessor::QueryProcessorImpl void process_record(CaliperMetadataAccessInterface& db, const EntryList& in_rec) { - if (filter.pass(db, in_rec)) { - auto rec = preprocessor.process(db, in_rec); + auto rec = preprocessor.process(db, in_rec); + if (filter.pass(db, rec)) { if (do_aggregate) aggregator.add(db, rec); diff --git a/src/reader/test/test_preprocessor.cpp b/src/reader/test/test_preprocessor.cpp index 6898d873a..1c2702400 100644 --- a/src/reader/test/test_preprocessor.cpp +++ b/src/reader/test/test_preprocessor.cpp @@ -432,4 +432,34 @@ TEST(PreprocessorTest, SumKernel) EXPECT_EQ(a_it->second.value().to_int(), 42); EXPECT_EQ(s_it->second.value().to_int(), 66); -} \ No newline at end of file +} + +TEST(PreprocessorTest, LeafKernel) +{ + CaliperMetadataDB db; + + Attribute ctx = + db.create_attribute("ctx", CALI_TYPE_STRING, CALI_ATTR_DEFAULT); + + Attribute a[2] = { ctx, ctx }; + Variant v[2] = { Variant("foo"), Variant("bar") }; + + Node* n_1 = db.make_tree_entry(2, a, v); + + EntryList rec; + rec.push_back(Entry(n_1)); + + QuerySpec spec; + spec.preprocess_ops.push_back(::make_spec("leaf", ::make_op("leaf", "ctx"))); + + Preprocessor pp(spec); + EntryList out = pp.process(db, rec); + + Attribute leaf_attr = db.get_attribute("leaf"); + + ASSERT_NE(leaf_attr, Attribute::invalid); + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[1].attribute(), leaf_attr.id()); + EXPECT_EQ(out[1].value(), n_1->data()); + EXPECT_NE(out[1].node()->parent(), n_1->parent()); +} diff --git a/src/tools/cali-query/cali-query.cpp b/src/tools/cali-query/cali-query.cpp index cce80bd14..e4e9fa963 100644 --- a/src/tools/cali-query/cali-query.cpp +++ b/src/tools/cali-query/cali-query.cpp @@ -302,10 +302,10 @@ int main(int argc, const char* argv[]) else snap_proc = aggregate; - if (!spec.preprocess_ops.empty()) - snap_proc = SnapshotFilterStep(Preprocessor(spec), snap_proc); if (spec.filter.selection == QuerySpec::FilterSelection::List) snap_proc = SnapshotFilterStep(RecordSelector(spec), snap_proc); + if (!spec.preprocess_ops.empty()) + snap_proc = SnapshotFilterStep(Preprocessor(spec), snap_proc); if (args.is_set("list-attributes")) { node_proc = AttributeExtract(snap_proc);