optimization: use finite-state machine instead of regular expression …

…to parse prometheus metrics (#1688)
alibaba · Aug 30, 2024 · 46d63b9 · 46d63b9
1 parent e183d4a
commit 46d63b9
Show file tree

Hide file tree

Showing 17 changed files with 727 additions and 272 deletions.
diff --git a/core/models/MetricEvent.cpp b/core/models/MetricEvent.cpp
@@ -32,6 +32,10 @@ void MetricEvent::SetName(const string& name) {
     mName = StringView(b.data, b.size);
 }
 
+void MetricEvent::SetNameNoCopy(StringView name) {
+    mName = name;
+}
+
 StringView MetricEvent::GetTag(StringView key) const {
     auto it = mTags.mInner.find(key);
     if (it != mTags.mInner.end()) {

diff --git a/core/models/MetricEvent.h b/core/models/MetricEvent.h
@@ -34,6 +34,7 @@ class MetricEvent : public PipelineEvent {
 
     StringView GetName() const { return mName; }
     void SetName(const std::string& name);
+    void SetNameNoCopy(StringView name);
 
     template <typename T>
     bool Is() const {

diff --git a/core/processor/inner/ProcessorPromParseMetricNative.cpp b/core/processor/inner/ProcessorPromParseMetricNative.cpp
@@ -43,15 +43,14 @@ bool ProcessorPromParseMetricNative::IsSupportedEvent(const PipelineEventPtr& e)
     return e.Is<LogEvent>();
 }
 
-// TODO(liqiang): update Parse metricEvent after TextParser merged
 bool ProcessorPromParseMetricNative::ProcessEvent(
     PipelineEventPtr& e, EventsContainer& newEvents, PipelineEventGroup& eGroup, uint64_t timestamp, uint32_t nanoSec) {
     if (!IsSupportedEvent(e)) {
         return false;
     }
     auto& sourceEvent = e.Cast<LogEvent>();
     std::unique_ptr<MetricEvent> metricEvent = eGroup.CreateMetricEvent();
-    if (mParser.ParseLine(sourceEvent.GetContent(prometheus::PROMETHEUS).to_string(), *metricEvent, timestamp)) {
+    if (mParser.ParseLine(sourceEvent.GetContent(prometheus::PROMETHEUS), timestamp, nanoSec, *metricEvent)) {
         newEvents.emplace_back(std::move(metricEvent));
     }
     return true;

diff --git a/core/prometheus/Utils.cpp b/core/prometheus/Utils.cpp
@@ -1,11 +1,9 @@
 #include "prometheus/Utils.h"
 
 #include <iomanip>
-#include <ios>
-#include <iostream>
-#include <sstream>
 
 #include "common/StringTools.h"
+#include "models/StringView.h"
 
 using namespace std;
 
@@ -42,4 +40,31 @@ uint64_t DurationToSecond(const std::string& duration) {
     return 60;
 }
 
+bool IsValidMetric(const StringView& line) {
+    for (auto c : line) {
+        if (c == ' ' || c == '\t') {
+            continue;
+        }
+        if (c == '#') {
+            return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+void SplitStringView(const std::string& s, char delimiter, std::vector<StringView>& result) {
+    size_t start = 0;
+    size_t end = 0;
+
+    while ((end = s.find(delimiter, start)) != std::string::npos) {
+        result.emplace_back(s.data() + start, end - start);
+        start = end + 1;
+    }
+    if (start < s.size()) {
+        result.emplace_back(s.data() + start, s.size() - start);
+    }
+}
+
+
 } // namespace logtail
diff --git a/core/prometheus/Utils.h b/core/prometheus/Utils.h
@@ -2,10 +2,18 @@
 
 
 #include <string>
+
+#include "models/StringView.h"
+
 namespace logtail {
 
 std::string URLEncode(const std::string& value);
 
 std::string SecondToDuration(uint64_t duration);
 uint64_t DurationToSecond(const std::string& duration);
+
+bool IsValidMetric(const StringView& line);
+
+void SplitStringView(const std::string& s, char delimiter, std::vector<StringView>& result);
+
 } // namespace logtail
diff --git a/core/prometheus/async/PromFuture.h b/core/prometheus/async/PromFuture.h
@@ -21,6 +21,10 @@ class PromFuture {
     ReadWriteLock mStateRWLock;
 
     std::vector<std::function<void(const HttpResponse&, uint64_t timestampMilliSec)>> mDoneCallbacks;
+
+#ifdef APSARA_UNIT_TEST_MAIN
+    friend class ScrapeSchedulerUnittest;
+#endif
 };
 
 } // namespace logtail