From 427b11bbb6c9ac6d9bb153804b6b101995a26e1d Mon Sep 17 00:00:00 2001 From: Kai <33246768+KayzzzZ@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:42:46 +0800 Subject: [PATCH] feat: support span serializer (#1903) * feat: support span serializer Signed-off-by: qianlu.kk --- core/constants/SpanConstants.cpp | 43 ++++++ core/constants/SpanConstants.h | 45 ++++++ core/constants/TagConstants.cpp | 3 + core/constants/TagConstants.h | 4 + core/models/SpanEvent.cpp | 90 ++++++++---- core/models/SpanEvent.h | 8 +- core/pipeline/serializer/SLSSerializer.cpp | 107 +++++++++++++- core/unittest/models/SpanEventUnittest.cpp | 20 +-- .../serializer/SLSSerializerUnittest.cpp | 134 ++++++++++++++++++ 9 files changed, 412 insertions(+), 42 deletions(-) create mode 100644 core/constants/SpanConstants.cpp create mode 100644 core/constants/SpanConstants.h diff --git a/core/constants/SpanConstants.cpp b/core/constants/SpanConstants.cpp new file mode 100644 index 0000000000..262c4f15b2 --- /dev/null +++ b/core/constants/SpanConstants.cpp @@ -0,0 +1,43 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "SpanConstants.h" + +namespace logtail { + + const std::string DEFAULT_TRACE_TAG_TRACE_ID = "traceId"; + const std::string DEFAULT_TRACE_TAG_SPAN_ID = "spanId"; + const std::string DEFAULT_TRACE_TAG_PARENT_ID = "parentSpanId"; + const std::string DEFAULT_TRACE_TAG_SPAN_NAME = "spanName"; + const std::string DEFAULT_TRACE_TAG_SERVICE_NAME = "serviceName"; + const std::string DEFAULT_TRACE_TAG_START_TIME_NANO = "startTime"; + const std::string DEFAULT_TRACE_TAG_END_TIME_NANO = "endTime"; + const std::string DEFAULT_TRACE_TAG_DURATION = "duration"; + const std::string DEFAULT_TRACE_TAG_ATTRIBUTES = "attributes"; + const std::string DEFAULT_TRACE_TAG_RESOURCE = "resources"; + const std::string DEFAULT_TRACE_TAG_LINKS = "links"; + const std::string DEFAULT_TRACE_TAG_EVENTS = "events"; + const std::string DEFAULT_TRACE_TAG_TIMESTAMP = "timestamp"; + const std::string DEFAULT_TRACE_TAG_STATUS_CODE = "statusCode"; + const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE = "statusMessage"; + const std::string DEFAULT_TRACE_TAG_SPAN_KIND = "kind"; + const std::string DEFAULT_TRACE_TAG_TRACE_STATE = "traceState"; + const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME = "name"; +#ifdef __ENTERPRISE__ + // for arms + const std::string DEFAULT_TRACE_TAG_APP_ID = "pid"; + const std::string DEFAULT_TRACE_TAG_IP = "ip"; +#endif + +} // namespace logtail \ No newline at end of file diff --git a/core/constants/SpanConstants.h b/core/constants/SpanConstants.h new file mode 100644 index 0000000000..36a9d2f030 --- /dev/null +++ b/core/constants/SpanConstants.h @@ -0,0 +1,45 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + extern const std::string DEFAULT_TRACE_TAG_TRACE_ID; + extern const std::string DEFAULT_TRACE_TAG_SPAN_ID; + extern const std::string DEFAULT_TRACE_TAG_PARENT_ID; + extern const std::string DEFAULT_TRACE_TAG_SPAN_NAME; + extern const std::string DEFAULT_TRACE_TAG_SERVICE_NAME; + extern const std::string DEFAULT_TRACE_TAG_START_TIME_NANO; + extern const std::string DEFAULT_TRACE_TAG_END_TIME_NANO; + extern const std::string DEFAULT_TRACE_TAG_DURATION; + extern const std::string DEFAULT_TRACE_TAG_ATTRIBUTES; + extern const std::string DEFAULT_TRACE_TAG_RESOURCE; + extern const std::string DEFAULT_TRACE_TAG_LINKS; + extern const std::string DEFAULT_TRACE_TAG_EVENTS; + extern const std::string DEFAULT_TRACE_TAG_TIMESTAMP; + extern const std::string DEFAULT_TRACE_TAG_STATUS_CODE; + extern const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE; + extern const std::string DEFAULT_TRACE_TAG_SPAN_KIND; + extern const std::string DEFAULT_TRACE_TAG_TRACE_STATE; + extern const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME ; +#ifdef __ENTERPRISE__ + // for arms + extern const std::string DEFAULT_TRACE_TAG_APP_ID; + extern const std::string DEFAULT_TRACE_TAG_IP; +#endif + +} // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.cpp b/core/constants/TagConstants.cpp index edd7ea4e55..4d4a865192 100644 --- a/core/constants/TagConstants.cpp +++ b/core/constants/TagConstants.cpp @@ -63,4 +63,7 @@ namespace logtail { const std::string DEFAULT_METRIC_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; const std::string DEFAULT_METRIC_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; +////////////////////////// TRACE //////////////////////// + + } // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.h b/core/constants/TagConstants.h index 52d4213961..0b83d4092d 100644 --- a/core/constants/TagConstants.h +++ b/core/constants/TagConstants.h @@ -44,4 +44,8 @@ namespace logtail { extern const std::string DEFAULT_METRIC_TAG_CONTAINER_IP; extern const std::string DEFAULT_METRIC_TAG_IMAGE_NAME; +////////////////////////// TRACE //////////////////////// + + + } // namespace logtail \ No newline at end of file diff --git a/core/models/SpanEvent.cpp b/core/models/SpanEvent.cpp index 812ccbe54b..4396a92860 100644 --- a/core/models/SpanEvent.cpp +++ b/core/models/SpanEvent.cpp @@ -15,6 +15,7 @@ */ #include "models/SpanEvent.h" +#include "constants/SpanConstants.h" using namespace std; @@ -75,16 +76,15 @@ size_t SpanEvent::SpanLink::DataSize() const { return mTraceId.size() + mSpanId.size() + mTraceState.size() + mTags.DataSize(); } -#ifdef APSARA_UNIT_TEST_MAIN Json::Value SpanEvent::SpanLink::ToJson() const { Json::Value root; - root["traceId"] = mTraceId.to_string(); - root["spanId"] = mSpanId.to_string(); + root[DEFAULT_TRACE_TAG_TRACE_ID] = mTraceId.to_string(); + root[DEFAULT_TRACE_TAG_SPAN_ID] = mSpanId.to_string(); if (!mTraceState.empty()) { - root["traceState"] = mTraceState.to_string(); + root[DEFAULT_TRACE_TAG_TRACE_STATE] = mTraceState.to_string(); } if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } @@ -92,17 +92,18 @@ Json::Value SpanEvent::SpanLink::ToJson() const { return root; } +#ifdef APSARA_UNIT_TEST_MAIN void SpanEvent::SpanLink::FromJson(const Json::Value& value) { - SetTraceId(value["traceId"].asString()); - SetSpanId(value["spanId"].asString()); - if (value.isMember("traceState")) { - string s = value["traceState"].asString(); + SetTraceId(value[DEFAULT_TRACE_TAG_TRACE_ID].asString()); + SetSpanId(value[DEFAULT_TRACE_TAG_SPAN_ID].asString()); + if (value.isMember(DEFAULT_TRACE_TAG_TRACE_STATE)) { + string s = value[DEFAULT_TRACE_TAG_TRACE_STATE].asString(); if (!s.empty()) { SetTraceState(s); } } - if (value.isMember("tags")) { - Json::Value tags = value["tags"]; + if (value.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = value[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } @@ -155,13 +156,12 @@ size_t SpanEvent::InnerEvent::DataSize() const { return sizeof(decltype(mTimestampNs)) + mName.size() + mTags.DataSize(); } -#ifdef APSARA_UNIT_TEST_MAIN Json::Value SpanEvent::InnerEvent::ToJson() const { Json::Value root; - root["name"] = mName.to_string(); - root["timestampNs"] = static_cast(mTimestampNs); + root[DEFAULT_TRACE_TAG_SPAN_EVENT_NAME] = mName.to_string(); + root[DEFAULT_TRACE_TAG_TIMESTAMP] = static_cast(mTimestampNs); if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } @@ -169,11 +169,12 @@ Json::Value SpanEvent::InnerEvent::ToJson() const { return root; } +#ifdef APSARA_UNIT_TEST_MAIN void SpanEvent::InnerEvent::FromJson(const Json::Value& value) { - SetName(value["name"].asString()); - SetTimestampNs(value["timestampNs"].asUInt64()); - if (value.isMember("tags")) { - Json::Value tags = value["tags"]; + SetName(value[DEFAULT_TRACE_TAG_SPAN_EVENT_NAME].asString()); + SetTimestampNs(value[DEFAULT_TRACE_TAG_TIMESTAMP].asUInt64()); + if (value.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = value[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } @@ -350,19 +351,19 @@ Json::Value SpanEvent::ToJson(bool enableEventMeta) const { root["startTimeNs"] = static_cast(mStartTimeNs); root["endTimeNs"] = static_cast(mEndTimeNs); if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } } if (!mEvents.empty()) { - Json::Value& events = root["events"]; + Json::Value& events = root[DEFAULT_TRACE_TAG_EVENTS]; for (const auto& event : mEvents) { events.append(event.ToJson()); } } if (!mLinks.empty()) { - Json::Value& links = root["links"]; + Json::Value& links = root[DEFAULT_TRACE_TAG_LINKS]; for (const auto& link : mLinks) { links.append(link.ToJson()); } @@ -405,21 +406,21 @@ bool SpanEvent::FromJson(const Json::Value& root) { } SetStartTimeNs(root["startTimeNs"].asUInt64()); SetEndTimeNs(root["endTimeNs"].asUInt64()); - if (root.isMember("tags")) { - Json::Value tags = root["tags"]; + if (root.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } } - if (root.isMember("events")) { - Json::Value events = root["events"]; + if (root.isMember(DEFAULT_TRACE_TAG_EVENTS)) { + Json::Value events = root[DEFAULT_TRACE_TAG_EVENTS]; for (const auto& event : events) { InnerEvent* e = AddEvent(); e->FromJson(event); } } - if (root.isMember("links")) { - Json::Value links = root["links"]; + if (root.isMember(DEFAULT_TRACE_TAG_LINKS)) { + Json::Value links = root[DEFAULT_TRACE_TAG_LINKS]; for (const auto& link : links) { SpanLink* l = AddLink(); l->FromJson(link); @@ -438,4 +439,37 @@ bool SpanEvent::FromJson(const Json::Value& root) { } #endif +const static std::string sSpanStatusCodeUnSet = "UNSET"; +const static std::string sSpanStatusCodeOk = "OK"; +const static std::string sSpanStatusCodeError = "ERROR"; + +const std::string& GetStatusString(SpanEvent::StatusCode status) { + switch (status) { + case SpanEvent::StatusCode::Unset: return sSpanStatusCodeUnSet; + case SpanEvent::StatusCode::Ok: return sSpanStatusCodeOk; + case SpanEvent::StatusCode::Error: return sSpanStatusCodeError; + default: return sSpanStatusCodeUnSet; + } +} + +const static std::string sSpanKindUnspecified = "unspecified"; +const static std::string sSpanKindInternal = "internal"; +const static std::string sSpanKindServer = "server"; +const static std::string sSpanKindClient = "client"; +const static std::string sSpanKindProducer = "producer"; +const static std::string sSpanKindConsumer = "consumer"; +const static std::string sSpanKindUnknown = "unknown"; + +const std::string& GetKindString(SpanEvent::Kind kind) { + switch (kind) { + case SpanEvent::Kind::Unspecified: return sSpanKindUnspecified; + case SpanEvent::Kind::Internal: return sSpanKindInternal; + case SpanEvent::Kind::Server: return sSpanKindServer; + case SpanEvent::Kind::Client: return sSpanKindClient; + case SpanEvent::Kind::Producer: return sSpanKindProducer; + case SpanEvent::Kind::Consumer: return sSpanKindConsumer; + default: return sSpanKindUnknown; + } +} + } // namespace logtail diff --git a/core/models/SpanEvent.h b/core/models/SpanEvent.h index 584d64a5d6..780664eb1a 100644 --- a/core/models/SpanEvent.h +++ b/core/models/SpanEvent.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "common/memory/SourceBuffer.h" #include "models/PipelineEvent.h" @@ -67,8 +68,8 @@ class SpanEvent : public PipelineEvent { size_t DataSize() const; -#ifdef APSARA_UNIT_TEST_MAIN Json::Value ToJson() const; +#ifdef APSARA_UNIT_TEST_MAIN void FromJson(const Json::Value& value); #endif @@ -107,8 +108,8 @@ class SpanEvent : public PipelineEvent { size_t DataSize() const; -#ifdef APSARA_UNIT_TEST_MAIN Json::Value ToJson() const; +#ifdef APSARA_UNIT_TEST_MAIN void FromJson(const Json::Value& value); #endif @@ -214,4 +215,7 @@ class SpanEvent : public PipelineEvent { #endif }; +const std::string& GetStatusString(SpanEvent::StatusCode status); +const std::string& GetKindString(SpanEvent::Kind kind); + } // namespace logtail diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 9ffb6b1541..6b9ec3888d 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -15,9 +15,12 @@ #include "pipeline/serializer/SLSSerializer.h" #include "common/Flags.h" +#include "constants/SpanConstants.h" #include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/LogGroupSerializer.h" +#include +#include DECLARE_FLAG_INT32(max_send_log_group_size); @@ -25,6 +28,29 @@ using namespace std; namespace logtail { +std::string SerializeSpanLinksToString(const SpanEvent& event) { + if (event.GetLinks().empty()) { + return ""; + } + Json::Value jsonLinks(Json::arrayValue); + for (const auto& link : event.GetLinks()) { + jsonLinks.append(link.ToJson()); + } + Json::StreamWriterBuilder writer; + return Json::writeString(writer, jsonLinks); +} +std::string SerializeSpanEventsToString(const SpanEvent& event) { + if (event.GetEvents().empty()) { + return ""; + } + Json::Value jsonEvents(Json::arrayValue); + for (const auto& event : event.GetEvents()) { + jsonEvents.append(event.ToJson()); + } + Json::StreamWriterBuilder writer; + return Json::writeString(writer, jsonEvents); +} + template <> bool Serializer>::DoSerialize(vector&& p, std::string& output, @@ -68,9 +94,10 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri // caculate serialized logGroup size first, where some critical results can be cached vector logSZ(group.mEvents.size()); vector> metricEventContentCache(group.mEvents.size()); + vector> spanEventContentCache(group.mEvents.size()); size_t logGroupSZ = 0; switch (eventType) { - case PipelineEvent::Type::LOG: + case PipelineEvent::Type::LOG:{ for (size_t i = 0; i < group.mEvents.size(); ++i) { const auto& e = group.mEvents[i].Cast(); if (e.Empty()) { @@ -83,7 +110,8 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logGroupSZ += GetLogSize(contentSZ, enableNs && e.GetTimestampNanosecond(), logSZ[i]); } break; - case PipelineEvent::Type::METRIC: + } + case PipelineEvent::Type::METRIC:{ for (size_t i = 0; i < group.mEvents.size(); ++i) { const auto& e = group.mEvents[i].Cast(); if (e.Is()) { @@ -107,7 +135,51 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logGroupSZ += GetLogSize(contentSZ, false, logSZ[i]); } break; + } case PipelineEvent::Type::SPAN: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& e = group.mEvents[i].Cast(); + size_t contentSZ = 0; + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_TRACE_ID.size(), e.GetTraceId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_ID.size(), e.GetSpanId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_PARENT_ID.size(), e.GetParentSpanId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_NAME.size(), e.GetName().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_KIND.size(), GetKindString(e.GetKind()).size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_STATUS_CODE.size(), GetStatusString(e.GetStatus()).size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_TRACE_STATE.size(), e.GetTraceState().size()); + // + // set tags and scope tags + Json::Value jsonVal; + for (auto it = e.TagsBegin(); it != e.TagsEnd(); ++it) { + jsonVal[it->first.to_string()] = it->second.to_string(); + } + for (auto it = e.ScopeTagsBegin(); it != e.ScopeTagsEnd(); ++it) { + jsonVal[it->first.to_string()] = it->second.to_string(); + } + Json::StreamWriterBuilder writer; + std::string attrString = Json::writeString(writer, jsonVal); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_ATTRIBUTES.size(), attrString.size()); + spanEventContentCache[i][0] = std::move(attrString); + + auto linkString = SerializeSpanLinksToString(e); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_LINKS.size(), linkString.size()); + spanEventContentCache[i][1] = std::move(linkString); + auto eventString = SerializeSpanEventsToString(e); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_EVENTS.size(), eventString.size()); + spanEventContentCache[i][2] = std::move(eventString); + + // time related + auto startTsNs = std::to_string(e.GetStartTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_START_TIME_NANO.size(), startTsNs.size()); + spanEventContentCache[i][3] = std::move(startTsNs); + auto endTsNs = std::to_string(e.GetEndTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_END_TIME_NANO.size(), endTsNs.size()); + spanEventContentCache[i][4] = std::move(endTsNs); + auto durationNs = std::to_string(e.GetEndTimeNs() - e.GetStartTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_DURATION.size(), durationNs.size()); + spanEventContentCache[i][5] = std::move(durationNs); + logGroupSZ += GetLogSize(contentSZ, false, logSZ[i]); + } break; case PipelineEvent::Type::RAW: for (size_t i = 0; i < group.mEvents.size(); ++i) { @@ -174,6 +246,37 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri } break; case PipelineEvent::Type::SPAN: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& spanEvent = group.mEvents[i].Cast(); + + serializer.StartToAddLog(logSZ[i]); + serializer.AddLogTime(spanEvent.GetTimestamp()); + // set trace_id span_id span_kind status etc + serializer.AddLogContent(DEFAULT_TRACE_TAG_TRACE_ID, spanEvent.GetTraceId()); + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_ID, spanEvent.GetSpanId()); + serializer.AddLogContent(DEFAULT_TRACE_TAG_PARENT_ID, spanEvent.GetParentSpanId()); + // span_name + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_NAME, spanEvent.GetName()); + // span_kind + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_KIND, GetKindString(spanEvent.GetKind())); + // status_code + serializer.AddLogContent(DEFAULT_TRACE_TAG_STATUS_CODE, GetStatusString(spanEvent.GetStatus())); + // trace state + serializer.AddLogContent(DEFAULT_TRACE_TAG_TRACE_STATE, spanEvent.GetTraceState()); + + serializer.AddLogContent(DEFAULT_TRACE_TAG_ATTRIBUTES, spanEventContentCache[i][0]); + + serializer.AddLogContent(DEFAULT_TRACE_TAG_LINKS, spanEventContentCache[i][1]); + serializer.AddLogContent(DEFAULT_TRACE_TAG_EVENTS, spanEventContentCache[i][2]); + + // start_time + serializer.AddLogContent(DEFAULT_TRACE_TAG_START_TIME_NANO, spanEventContentCache[i][3]); + // end_time + serializer.AddLogContent(DEFAULT_TRACE_TAG_END_TIME_NANO, spanEventContentCache[i][4]); + // duration + serializer.AddLogContent(DEFAULT_TRACE_TAG_DURATION, spanEventContentCache[i][5]); + + } break; case PipelineEvent::Type::RAW: for (size_t i = 0; i < group.mEvents.size(); ++i) { diff --git a/core/unittest/models/SpanEventUnittest.cpp b/core/unittest/models/SpanEventUnittest.cpp index 768097a65b..693538324d 100644 --- a/core/unittest/models/SpanEventUnittest.cpp +++ b/core/unittest/models/SpanEventUnittest.cpp @@ -319,13 +319,13 @@ void SpanEventUnittest::TestToJson() { "kind": 3, "startTimeNs": 1715826723000000000, "endTimeNs": 1715826725000000000, - "tags": { + "attributes": { "key1": "value1" }, "events": [ { "name": "test_event", - "timestampNs": 1715826724000000000 + "timestamp": 1715826724000000000 } ], "links": [ @@ -358,13 +358,13 @@ void SpanEventUnittest::TestFromJson() { "kind": 3, "startTimeNs": 1715826723000000000, "endTimeNs": 1715826725000000000, - "tags": { + "attributes": { "key1": "value1" }, "events": [ { "name": "test_event", - "timestampNs": 1715826724000000000 + "timestamp": 1715826724000000000 } ], "links": [ @@ -513,8 +513,8 @@ void InnerEventUnittest::TestToJson() { Json::Value eventJson; string eventStr = R"({ "name": "test", - "timestampNs": 1715826723000000000, - "tags": { + "timestamp": 1715826723000000000, + "attributes": { "key1": "value1" } })"; @@ -528,8 +528,8 @@ void InnerEventUnittest::TestFromJson() { Json::Value eventJson; string eventStr = R"({ "name": "test", - "timestampNs": 1715826723000000000, - "tags": { + "timestamp": 1715826723000000000, + "attributes": { "key1": "value1" } })"; @@ -659,7 +659,7 @@ void SpanLinkUnittest::TestToJson() { "traceId": "test_trace_id", "spanId": "test_span_id", "traceState": "normal", - "tags": { + "attributes": { "key1": "value1" } })"; @@ -675,7 +675,7 @@ void SpanLinkUnittest::TestFromJson() { "traceId": "test_trace_id", "spanId": "test_span_id", "traceState": "normal", - "tags": { + "attributes": { "key1": "value1" } })"; diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index 603f95f953..75a36a307d 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -41,6 +41,7 @@ class SLSSerializerUnittest : public ::testing::Test { BatchedEvents CreateBatchedMetricEvents(bool enableNanosecond, uint32_t nanoTimestamp, bool emptyValue, bool onlyOneTag); BatchedEvents CreateBatchedRawEvents(bool enableNanosecond, bool emptyContent); + BatchedEvents CreateBatchedSpanEvents(); static unique_ptr sFlusher; @@ -215,6 +216,86 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { } { // span + string res, errorMsg; + auto events = CreateBatchedSpanEvents(); + APSARA_TEST_EQUAL(events.mEvents.size(), 1); + APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(13, logGroup.logs(0).contents_size()); + // traceid + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "traceId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "trace-1-2-3-4-5"); + // span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "spanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "span-1-2-3-4-5"); + // parent span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "parentSpanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "parent-1-2-3-4-5"); + // spanName + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "spanName"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "/oneagent/qianlu/local/1"); + // kind + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).key(), "kind"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).value(), "client"); + // code + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).key(), "statusCode"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).value(), "OK"); + // traceState + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).key(), "traceState"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).value(), "test-state"); + // attributes + APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).key(), "attributes"); + auto attrs = logGroup.logs(0).contents(7).value(); + Json::Value jsonVal; + Json::CharReaderBuilder readerBuilder; + std::string errs; + + std::istringstream s(attrs); + bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 10); + APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); + APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); + // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); + // links + APSARA_TEST_EQUAL(logGroup.logs(0).contents(8).key(), "links"); + + auto linksStr = logGroup.logs(0).contents(8).value(); + + std::istringstream ss(linksStr); + ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1); + for (auto& link : jsonVal) { + APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); + APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); + APSARA_TEST_EQUAL(link["traceState"].asString(), "inner-link-trace-state"); + } + // events + APSARA_TEST_EQUAL(logGroup.logs(0).contents(9).key(), "events"); + auto eventsStr = logGroup.logs(0).contents(9).value(); + std::istringstream sss(eventsStr); + ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1); + for (auto& event : jsonVal) { + APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); + APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); + } + // start + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).key(), "startTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).value(), "1000"); + + // end + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).key(), "endTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).value(), "2000"); + + // duration + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).key(), "duration"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).value(), "1000"); } { // raw @@ -391,6 +472,59 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedRawEvents(bool enableNanosecon return batch; } +BatchedEvents SLSSerializerUnittest::CreateBatchedSpanEvents() { + PipelineEventGroup group(make_shared()); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "aaa"); + group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "bbb"); + auto now = std::chrono::system_clock::now(); + auto duration = now.time_since_epoch(); + auto seconds = std::chrono::duration_cast(duration).count(); + // auto nano = std::chrono::duration_cast(duration).count(); + StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); + group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); + group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); + SpanEvent* spanEvent = group.AddSpanEvent(); + spanEvent->SetScopeTag(std::string("scope-tag-0"), std::string("scope-value-0")); + spanEvent->SetTag(std::string("workloadName"), std::string("arms-oneagent-test-ql")); + spanEvent->SetTag(std::string("workloadKind"), std::string("faceless")); + spanEvent->SetTag(std::string("source_ip"), std::string("10.54.0.33")); + spanEvent->SetTag(std::string("host"), std::string("10.54.0.33")); + spanEvent->SetTag(std::string("rpc"), std::string("/oneagent/qianlu/local/1")); + spanEvent->SetTag(std::string("rpcType"), std::string("25")); + spanEvent->SetTag(std::string("callType"), std::string("http-client")); + spanEvent->SetTag(std::string("statusCode"), std::string("200")); + spanEvent->SetTag(std::string("version"), std::string("HTTP1.1")); + auto innerEvent = spanEvent->AddEvent(); + innerEvent->SetTag(std::string("innner-event-key-0"), std::string("inner-event-value-0")); + innerEvent->SetTag(std::string("innner-event-key-1"), std::string("inner-event-value-1")); + innerEvent->SetName("inner-event"); + innerEvent->SetTimestampNs(1000); + auto innerLink = spanEvent->AddLink(); + innerLink->SetTag(std::string("innner-link-key-0"), std::string("inner-link-value-0")); + innerLink->SetTag(std::string("innner-link-key-1"), std::string("inner-link-value-1")); + innerLink->SetTraceId("inner-link-traceid"); + innerLink->SetSpanId("inner-link-spanid"); + innerLink->SetTraceState("inner-link-trace-state"); + spanEvent->SetName("/oneagent/qianlu/local/1"); + spanEvent->SetKind(SpanEvent::Kind::Client); + spanEvent->SetStatus(SpanEvent::StatusCode::Ok); + spanEvent->SetSpanId("span-1-2-3-4-5"); + spanEvent->SetTraceId("trace-1-2-3-4-5"); + spanEvent->SetParentSpanId("parent-1-2-3-4-5"); + spanEvent->SetTraceState("test-state"); + spanEvent->SetStartTimeNs(1000); + spanEvent->SetEndTimeNs(2000); + spanEvent->SetTimestamp(seconds); + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + return batch; +} + UNIT_TEST_CASE(SLSSerializerUnittest, TestSerializeEventGroup) UNIT_TEST_CASE(SLSSerializerUnittest, TestSerializeEventGroupList)