Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix lifetime for sdk::ReadWriteLogRecord #3147

Open
wants to merge 24 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
aa4471d
Fix lifetime for sdk::ReadWriteLogRecord
owent Nov 15, 2024
8c86bbd
Fix unit test
owent Nov 15, 2024
cae98c6
Fix unit test
owent Nov 15, 2024
b933ee8
Merge branch 'main' into fix_lifetime_in_log_record
owent Nov 15, 2024
cfd5db3
Fix a typo
owent Nov 16, 2024
67f66c7
Merge branch 'main' into fix_lifetime_in_log_record
owent Nov 20, 2024
4b8012d
Merge branch 'main' into fix_lifetime_in_log_record
owent Nov 21, 2024
5595d31
Merge branch 'main' into fix_lifetime_in_log_record
lalitb Nov 22, 2024
40c4632
Merge remote-tracking branch 'github/main' into fix_lifetime_in_log_r…
owent Nov 25, 2024
282efec
Implement a new LogRecordData to store both Owned attributes and attr…
owent Nov 25, 2024
3da4bed
Merge branch 'main' into fix_lifetime_in_log_record
owent Nov 26, 2024
cf758d1
Merge branch 'main' into fix_lifetime_in_log_record
owent Nov 27, 2024
da602f3
Merge branch 'main' into fix_lifetime_in_log_record
owent Dec 5, 2024
94cf407
Merge branch 'main' into fix_lifetime_in_log_record
lalitb Dec 6, 2024
68178bf
Merge branch 'main' into fix_lifetime_in_log_record
owent Dec 7, 2024
4327f87
Merge branch 'main' into fix_lifetime_in_log_record
owent Dec 12, 2024
2f89435
Merge remote-tracking branch 'github/main' into fix_lifetime_in_log_r…
owent Dec 20, 2024
a3cffa7
Merge branch 'main' into fix_lifetime_in_log_record
owent Jan 8, 2025
1fc10b0
Merge branch 'main' into fix_lifetime_in_log_record
owent Jan 21, 2025
b4bfe00
Merge branch 'main' into fix_lifetime_in_log_record
owent Jan 22, 2025
b173dd7
Merge remote-tracking branch 'opentelemetry/main' into fix_lifetime_i…
owent Jan 28, 2025
b5198ae
Fixes iwyu
owent Jan 29, 2025
156c229
Merge remote-tracking branch 'opentelemetry/main' into fix_lifetime_i…
owent Jan 29, 2025
fabc4a0
Merge branch 'main' into fix_lifetime_in_log_record
owent Jan 31, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions exporters/ostream/src/log_record_exporter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
#include "opentelemetry/sdk/common/exporter_utils.h"
#include "opentelemetry/sdk/common/global_log_handler.h"
#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h"
#include "opentelemetry/sdk/logs/read_write_log_record.h"
#include "opentelemetry/sdk/logs/log_record_data.h"
#include "opentelemetry/sdk/logs/recordable.h"
#include "opentelemetry/sdk/resource/resource.h"
#include "opentelemetry/trace/span_id.h"
Expand All @@ -47,7 +47,7 @@ OStreamLogRecordExporter::OStreamLogRecordExporter(std::ostream &sout) noexcept

std::unique_ptr<sdklogs::Recordable> OStreamLogRecordExporter::MakeRecordable() noexcept
{
return std::unique_ptr<sdklogs::Recordable>(new sdklogs::ReadWriteLogRecord());
return std::unique_ptr<sdklogs::Recordable>(new sdklogs::LogRecordData());
}

sdk::common::ExportResult OStreamLogRecordExporter::Export(
Expand All @@ -62,8 +62,8 @@ sdk::common::ExportResult OStreamLogRecordExporter::Export(

for (auto &record : records)
{
auto log_record = std::unique_ptr<sdklogs::ReadWriteLogRecord>(
static_cast<sdklogs::ReadWriteLogRecord *>(record.release()));
auto log_record = std::unique_ptr<sdklogs::LogRecordData>(
static_cast<sdklogs::LogRecordData *>(record.release()));

if (log_record == nullptr)
{
Expand Down
21 changes: 11 additions & 10 deletions exporters/ostream/test/ostream_log_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "opentelemetry/nostd/utility.h"
#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h"
#include "opentelemetry/sdk/logs/exporter.h"
#include "opentelemetry/sdk/logs/log_record_data.h"
#include "opentelemetry/sdk/logs/logger_provider.h"
#include "opentelemetry/sdk/logs/processor.h"
#include "opentelemetry/sdk/logs/read_write_log_record.h"
Expand Down Expand Up @@ -82,7 +83,7 @@ TEST(OStreamLogRecordExporter, Shutdown)

// After processor/exporter is shutdown, no logs should be sent to stream
auto record = exporter->MakeRecordable();
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetBody("Log record not empty");
static_cast<sdklogs::LogRecordData *>(record.get())->SetBody("Log record not empty");
exporter->Export(nostd::span<std::unique_ptr<sdklogs::Recordable>>(&record, 1));

// Restore original stringstream buffer
Expand Down Expand Up @@ -171,12 +172,12 @@ TEST(OStreamLogRecordExporter, SimpleLogToCout)
// Create a log record and manually timestamp, severity, name, message
common::SystemTimestamp now(std::chrono::system_clock::now());

auto record = std::unique_ptr<sdklogs::Recordable>(new sdklogs::ReadWriteLogRecord());
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetTimestamp(now);
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetObservedTimestamp(now);
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())
auto record = std::unique_ptr<sdklogs::Recordable>(new sdklogs::LogRecordData());
static_cast<sdklogs::LogRecordData *>(record.get())->SetTimestamp(now);
static_cast<sdklogs::LogRecordData *>(record.get())->SetObservedTimestamp(now);
static_cast<sdklogs::LogRecordData *>(record.get())
->SetSeverity(logs_api::Severity::kTrace); // kTrace has enum value of 1
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetBody("Message");
static_cast<sdklogs::LogRecordData *>(record.get())->SetBody("Message");

opentelemetry::sdk::instrumentationscope::InstrumentationScope instrumentation_scope =
GetTestInstrumentationScope();
Expand Down Expand Up @@ -248,10 +249,10 @@ TEST(OStreamLogRecordExporter, LogWithStringAttributesToCerr)

// Set resources for this log record only of type <string, string>
auto resource = opentelemetry::sdk::resource::Resource::Create({{"key1", "val1"}});
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetResource(resource);
static_cast<sdklogs::LogRecordData *>(record.get())->SetResource(resource);

// Set attributes to this log record of type <string, AttributeValue>
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetAttribute("a", true);
static_cast<sdklogs::LogRecordData *>(record.get())->SetAttribute("a", true);

opentelemetry::sdk::instrumentationscope::InstrumentationScope instrumentation_scope =
GetTestInstrumentationScope();
Expand Down Expand Up @@ -326,12 +327,12 @@ TEST(OStreamLogRecordExporter, LogWithVariantTypesToClog)
nostd::span<int> data1{array1.data(), array1.size()};

auto resource = opentelemetry::sdk::resource::Resource::Create({{"res1", data1}});
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())->SetResource(resource);
static_cast<sdklogs::LogRecordData *>(record.get())->SetResource(resource);

// Set resources for this log record of bool types as the value
// e.g. key/value is a par of type <string, array of bools>
std::array<bool, 3> array = {false, true, false};
static_cast<sdklogs::ReadWriteLogRecord *>(record.get())
static_cast<sdklogs::LogRecordData *>(record.get())
->SetAttribute("attr1", nostd::span<bool>{array.data(), array.size()});

opentelemetry::sdk::instrumentationscope::InstrumentationScope instrumentation_scope =
Expand Down
3 changes: 3 additions & 0 deletions exporters/otlp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,9 @@ if(BUILD_TESTING)
else()
find_library(GMOCK_LIB gmock PATH_SUFFIXES lib)
endif()
if(NOT GMOCK_LIB)
unset(GMOCK_LIB CACHE)
endif()
endif()

if(WITH_OTLP_GRPC)
Expand Down
3 changes: 3 additions & 0 deletions exporters/zipkin/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ if(BUILD_TESTING)
else()
find_library(GMOCK_LIB gmock PATH_SUFFIXES lib)
endif()
if(NOT GMOCK_LIB)
unset(GMOCK_LIB CACHE)
endif()

add_executable(zipkin_exporter_test test/zipkin_exporter_test.cc)

Expand Down
245 changes: 245 additions & 0 deletions sdk/include/opentelemetry/sdk/common/attribute_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <cstdint>
#include <initializer_list>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
Expand Down Expand Up @@ -49,6 +50,8 @@ using OwnedAttributeValue = nostd::variant<bool,
std::vector<uint64_t>,
std::vector<uint8_t>>;

using OwnedAttributeView = nostd::variant<std::vector<nostd::string_view>, std::unique_ptr<bool[]>>;

enum OwnedAttributeType
{
kTypeBool,
Expand Down Expand Up @@ -215,6 +218,248 @@ class OrderedAttributeMap : public std::map<std::string, OwnedAttributeValue>
AttributeConverter converter_;
};

/**
* Class for storing attributes.
*/
struct MixedAttributeMapStorage
{
std::unordered_map<std::string, opentelemetry::common::AttributeValue> attributes;
AttributeMap owned_attributes;
std::unordered_map<std::string, OwnedAttributeView> owened_attributes_view;
};

/**
* Set an owned copy (OwnedAttributeValue) and attribute view of a non-owning AttributeValue.
*/
class MixedAttributeViewSetter
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@owent - Do we still need this class?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's still used by MixedAttributeMap , which is used by LogRecordData.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@owent As we agreed here, can we implement the LogRecordData to only contain the owning types, not the mixed types. The motive is to keep the public API simple and minimal. And this will be consistent with SpanData implementation.

Copy link
Member Author

@owent owent Dec 20, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there are some differents between LogRecordData and SpanData.

SpanData::GetAttributes returns std::unordered_map<std::string, opentelemetry::sdk::common::OwnedAttributeValue> but LogRecordData::GetAttributes returns std::unordered_map<std::string, opentelemetry::common::AttributeValue>.
We need storage to store span<T> in opentelemetry::common::AttributeValue, or do you think we can break the API of
ReadableLogRecord::GetAttributes ?

@lalitb

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any idea about keep the mixed types or change the API of ReadableLogRecord::GetAttributes?

{
public:
inline MixedAttributeViewSetter(const nostd::string_view &key,
MixedAttributeMapStorage &storage,
AttributeConverter &converter) noexcept
: key_(&key), storage_(&storage), converter_(&converter)
{}

void operator()(bool v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(int32_t v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(uint32_t v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(int64_t v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(uint64_t v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(double v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
storage_->attributes[std::string(*key_)] = v;
}

void operator()(nostd::string_view v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::string>(owned_value);
}

void operator()(const char *v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::string>(owned_value).c_str();
}

void operator()(nostd::span<const uint8_t> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<uint8_t>>(owned_value);
}

void operator()(nostd::span<const bool> v)
{
storage_->owned_attributes[std::string(*key_)] = (*converter_)(v);
if (v.empty())
{
storage_->attributes[std::string(*key_)] = nostd::span<const bool>{};
}
else
{
std::unique_ptr<bool[]> owned_view{new bool[v.size()]};
for (size_t i = 0; i < v.size(); i++)
{
owned_view[i] = v[i];
}

storage_->attributes[std::string(*key_)] =
nostd::span<const bool>{owned_view.get(), v.size()};
storage_->owened_attributes_view[std::string(*key_)] = std::move(owned_view);
}
}

void operator()(nostd::span<const int32_t> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<int32_t>>(owned_value);
}

void operator()(nostd::span<const uint32_t> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<uint32_t>>(owned_value);
}

void operator()(nostd::span<const int64_t> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<int64_t>>(owned_value);
}

void operator()(nostd::span<const uint64_t> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<uint64_t>>(owned_value);
}

void operator()(nostd::span<const double> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);
storage_->attributes[std::string(*key_)] = nostd::get<std::vector<double>>(owned_value);
}

void operator()(nostd::span<const nostd::string_view> v)
{
auto &owned_value = storage_->owned_attributes[std::string(*key_)];
owned_value = (*converter_)(v);

if (v.empty())
{
storage_->attributes[std::string(*key_)] = nostd::span<const nostd::string_view>{};
}
else
{
auto &owned_view = storage_->owened_attributes_view[std::string(*key_)];
owned_view = std::vector<nostd::string_view>{};
auto &owned_vector = nostd::get<std::vector<nostd::string_view>>(owned_view);

owned_vector.reserve(v.size());
for (auto &data : nostd::get<std::vector<std::string>>(owned_value))
{
owned_vector.push_back(data);
}

storage_->attributes[std::string(*key_)] =
nostd::span<const nostd::string_view>{owned_vector.data(), owned_vector.size()};
}
}

private:
const nostd::string_view *key_;
MixedAttributeMapStorage *storage_;
AttributeConverter *converter_;
};

/**
* Class for storing attributes and attribute view.
*/
class MixedAttributeMap
{
public:
// Construct empty attribute map
MixedAttributeMap() {}

// Construct attribute map and populate with attributes
MixedAttributeMap(const opentelemetry::common::KeyValueIterable &attributes) : MixedAttributeMap()
{
attributes.ForEachKeyValue(
[&](nostd::string_view key, opentelemetry::common::AttributeValue value) noexcept {
nostd::visit(MixedAttributeViewSetter(key, storage_, converter_), value);
return true;
});
}

// Construct attribute map and populate with optional attributes
MixedAttributeMap(const opentelemetry::common::KeyValueIterable *attributes) : MixedAttributeMap()
{
if (attributes != nullptr)
{
attributes->ForEachKeyValue(
[&](nostd::string_view key, opentelemetry::common::AttributeValue value) noexcept {
nostd::visit(MixedAttributeViewSetter(key, storage_, converter_), value);
return true;
});
}
}

// Construct map from initializer list by applying `SetAttribute` transform for every attribute
MixedAttributeMap(
std::initializer_list<std::pair<nostd::string_view, opentelemetry::common::AttributeValue>>
attributes)
: MixedAttributeMap()
{
for (auto &kv : attributes)
{
nostd::visit(MixedAttributeViewSetter(kv.first, storage_, converter_), kv.second);
}
}

// Returns a reference to this map
const std::unordered_map<std::string, opentelemetry::common::AttributeValue> &GetAttributes()
const noexcept
{
return storage_.attributes;
}

const AttributeMap &GetOwnedAttributes() const noexcept { return storage_.owned_attributes; }

// Convert non-owning key-value to owning std::string(key) and OwnedAttributeValue(value)
void SetAttribute(nostd::string_view key,
const opentelemetry::common::AttributeValue &value) noexcept
{
nostd::visit(MixedAttributeViewSetter(key, storage_, converter_), value);
}

void Reserve(AttributeMap::size_type size)
{
storage_.attributes.reserve(size);
storage_.owned_attributes.reserve(size);
}

AttributeMap::size_type Size() const noexcept { return storage_.attributes.size(); }

bool Empty() const noexcept { return storage_.attributes.empty(); }

private:
MixedAttributeMapStorage storage_;
AttributeConverter converter_;
};

} // namespace common
} // namespace sdk
OPENTELEMETRY_END_NAMESPACE
4 changes: 2 additions & 2 deletions sdk/include/opentelemetry/sdk/logs/exporter.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ class OPENTELEMETRY_EXPORT LogRecordExporter
/**
* Create a log recordable. This object will be used to record log data and
* will subsequently be passed to LogRecordExporter::Export. Vendors can implement
* custom recordables or use the default ReadWriteLogRecord recordable provided by the
* SDK.
* custom recordables or use the default ReadWriteLogRecord/LogRecordData recordable provided by
* the SDK.
* @return a newly initialized Recordable object
*
* Note: This method must be callable from multiple threads.
Expand Down
Loading
Loading