Commit 3a3b7e46 by Abseil Team Committed by Copybara-Service

Reduce memory consumption of structured logging proto encoding by passing tag value

The proto encoding for structured logging currently pessimistically assumes each numeric tag value for encoded fields could be up to `UINT64_MAX`.

In practice, the tag values we care about are all < 16, which only requires 1 byte
of buffer space.

This CL improves the memory consumption by specifying the tag value when
calculating the buffer size needed for the structured logging proto.

PiperOrigin-RevId: 696118135
Change-Id: Iee67deef568cb4df7646d3ddd40c14b490ca0e45
parent 27a0c730
......@@ -578,16 +578,17 @@ void LogMessage::LogBacktraceIfNeeded() {
template <LogMessage::StringType str_type>
void LogMessage::CopyToEncodedBuffer(absl::string_view str) {
auto encoded_remaining_copy = data_->encoded_remaining();
constexpr uint8_t tag_value = str_type == StringType::kLiteral
? ValueTag::kStringLiteral
: ValueTag::kString;
auto start = EncodeMessageStart(
EventTag::kValue, BufferSizeFor(WireType::kLengthDelimited) + str.size(),
EventTag::kValue,
BufferSizeFor(tag_value, WireType::kLengthDelimited) + str.size(),
&encoded_remaining_copy);
// If the `logging.proto.Event.value` field header did not fit,
// `EncodeMessageStart` will have zeroed `encoded_remaining_copy`'s size and
// `EncodeStringTruncate` will fail too.
if (EncodeStringTruncate(str_type == StringType::kLiteral
? ValueTag::kStringLiteral
: ValueTag::kString,
str, &encoded_remaining_copy)) {
if (EncodeStringTruncate(tag_value, str, &encoded_remaining_copy)) {
// The string may have been truncated, but the field header fit.
EncodeMessageLength(start, &encoded_remaining_copy);
data_->encoded_remaining() = encoded_remaining_copy;
......@@ -604,13 +605,14 @@ template void LogMessage::CopyToEncodedBuffer<
template <LogMessage::StringType str_type>
void LogMessage::CopyToEncodedBuffer(char ch, size_t num) {
auto encoded_remaining_copy = data_->encoded_remaining();
constexpr uint8_t tag_value = str_type == StringType::kLiteral
? ValueTag::kStringLiteral
: ValueTag::kString;
auto value_start = EncodeMessageStart(
EventTag::kValue, BufferSizeFor(WireType::kLengthDelimited) + num,
EventTag::kValue,
BufferSizeFor(tag_value, WireType::kLengthDelimited) + num,
&encoded_remaining_copy);
auto str_start = EncodeMessageStart(str_type == StringType::kLiteral
? ValueTag::kStringLiteral
: ValueTag::kString,
num, &encoded_remaining_copy);
auto str_start = EncodeMessageStart(tag_value, num, &encoded_remaining_copy);
if (str_start.data()) {
// The field headers fit.
log_internal::AppendTruncated(ch, num, encoded_remaining_copy);
......
......@@ -35,9 +35,6 @@ void EncodeRawVarint(uint64_t value, size_t size, absl::Span<char> *buf) {
}
buf->remove_prefix(size);
}
constexpr uint64_t MakeTagType(uint64_t tag, WireType type) {
return tag << 3 | static_cast<uint64_t>(type);
}
} // namespace
bool EncodeVarint(uint64_t tag, uint64_t value, absl::Span<char> *buf) {
......
......@@ -199,23 +199,33 @@ constexpr uint64_t MaxVarintForSize(size_t size) {
return size >= 10 ? (std::numeric_limits<uint64_t>::max)()
: (static_cast<uint64_t>(1) << size * 7) - 1;
}
constexpr uint64_t MakeTagType(uint64_t tag, WireType type) {
return tag << 3 | static_cast<uint64_t>(type);
}
// `BufferSizeFor` returns a number of bytes guaranteed to be sufficient to
// store encoded fields of the specified WireTypes regardless of tag numbers and
// data values. This only makes sense for `WireType::kLengthDelimited` if you
// add in the length of the contents yourself, e.g. for string and bytes fields
// by adding the lengths of any encoded strings to the return value or for
// submessage fields by enumerating the fields you may encode into their
// contents.
constexpr size_t BufferSizeFor() { return 0; }
template <typename... T>
constexpr size_t BufferSizeFor(WireType type, T... tail) {
// tag_type + data + ...
return MaxVarintSize() +
(type == WireType::kVarint ? MaxVarintSize() : //
type == WireType::k64Bit ? 8 : //
type == WireType::k32Bit ? 4 : MaxVarintSize()) + //
BufferSizeFor(tail...);
// store encoded fields as `(tag, WireType)`, regardless of data values. This
// only makes sense for `WireType::kLengthDelimited` if you add in the length of
// the contents yourself, e.g. for string and bytes fields by adding the lengths
// of any encoded strings to the return value or for submessage fields by
// enumerating the fields you may encode into their contents.
constexpr size_t BufferSizeFor(uint64_t tag, WireType type) {
size_t buffer_size = VarintSize(MakeTagType(tag, type));
switch (type) {
case WireType::kVarint:
buffer_size += MaxVarintSize();
break;
case WireType::k64Bit:
buffer_size += size_t{8};
break;
case WireType::kLengthDelimited:
buffer_size += MaxVarintSize();
break;
case WireType::k32Bit:
buffer_size += size_t{4};
break;
}
return buffer_size;
}
// absl::Span<const char> represents a view into the un-processed space in a
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment