Skip to content

A
abseil-cpp
Commit 52835439 by Martijn Vels Committed by Copybara-Service
Options

Refactor InlineData to allow for memory sanitizer changes step 2 

PiperOrigin-RevId: 501074382
Change-Id: I26a59ee6452855685ffe89469c352e6384060f59
parent f9e2a524
Showing with 68 additions and 53 deletions
absl/strings/cord.cc
...@@ -168,9 +168,7 @@ constexpr unsigned char Cord::InlineRep::kMaxInline; ...@@ -168,9 +168,7 @@ constexpr unsigned char Cord::InlineRep::kMaxInline;
inline void Cord::InlineRep::set_data(const char* data, size_t n) { inline void Cord::InlineRep::set_data(const char* data, size_t n) {
static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15"); static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15");
data_.set_inline_data(data, n);
cord_internal::SmallMemmove<true>(data_.as_chars(), data, n);
set_inline_size(n);
} }
inline char* Cord::InlineRep::set_data(size_t n) { inline char* Cord::InlineRep::set_data(size_t n) {
...@@ -439,8 +437,8 @@ void Cord::InlineRep::AppendArray(absl::string_view src, ...@@ -439,8 +437,8 @@ void Cord::InlineRep::AppendArray(absl::string_view src,
size_t inline_length = inline_size(); size_t inline_length = inline_size();
if (src.size() <= kMaxInline - inline_length) { if (src.size() <= kMaxInline - inline_length) {
// Append new data to embedded array // Append new data to embedded array
memcpy(data_.as_chars() + inline_length, src.data(), src.size());
set_inline_size(inline_length + src.size()); set_inline_size(inline_length + src.size());
memcpy(data_.as_chars() + inline_length, src.data(), src.size());
return; return;
} }
...@@ -622,9 +620,9 @@ void Cord::PrependArray(absl::string_view src, MethodIdentifier method) { ...@@ -622,9 +620,9 @@ void Cord::PrependArray(absl::string_view src, MethodIdentifier method) {
if (cur_size + src.size() <= InlineRep::kMaxInline) { if (cur_size + src.size() <= InlineRep::kMaxInline) {
// Use embedded storage. // Use embedded storage.
InlineData data; InlineData data;
data.set_inline_size(cur_size + src.size());
memcpy(data.as_chars(), src.data(), src.size()); memcpy(data.as_chars(), src.data(), src.size());
memcpy(data.as_chars() + src.size(), contents_.data(), cur_size); memcpy(data.as_chars() + src.size(), contents_.data(), cur_size);
data.set_inline_size(cur_size + src.size());
contents_.data_ = data; contents_.data_ = data;
return; return;
} }
...@@ -638,8 +636,8 @@ void Cord::AppendPrecise(absl::string_view src, MethodIdentifier method) { ...@@ -638,8 +636,8 @@ void Cord::AppendPrecise(absl::string_view src, MethodIdentifier method) {
assert(src.size() <= cord_internal::kMaxFlatLength); assert(src.size() <= cord_internal::kMaxFlatLength);
if (contents_.remaining_inline_capacity() >= src.size()) { if (contents_.remaining_inline_capacity() >= src.size()) {
const size_t inline_length = contents_.inline_size(); const size_t inline_length = contents_.inline_size();
memcpy(contents_.data_.as_chars() + inline_length, src.data(), src.size());
contents_.set_inline_size(inline_length + src.size()); contents_.set_inline_size(inline_length + src.size());
memcpy(contents_.data_.as_chars() + inline_length, src.data(), src.size());
} else { } else {
contents_.AppendTree(CordRepFlat::Create(src), method); contents_.AppendTree(CordRepFlat::Create(src), method);
} }
...@@ -651,9 +649,9 @@ void Cord::PrependPrecise(absl::string_view src, MethodIdentifier method) { ...@@ -651,9 +649,9 @@ void Cord::PrependPrecise(absl::string_view src, MethodIdentifier method) {
if (contents_.remaining_inline_capacity() >= src.size()) { if (contents_.remaining_inline_capacity() >= src.size()) {
const size_t cur_size = contents_.inline_size(); const size_t cur_size = contents_.inline_size();
InlineData data; InlineData data;
data.set_inline_size(cur_size + src.size());
memcpy(data.as_chars(), src.data(), src.size()); memcpy(data.as_chars(), src.data(), src.size());
memcpy(data.as_chars() + src.size(), contents_.data(), cur_size); memcpy(data.as_chars() + src.size(), contents_.data(), cur_size);
data.set_inline_size(cur_size + src.size());
contents_.data_ = data; contents_.data_ = data;
} else { } else {
contents_.PrependTree(CordRepFlat::Create(src), method); contents_.PrependTree(CordRepFlat::Create(src), method);
...@@ -746,6 +744,7 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { ...@@ -746,6 +744,7 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const {
} }
if (new_size <= InlineRep::kMaxInline) { if (new_size <= InlineRep::kMaxInline) {
sub_cord.contents_.set_inline_size(new_size);
char* dest = sub_cord.contents_.data_.as_chars(); char* dest = sub_cord.contents_.data_.as_chars();
Cord::ChunkIterator it = chunk_begin(); Cord::ChunkIterator it = chunk_begin();
it.AdvanceBytes(pos); it.AdvanceBytes(pos);
...@@ -757,7 +756,6 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { ...@@ -757,7 +756,6 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const {
++it; ++it;
} }
cord_internal::SmallMemmove(dest, it->data(), remaining_size); cord_internal::SmallMemmove(dest, it->data(), remaining_size);
sub_cord.contents_.set_inline_size(new_size);
return sub_cord; return sub_cord;
} }
......
absl/strings/cord.h
...@@ -874,15 +874,14 @@ class Cord { ...@@ -874,15 +874,14 @@ class Cord {
void PrependTreeToTree(CordRep* tree, MethodIdentifier method); void PrependTreeToTree(CordRep* tree, MethodIdentifier method);
void PrependTree(CordRep* tree, MethodIdentifier method); void PrependTree(CordRep* tree, MethodIdentifier method);
bool IsSame(const InlineRep& other) const { bool IsSame(const InlineRep& other) const { return data_ == other.data_; }
return memcmp(&data_, &other.data_, sizeof(data_)) == 0;
}
void CopyTo(std::string* dst) const { void CopyTo(std::string* dst) const {
// memcpy is much faster when operating on a known size. On most supported // memcpy is much faster when operating on a known size. On most supported
// platforms, the small string optimization is large enough that resizing // platforms, the small string optimization is large enough that resizing
// to 15 bytes does not cause a memory allocation. // to 15 bytes does not cause a memory allocation.
absl::strings_internal::STLStringResizeUninitialized(dst, kMaxInline); absl::strings_internal::STLStringResizeUninitialized(dst, kMaxInline);
memcpy(&(*dst)[0], data_.as_chars(), kMaxInline); data_.copy_max_inline_to(&(*dst)[0]);
// erase is faster than resize because the logic for memory allocation is // erase is faster than resize because the logic for memory allocation is
// not needed. // not needed.
dst->erase(inline_size()); dst->erase(inline_size());
...@@ -1023,46 +1022,6 @@ extern std::ostream& operator<<(std::ostream& out, const Cord& cord); ...@@ -1023,46 +1022,6 @@ extern std::ostream& operator<<(std::ostream& out, const Cord& cord);
namespace cord_internal { namespace cord_internal {
// Fast implementation of memmove for up to 15 bytes. This implementation is
// safe for overlapping regions. If nullify_tail is true, the destination is
// padded with '\0' up to 15 bytes.
template <bool nullify_tail = false>
inline void SmallMemmove(char* dst, const char* src, size_t n) {
if (n >= 8) {
assert(n <= 15);
uint64_t buf1;
uint64_t buf2;
memcpy(&buf1, src, 8);
memcpy(&buf2, src + n - 8, 8);
if (nullify_tail) {
memset(dst + 7, 0, 8);
}
memcpy(dst, &buf1, 8);
memcpy(dst + n - 8, &buf2, 8);
} else if (n >= 4) {
uint32_t buf1;
uint32_t buf2;
memcpy(&buf1, src, 4);
memcpy(&buf2, src + n - 4, 4);
if (nullify_tail) {
memset(dst + 4, 0, 4);
memset(dst + 7, 0, 8);
}
memcpy(dst, &buf1, 4);
memcpy(dst + n - 4, &buf2, 4);
} else {
if (n != 0) {
dst[0] = src[0];
dst[n / 2] = src[n / 2];
dst[n - 1] = src[n - 1];
}
if (nullify_tail) {
memset(dst + 7, 0, 8);
memset(dst + n, 0, 8);
}
}
}
// Does non-template-specific `CordRepExternal` initialization. // Does non-template-specific `CordRepExternal` initialization.
// Requires `data` to be non-empty. // Requires `data` to be non-empty.
void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep); void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep);
...@@ -1186,7 +1145,7 @@ inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity( ...@@ -1186,7 +1145,7 @@ inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity(
size_t len = data_.inline_size(); size_t len = data_.inline_size();
auto* result = CordRepFlat::New(len + extra); auto* result = CordRepFlat::New(len + extra);
result->length = len; result->length = len;
memcpy(result->Data(), data_.as_chars(), InlineRep::kMaxInline); data_.copy_max_inline_to(result->Data());
return result; return result;
} }
......
absl/strings/internal/cord_internal.h
...@@ -91,6 +91,46 @@ enum Constants { ...@@ -91,6 +91,46 @@ enum Constants {
// Emits a fatal error "Unexpected node type: xyz" and aborts the program. // Emits a fatal error "Unexpected node type: xyz" and aborts the program.
ABSL_ATTRIBUTE_NORETURN void LogFatalNodeType(CordRep* rep); ABSL_ATTRIBUTE_NORETURN void LogFatalNodeType(CordRep* rep);
// Fast implementation of memmove for up to 15 bytes. This implementation is
// safe for overlapping regions. If nullify_tail is true, the destination is
// padded with '\0' up to 15 bytes.
template <bool nullify_tail = false>
inline void SmallMemmove(char* dst, const char* src, size_t n) {
if (n >= 8) {
assert(n <= 15);
uint64_t buf1;
uint64_t buf2;
memcpy(&buf1, src, 8);
memcpy(&buf2, src + n - 8, 8);
if (nullify_tail) {
memset(dst + 7, 0, 8);
}
memcpy(dst, &buf1, 8);
memcpy(dst + n - 8, &buf2, 8);
} else if (n >= 4) {
uint32_t buf1;
uint32_t buf2;
memcpy(&buf1, src, 4);
memcpy(&buf2, src + n - 4, 4);
if (nullify_tail) {
memset(dst + 4, 0, 4);
memset(dst + 7, 0, 8);
}
memcpy(dst, &buf1, 4);
memcpy(dst + n - 4, &buf2, 4);
} else {
if (n != 0) {
dst[0] = src[0];
dst[n / 2] = src[n / 2];
dst[n - 1] = src[n - 1];
}
if (nullify_tail) {
memset(dst + 7, 0, 8);
memset(dst + n, 0, 8);
}
}
}
// Compact class for tracking the reference count and state flags for CordRep // Compact class for tracking the reference count and state flags for CordRep
// instances. Data is stored in an atomic int32_t for compactness and speed. // instances. Data is stored in an atomic int32_t for compactness and speed.
class RefcountAndFlags { class RefcountAndFlags {
...@@ -479,6 +519,13 @@ class InlineData { ...@@ -479,6 +519,13 @@ class InlineData {
constexpr InlineData(const InlineData& rhs) = default; constexpr InlineData(const InlineData& rhs) = default;
InlineData& operator=(const InlineData& rhs) = default; InlineData& operator=(const InlineData& rhs) = default;
friend bool operator==(const InlineData& lhs, const InlineData& rhs) {
return memcmp(&lhs, &rhs, sizeof(lhs)) == 0;
}
friend bool operator!=(const InlineData& lhs, const InlineData& rhs) {
return !operator==(lhs, rhs);
}
// Returns true if the current instance is empty. // Returns true if the current instance is empty.
// The 'empty value' is an inlined data value of zero length. // The 'empty value' is an inlined data value of zero length.
bool is_empty() const { return rep_.tag() == 0; } bool is_empty() const { return rep_.tag() == 0; }
...@@ -561,6 +608,17 @@ class InlineData { ...@@ -561,6 +608,17 @@ class InlineData {
return rep_.tree(); return rep_.tree();
} }
void set_inline_data(const char* data, size_t n) {
ABSL_ASSERT(n <= kMaxInline);
rep_.set_tag(static_cast<int8_t>(n << 1));
SmallMemmove<true>(rep_.as_chars(), data, n);
}
void copy_max_inline_to(char* dst) const {
assert(!is_tree());
memcpy(dst, as_chars(), kMaxInline);
}
// Initialize this instance to holding the tree value `rep`, // Initialize this instance to holding the tree value `rep`,
// initializing the cordz_info to null, i.e.: 'not profiled'. // initializing the cordz_info to null, i.e.: 'not profiled'.
void make_tree(CordRep* rep) { rep_.make_tree(rep); } void make_tree(CordRep* rep) { rep_.make_tree(rep); }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment