Export of internal Abseil changes

--
a0491c8d790972cd80e2d720fe1fdf5f711a6f1a by Greg Falcon <gfalcon@google.com>:

Stop directly accessing CordRepFlat data via CordRep::data.

The old pattern of access breaks the `CordRep` type abstraction; since `CordRep::data` is not in general guaranteed to contain the chunk's data, we shouldn't access it that way.

This incidentally adds an assertion check (via the flat() accessor) that the CordRep is indeed flat on each such access, but a manual inspection of the code, as well as the fact that this code currently works, suggest that this is always true.)

PiperOrigin-RevId: 351592344

--
f40c3b43ca5b1d7e23cd45f1ffac1783105ac1a3 by Abseil Team <absl-team@google.com>:

Revert 18abb2902b9f06c63a968b24d3dda785ebf99a22

PiperOrigin-RevId: 351523518

--
18abb2902b9f06c63a968b24d3dda785ebf99a22 by Abseil Team <absl-team@google.com>:

Internal change

PiperOrigin-RevId: 351512412

--
9b881602d45e95e06089792c7627cd56528a255a by Abseil Team <absl-team@google.com>:

Keep time's global state in a cacheline-aligned structure.

Keeping the global state as separate global variables results in two issues:
1) False sharing with adjacent global data (e.g., cycle clock source), since
   the global fields are updated every O(10usec).
2) The hot global fields (e.g., seq and samples) can reside on different
   cache lines.

To fix this, simply wrap the global data in a ABSL_CACHE_ALIGNED structure.
This is similar to what we do for MutexGlobals.

PiperOrigin-RevId: 351389466
GitOrigin-RevId: a0491c8d790972cd80e2d720fe1fdf5f711a6f1a
Change-Id: Ie0fa80112043381cd37c84e2ab2b7334839f54b5

absl/strings/cord.cc

@@ -208,9 +208,9 @@ static CordRep* NewTree(const char* data,
   size_t n = 0;
   size_t n = 0;
   do {
   do {
     const size_t len = std::min(length, kMaxFlatLength);
     const size_t len = std::min(length, kMaxFlatLength);
-    CordRep* rep = CordRepFlat::New(len + alloc_hint);
+    CordRepFlat* rep = CordRepFlat::New(len + alloc_hint);
     rep->length = len;
     rep->length = len;
-    memcpy(rep->data, data, len);
+    memcpy(rep->Data(), data, len);
     reps[n++] = VerifyTree(rep);
     reps[n++] = VerifyTree(rep);
     data += len;
     data += len;
     length -= len;
     length -= len;
@@ -272,10 +272,10 @@ inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) {
     return data_.as_tree.rep;
     return data_.as_tree.rep;
   }
   }
 
 
-  CordRep* result = CordRepFlat::New(len + extra_hint);
+  CordRepFlat* result = CordRepFlat::New(len + extra_hint);
   result->length = len;
   result->length = len;
   static_assert(kMinFlatLength >= sizeof(data_.as_chars), "");
   static_assert(kMinFlatLength >= sizeof(data_.as_chars), "");
-  memcpy(result->data, data_.as_chars, sizeof(data_.as_chars));
+  memcpy(result->Data(), data_.as_chars, sizeof(data_.as_chars));
   set_tree(result);
   set_tree(result);
   return result;
   return result;
 }
 }
@@ -349,7 +349,7 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region,
   }
   }
   dst->length += size_increase;
   dst->length += size_increase;
 
 
-  *region = dst->data + in_use;
+  *region = dst->flat()->Data() + in_use;
   *size = size_increase;
   *size = size_increase;
   return true;
   return true;
 }
 }
@@ -381,7 +381,7 @@ void Cord::InlineRep::GetAppendRegion(char** region, size_t* size,
   CordRepFlat* new_node =
   CordRepFlat* new_node =
       CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length));
       CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length));
   new_node->length = std::min(new_node->Capacity(), max_length);
   new_node->length = std::min(new_node->Capacity(), max_length);
-  *region = new_node->data;
+  *region = new_node->Data();
   *size = new_node->length;
   *size = new_node->length;
   replace_tree(Concat(root, new_node));
   replace_tree(Concat(root, new_node));
 }
 }
@@ -407,7 +407,7 @@ void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) {
   // Allocate new node.
   // Allocate new node.
   CordRepFlat* new_node = CordRepFlat::New(root->length);
   CordRepFlat* new_node = CordRepFlat::New(root->length);
   new_node->length = new_node->Capacity();
   new_node->length = new_node->Capacity();
-  *region = new_node->data;
+  *region = new_node->Data();
   *size = new_node->length;
   *size = new_node->length;
   replace_tree(Concat(root, new_node));
   replace_tree(Concat(root, new_node));
 }
 }
@@ -523,7 +523,7 @@ Cord& Cord::operator=(absl::string_view src) {
       tree->flat()->Capacity() >= length &&
       tree->flat()->Capacity() >= length &&
       tree->refcount.IsOne()) {
       tree->refcount.IsOne()) {
     // Copy in place if the existing FLAT node is reusable.
     // Copy in place if the existing FLAT node is reusable.
-    memmove(tree->data, data, length);
+    memmove(tree->flat()->Data(), data, length);
     tree->length = length;
     tree->length = length;
     VerifyTree(tree);
     VerifyTree(tree);
     return *this;
     return *this;
@@ -578,8 +578,8 @@ void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) {
     root = CordRepFlat::New(std::max<size_t>(size1, size2));
     root = CordRepFlat::New(std::max<size_t>(size1, size2));
     appended = std::min(
     appended = std::min(
         src_size, root->flat()->Capacity() - inline_length);
         src_size, root->flat()->Capacity() - inline_length);
-    memcpy(root->data, data_.as_chars, inline_length);
-    memcpy(root->data + inline_length, src_data, appended);
+    memcpy(root->flat()->Data(), data_.as_chars, inline_length);
+    memcpy(root->flat()->Data() + inline_length, src_data, appended);
     root->length = inline_length + appended;
     root->length = inline_length + appended;
     set_tree(root);
     set_tree(root);
   }
   }
@@ -635,7 +635,7 @@ inline void Cord::AppendImpl(C&& src) {
     }
     }
     if (src_tree->tag >= FLAT) {
     if (src_tree->tag >= FLAT) {
       // src tree just has one flat node.
       // src tree just has one flat node.
-      contents_.AppendArray(src_tree->data, src_size);
+      contents_.AppendArray(src_tree->flat()->Data(), src_size);
       return;
       return;
     }
     }
     if (&src == this) {
     if (&src == this) {
@@ -1093,7 +1093,7 @@ inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const {
 
 
   CordRep* node = tree();
   CordRep* node = tree();
   if (node->tag >= FLAT) {
   if (node->tag >= FLAT) {
-    return absl::string_view(node->data, node->length);
+    return absl::string_view(node->flat()->Data(), node->length);
   }
   }
 
 
   if (node->tag == EXTERNAL) {
   if (node->tag == EXTERNAL) {
@@ -1116,7 +1116,7 @@ inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const {
   }
   }
 
 
   if (node->tag >= FLAT) {
   if (node->tag >= FLAT) {
-    return absl::string_view(node->data + offset, length);
+    return absl::string_view(node->flat()->Data() + offset, length);
   }
   }
 
 
   assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here");
   assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here");
@@ -1329,7 +1329,7 @@ Cord::ChunkIterator& Cord::ChunkIterator::AdvanceStack() {
   assert(node->tag == EXTERNAL || node->tag >= FLAT);
   assert(node->tag == EXTERNAL || node->tag >= FLAT);
   assert(length != 0);
   assert(length != 0);
   const char* data =
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->data;
+      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset, length);
   current_chunk_ = absl::string_view(data + offset, length);
   current_leaf_ = node;
   current_leaf_ = node;
   return *this;
   return *this;
@@ -1362,8 +1362,8 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
     // Range to read is a proper subrange of the current chunk.
     // Range to read is a proper subrange of the current chunk.
     assert(current_leaf_ != nullptr);
     assert(current_leaf_ != nullptr);
     CordRep* subnode = CordRep::Ref(current_leaf_);
     CordRep* subnode = CordRep::Ref(current_leaf_);
-    const char* data =
-        subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
+    const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
+                                                : subnode->flat()->Data();
     subnode = NewSubstring(subnode, current_chunk_.data() - data, n);
     subnode = NewSubstring(subnode, current_chunk_.data() - data, n);
     subcord.contents_.set_tree(VerifyTree(subnode));
     subcord.contents_.set_tree(VerifyTree(subnode));
     RemoveChunkPrefix(n);
     RemoveChunkPrefix(n);
@@ -1375,8 +1375,8 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
   assert(current_leaf_ != nullptr);
   assert(current_leaf_ != nullptr);
   CordRep* subnode = CordRep::Ref(current_leaf_);
   CordRep* subnode = CordRep::Ref(current_leaf_);
   if (current_chunk_.size() < subnode->length) {
   if (current_chunk_.size() < subnode->length) {
-    const char* data =
-        subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
+    const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
+                                                : subnode->flat()->Data();
     subnode = NewSubstring(subnode, current_chunk_.data() - data,
     subnode = NewSubstring(subnode, current_chunk_.data() - data,
                            current_chunk_.size());
                            current_chunk_.size());
   }
   }
@@ -1444,7 +1444,7 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) {
     subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n));
     subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n));
   }
   }
   const char* data =
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->data;
+      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_leaf_ = node;
   current_leaf_ = node;
   bytes_remaining_ -= n;
   bytes_remaining_ -= n;
@@ -1511,7 +1511,7 @@ void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) {
   assert(node->tag == EXTERNAL || node->tag >= FLAT);
   assert(node->tag == EXTERNAL || node->tag >= FLAT);
   assert(length > n);
   assert(length > n);
   const char* data =
   const char* data =
-      node->tag == EXTERNAL ? node->external()->base : node->data;
+      node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_chunk_ = absl::string_view(data + offset + n, length - n);
   current_leaf_ = node;
   current_leaf_ = node;
   bytes_remaining_ -= n;
   bytes_remaining_ -= n;
@@ -1529,7 +1529,7 @@ char Cord::operator[](size_t i) const {
     assert(offset < rep->length);
     assert(offset < rep->length);
     if (rep->tag >= FLAT) {
     if (rep->tag >= FLAT) {
       // Get the "i"th character directly from the flat array.
       // Get the "i"th character directly from the flat array.
-      return rep->data[offset];
+      return rep->flat()->Data()[offset];
     } else if (rep->tag == EXTERNAL) {
     } else if (rep->tag == EXTERNAL) {
       // Get the "i"th character from the external array.
       // Get the "i"th character from the external array.
       return rep->external()->base[offset];
       return rep->external()->base[offset];
@@ -1562,7 +1562,7 @@ absl::string_view Cord::FlattenSlowPath() {
   if (total_size <= kMaxFlatLength) {
   if (total_size <= kMaxFlatLength) {
     new_rep = CordRepFlat::New(total_size);
     new_rep = CordRepFlat::New(total_size);
     new_rep->length = total_size;
     new_rep->length = total_size;
-    new_buffer = new_rep->data;
+    new_buffer = new_rep->flat()->Data();
     CopyToArraySlowPath(new_buffer);
     CopyToArraySlowPath(new_buffer);
   } else {
   } else {
     new_buffer = std::allocator<char>().allocate(total_size);
     new_buffer = std::allocator<char>().allocate(total_size);
@@ -1583,7 +1583,7 @@ absl::string_view Cord::FlattenSlowPath() {
 /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) {
 /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) {
   assert(rep != nullptr);
   assert(rep != nullptr);
   if (rep->tag >= FLAT) {
   if (rep->tag >= FLAT) {
-    *fragment = absl::string_view(rep->data, rep->length);
+    *fragment = absl::string_view(rep->flat()->Data(), rep->length);
     return true;
     return true;
   } else if (rep->tag == EXTERNAL) {
   } else if (rep->tag == EXTERNAL) {
     *fragment = absl::string_view(rep->external()->base, rep->length);
     *fragment = absl::string_view(rep->external()->base, rep->length);
@@ -1591,8 +1591,8 @@ absl::string_view Cord::FlattenSlowPath() {
   } else if (rep->tag == SUBSTRING) {
   } else if (rep->tag == SUBSTRING) {
     CordRep* child = rep->substring()->child;
     CordRep* child = rep->substring()->child;
     if (child->tag >= FLAT) {
     if (child->tag >= FLAT) {
-      *fragment =
-          absl::string_view(child->data + rep->substring()->start, rep->length);
+      *fragment = absl::string_view(
+          child->flat()->Data() + rep->substring()->start, rep->length);
       return true;
       return true;
     } else if (child->tag == EXTERNAL) {
     } else if (child->tag == EXTERNAL) {
       *fragment = absl::string_view(
       *fragment = absl::string_view(
@@ -1680,7 +1680,7 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os) {
         *os << "FLAT cap=" << rep->flat()->Capacity()
         *os << "FLAT cap=" << rep->flat()->Capacity()
             << " [";
             << " [";
         if (include_data)
         if (include_data)
-          *os << absl::CEscape(std::string(rep->data, rep->length));
+          *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length));
         *os << "]\n";
         *os << "]\n";
       }
       }
       if (stack.empty()) break;
       if (stack.empty()) break;

absl/strings/internal/cord_internal.h

@@ -166,7 +166,7 @@ enum CordRepKind {
 struct CordRep {
 struct CordRep {
   CordRep() = default;
   CordRep() = default;
   constexpr CordRep(Refcount::Immortal immortal, size_t l)
   constexpr CordRep(Refcount::Immortal immortal, size_t l)
-      : length(l), refcount(immortal), tag(EXTERNAL), data{} {}
+      : length(l), refcount(immortal), tag(EXTERNAL), storage{} {}
 
 
   // The following three fields have to be less than 32 bytes since
   // The following three fields have to be less than 32 bytes since
   // that is the smallest supported flat node size.
   // that is the smallest supported flat node size.
@@ -175,7 +175,7 @@ struct CordRep {
   // If tag < FLAT, it represents CordRepKind and indicates the type of node.
   // If tag < FLAT, it represents CordRepKind and indicates the type of node.
   // Otherwise, the node type is CordRepFlat and the tag is the encoded size.
   // Otherwise, the node type is CordRepFlat and the tag is the encoded size.
   uint8_t tag;
   uint8_t tag;
-  char data[1];  // Starting point for flat array: MUST BE LAST FIELD of CordRep
+  char storage[1];  // Starting point for flat array: MUST BE LAST FIELD
 
 
   inline CordRepConcat* concat();
   inline CordRepConcat* concat();
   inline const CordRepConcat* concat() const;
   inline const CordRepConcat* concat() const;
@@ -219,8 +219,8 @@ struct CordRepConcat : public CordRep {
   CordRep* left;
   CordRep* left;
   CordRep* right;
   CordRep* right;
 
 
-  uint8_t depth() const { return static_cast<uint8_t>(data[0]); }
-  void set_depth(uint8_t depth) { data[0] = static_cast<char>(depth); }
+  uint8_t depth() const { return static_cast<uint8_t>(storage[0]); }
+  void set_depth(uint8_t depth) { storage[0] = static_cast<char>(depth); }
 };
 };
 
 
 struct CordRepSubstring : public CordRep {
 struct CordRepSubstring : public CordRep {

absl/strings/internal/cord_rep_flat.h

@@ -37,7 +37,7 @@ namespace cord_internal {
 // ideally a 'nice' size aligning with allocation and cacheline sizes like 32.
 // ideally a 'nice' size aligning with allocation and cacheline sizes like 32.
 // kMaxFlatSize is bounded by the size resulting in a computed tag no greater
 // kMaxFlatSize is bounded by the size resulting in a computed tag no greater
 // than MAX_FLAT_TAG. MAX_FLAT_TAG provides for additional 'high' tag values.
 // than MAX_FLAT_TAG. MAX_FLAT_TAG provides for additional 'high' tag values.
-static constexpr size_t kFlatOverhead = offsetof(CordRep, data);
+static constexpr size_t kFlatOverhead = offsetof(CordRep, storage);
 static constexpr size_t kMinFlatSize = 32;
 static constexpr size_t kMinFlatSize = 32;
 static constexpr size_t kMaxFlatSize = 4096;
 static constexpr size_t kMaxFlatSize = 4096;
 static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead;
 static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead;
@@ -115,6 +115,9 @@ struct CordRepFlat : public CordRep {
 #endif
 #endif
   }
   }
 
 
+  char* Data() { return storage; }
+  const char* Data() const { return storage; }
+
   // Returns the maximum capacity (payload size) of this instance.
   // Returns the maximum capacity (payload size) of this instance.
   size_t Capacity() const { return TagToLength(tag); }
   size_t Capacity() const { return TagToLength(tag); }