Export of internal Abseil changes

--
007ce045d5d38a727ededdb5bf06e64785fd73bd by Martijn Vels <mvels@google.com>:

Add `cord_enable_btree` feature flag (default false).

PiperOrigin-RevId: 383729939

--
98e7dc6a0407b0fd7b8713d883cdb3a766e0583d by Benjamin Barenblat <bbaren@google.com>:

Eliminate some byte swapping from randen_slow

Stop swapping bytes when serializing randen_slow’s Vector128 into and
out of memory. Instead, simply index different bytes in the AES round
function. This requires byte swapping the te{0..3} lookup tables, but it
produces an 8% speedup on my Xeon W-2135.

PiperOrigin-RevId: 383689402

--
180b6bf45049188840d439b16a28e6b968669340 by Evan Brown <ezb@google.com>:

Minor simplification in drop_deletes_without_resize() - save probe_offset outside the lambda.

Also, add some consts, avoid an auto, and use lambda capture by value instead of reference.

I realized that the compiler can already optimize this - https://godbolt.org/z/Wxd9c4TfK, but I think this way makes the code a bit clearer.

PiperOrigin-RevId: 383646658

--
781706a974c4dc1c0abbb6b801fca0550229e883 by Martijn Vels <mvels@google.com>:

Change storage to contain 3 bytes.

As per the comments in the code, this allows us to utilize all available space in CordRep that may otherwise be 'lost' in padding in derived clases. For the upcoming CordrepBtree class, we want a strong guarantee on having a 64 bytes aligned implementation.

PiperOrigin-RevId: 383633963

--
8fe22ecf92492fa6649938a2215934ebfe01c714 by Derek Mauro <dmauro@google.com>:

Remove reference to str_format_arg.h, which no longer exists

PiperOrigin-RevId: 383517865

--
79397f3b18f18c1e2d7aea993b687329d626ce64 by Benjamin Barenblat <bbaren@google.com>:

Use absl::uint128 for AES random number generator

Replace randen’s internal 128-bit integer struct, u64x2, with
absl::uint128. This eliminates some code and improves support for
big-endian platforms.
PiperOrigin-RevId: 383475671
GitOrigin-RevId: 007ce045d5d38a727ededdb5bf06e64785fd73bd
Change-Id: Ia9d9c40de557221f1744fb0d6d4d6ca7ac569070

absl/container/internal/raw_hash_set.h

@@ -1654,18 +1654,18 @@ class raw_hash_set {
     slot_type* slot = reinterpret_cast<slot_type*>(&raw);
     slot_type* slot = reinterpret_cast<slot_type*>(&raw);
     for (size_t i = 0; i != capacity_; ++i) {
     for (size_t i = 0; i != capacity_; ++i) {
       if (!IsDeleted(ctrl_[i])) continue;
       if (!IsDeleted(ctrl_[i])) continue;
-      size_t hash = PolicyTraits::apply(HashElement{hash_ref()},
-                                        PolicyTraits::element(slots_ + i));
-      auto target = find_first_non_full(ctrl_, hash, capacity_);
-      size_t new_i = target.offset;
+      const size_t hash = PolicyTraits::apply(
+          HashElement{hash_ref()}, PolicyTraits::element(slots_ + i));
+      const FindInfo target = find_first_non_full(ctrl_, hash, capacity_);
+      const size_t new_i = target.offset;
       total_probe_length += target.probe_length;
       total_probe_length += target.probe_length;
 
 
       // Verify if the old and new i fall within the same group wrt the hash.
       // Verify if the old and new i fall within the same group wrt the hash.
       // If they do, we don't need to move the object as it falls already in the
       // If they do, we don't need to move the object as it falls already in the
       // best probe we can.
       // best probe we can.
-      const auto probe_index = [&](size_t pos) {
-        return ((pos - probe(ctrl_, hash, capacity_).offset()) & capacity_) /
-               Group::kWidth;
+      const size_t probe_offset = probe(ctrl_, hash, capacity_).offset();
+      const auto probe_index = [probe_offset, this](size_t pos) {
+        return ((pos - probe_offset) & capacity_) / Group::kWidth;
       };
       };
 
 
       // Element doesn't move.
       // Element doesn't move.

absl/random/internal/BUILD.bazel

@@ -297,6 +297,7 @@ cc_library(
         ":platform",
         ":platform",
         "//absl/base:config",
         "//absl/base:config",
         "//absl/base:core_headers",
         "//absl/base:core_headers",
+        "//absl/numeric:int128",
     ],
     ],
 )
 )
 
 
@@ -337,6 +338,7 @@ cc_library(
         ":platform",
         ":platform",
         "//absl/base:config",
         "//absl/base:config",
         "//absl/base:core_headers",
         "//absl/base:core_headers",
+        "//absl/numeric:int128",
     ],
     ],
 )
 )
 
 

absl/random/internal/randen_hwaes.cc

@@ -23,6 +23,7 @@
 #include <cstring>
 #include <cstring>
 
 
 #include "absl/base/attributes.h"
 #include "absl/base/attributes.h"
+#include "absl/numeric/int128.h"
 #include "absl/random/internal/platform.h"
 #include "absl/random/internal/platform.h"
 #include "absl/random/internal/randen_traits.h"
 #include "absl/random/internal/randen_traits.h"
 
 
@@ -120,11 +121,6 @@ namespace {
 
 
 using absl::random_internal::RandenTraits;
 using absl::random_internal::RandenTraits;
 
 
-// Randen operates on 128-bit vectors.
-struct alignas(16) u64x2 {
-  uint64_t data[2];
-};
-
 }  // namespace
 }  // namespace
 
 
 // TARGET_CRYPTO defines a crypto attribute for each architecture.
 // TARGET_CRYPTO defines a crypto attribute for each architecture.
@@ -186,7 +182,7 @@ inline ABSL_TARGET_CRYPTO Vector128 AesRound(const Vector128& state,
 }
 }
 
 
 // Enables native loads in the round loop by pre-swapping.
 // Enables native loads in the round loop by pre-swapping.
-inline ABSL_TARGET_CRYPTO void SwapEndian(u64x2* state) {
+inline ABSL_TARGET_CRYPTO void SwapEndian(absl::uint128* state) {
   for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) {
   for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) {
     Vector128Store(ReverseBytes(Vector128Load(state + block)), state + block);
     Vector128Store(ReverseBytes(Vector128Load(state + block)), state + block);
   }
   }
@@ -327,7 +323,7 @@ namespace {
 
 
 // Block shuffles applies a shuffle to the entire state between AES rounds.
 // Block shuffles applies a shuffle to the entire state between AES rounds.
 // Improved odd-even shuffle from "New criterion for diffusion property".
 // Improved odd-even shuffle from "New criterion for diffusion property".
-inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) {
+inline ABSL_TARGET_CRYPTO void BlockShuffle(absl::uint128* state) {
   static_assert(RandenTraits::kFeistelBlocks == 16,
   static_assert(RandenTraits::kFeistelBlocks == 16,
                 "Expecting 16 FeistelBlocks.");
                 "Expecting 16 FeistelBlocks.");
 
 
@@ -374,8 +370,9 @@ inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) {
 // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in
 // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in
 // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel
 // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel
 // XORs are 'free' (included in the second AES instruction).
 // XORs are 'free' (included in the second AES instruction).
-inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound(
-    u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) {
+inline ABSL_TARGET_CRYPTO const absl::uint128* FeistelRound(
+    absl::uint128* state,
+    const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) {
   static_assert(RandenTraits::kFeistelBlocks == 16,
   static_assert(RandenTraits::kFeistelBlocks == 16,
                 "Expecting 16 FeistelBlocks.");
                 "Expecting 16 FeistelBlocks.");
 
 
@@ -436,7 +433,8 @@ inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound(
 // 2^64 queries if the round function is a PRF. This is similar to the b=8 case
 // 2^64 queries if the round function is a PRF. This is similar to the b=8 case
 // of Simpira v2, but more efficient than its generic construction for b=16.
 // of Simpira v2, but more efficient than its generic construction for b=16.
 inline ABSL_TARGET_CRYPTO void Permute(
 inline ABSL_TARGET_CRYPTO void Permute(
-    u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) {
+    absl::uint128* state,
+    const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) {
   // (Successfully unrolled; the first iteration jumps into the second half)
   // (Successfully unrolled; the first iteration jumps into the second half)
 #ifdef __clang__
 #ifdef __clang__
 #pragma clang loop unroll_count(2)
 #pragma clang loop unroll_count(2)
@@ -473,10 +471,11 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Absorb(const void* seed_void,
   static_assert(RandenTraits::kStateBytes / sizeof(Vector128) == 16,
   static_assert(RandenTraits::kStateBytes / sizeof(Vector128) == 16,
                 "Unexpected Randen kStateBlocks");
                 "Unexpected Randen kStateBlocks");
 
 
-  auto* state =
-      reinterpret_cast<u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(state_void);
+  auto* state = reinterpret_cast<absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>(
+      state_void);
   const auto* seed =
   const auto* seed =
-      reinterpret_cast<const u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(seed_void);
+      reinterpret_cast<const absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>(
+          seed_void);
 
 
   Vector128 b1 = Vector128Load(state + 1);
   Vector128 b1 = Vector128Load(state + 1);
   b1 ^= Vector128Load(seed + 0);
   b1 ^= Vector128Load(seed + 0);
@@ -545,8 +544,8 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Generate(const void* keys_void,
   static_assert(RandenTraits::kCapacityBytes == sizeof(Vector128),
   static_assert(RandenTraits::kCapacityBytes == sizeof(Vector128),
                 "Capacity mismatch");
                 "Capacity mismatch");
 
 
-  auto* state = reinterpret_cast<u64x2*>(state_void);
-  const auto* keys = reinterpret_cast<const u64x2*>(keys_void);
+  auto* state = reinterpret_cast<absl::uint128*>(state_void);
+  const auto* keys = reinterpret_cast<const absl::uint128*>(keys_void);
 
 
   const Vector128 prev_inner = Vector128Load(state);
   const Vector128 prev_inner = Vector128Load(state);
 
 

absl/strings/cord_ring_test.cc

@@ -223,7 +223,7 @@ CordRepExternal* MakeFakeExternal(size_t length) {
     std::string s;
     std::string s;
     explicit Rep(size_t len) {
     explicit Rep(size_t len) {
       this->tag = EXTERNAL;
       this->tag = EXTERNAL;
-      this->base = this->storage;
+      this->base = reinterpret_cast<const char*>(this->storage);
       this->length = len;
       this->length = len;
       this->releaser_invoker = [](CordRepExternal* self) {
       this->releaser_invoker = [](CordRepExternal* self) {
         delete static_cast<Rep*>(self);
         delete static_cast<Rep*>(self);

absl/strings/internal/cord_internal.cc

@@ -25,6 +25,7 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 ABSL_NAMESPACE_BEGIN
 namespace cord_internal {
 namespace cord_internal {
 
 
+ABSL_CONST_INIT std::atomic<bool> cord_btree_enabled(kCordEnableBtreeDefault);
 ABSL_CONST_INIT std::atomic<bool> cord_ring_buffer_enabled(
 ABSL_CONST_INIT std::atomic<bool> cord_ring_buffer_enabled(
     kCordEnableRingBufferDefault);
     kCordEnableRingBufferDefault);
 ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled(
 ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled(

absl/strings/internal/cord_internal.h

@@ -37,13 +37,19 @@ class CordzInfo;
 
 
 // Default feature enable states for cord ring buffers
 // Default feature enable states for cord ring buffers
 enum CordFeatureDefaults {
 enum CordFeatureDefaults {
+  kCordEnableBtreeDefault = false,
   kCordEnableRingBufferDefault = false,
   kCordEnableRingBufferDefault = false,
   kCordShallowSubcordsDefault = false
   kCordShallowSubcordsDefault = false
 };
 };
 
 
+extern std::atomic<bool> cord_btree_enabled;
 extern std::atomic<bool> cord_ring_buffer_enabled;
 extern std::atomic<bool> cord_ring_buffer_enabled;
 extern std::atomic<bool> shallow_subcords_enabled;
 extern std::atomic<bool> shallow_subcords_enabled;
 
 
+inline void enable_cord_btree(bool enable) {
+  cord_btree_enabled.store(enable, std::memory_order_relaxed);
+}
+
 inline void enable_cord_ring_buffer(bool enable) {
 inline void enable_cord_ring_buffer(bool enable) {
   cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed);
   cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed);
 }
 }
@@ -193,7 +199,16 @@ struct CordRep {
   // If tag < FLAT, it represents CordRepKind and indicates the type of node.
   // If tag < FLAT, it represents CordRepKind and indicates the type of node.
   // Otherwise, the node type is CordRepFlat and the tag is the encoded size.
   // Otherwise, the node type is CordRepFlat and the tag is the encoded size.
   uint8_t tag;
   uint8_t tag;
-  char storage[1];  // Starting point for flat array: MUST BE LAST FIELD
+
+  // `storage` provides two main purposes:
+  // - the starting point for FlatCordRep.Data() [flexible-array-member]
+  // - 3 bytes of additional storage for use by derived classes.
+  // The latter is used by CordrepConcat and CordRepBtree. CordRepConcat stores
+  // a 'depth' value in storage[0], and the (future) CordRepBtree class stores
+  // `height`, `begin` and `end` in the 3 entries. Otherwise we would need to
+  // allocate room for these in the derived class, as not all compilers reuse
+  // padding space from the base class (clang and gcc do, MSVC does not, etc)
+  uint8_t storage[3];
 
 
   inline CordRepRing* ring();
   inline CordRepRing* ring();
   inline const CordRepRing* ring() const;
   inline const CordRepRing* ring() const;
@@ -228,8 +243,8 @@ struct CordRepConcat : public CordRep {
   CordRep* left;
   CordRep* left;
   CordRep* right;
   CordRep* right;
 
 
-  uint8_t depth() const { return static_cast<uint8_t>(storage[0]); }
-  void set_depth(uint8_t depth) { storage[0] = static_cast<char>(depth); }
+  uint8_t depth() const { return storage[0]; }
+  void set_depth(uint8_t depth) { storage[0] = depth; }
 };
 };
 
 
 struct CordRepSubstring : public CordRep {
 struct CordRepSubstring : public CordRep {

absl/strings/internal/cord_rep_flat.h

@@ -118,8 +118,8 @@ struct CordRepFlat : public CordRep {
   }
   }
 
 
   // Returns a pointer to the data inside this flat rep.
   // Returns a pointer to the data inside this flat rep.
-  char* Data() { return storage; }
-  const char* Data() const { return storage; }
+  char* Data() { return reinterpret_cast<char*>(storage); }
+  const char* Data() const { return reinterpret_cast<const char*>(storage); }
 
 
   // Returns the maximum capacity (payload size) of this instance.
   // Returns the maximum capacity (payload size) of this instance.
   size_t Capacity() const { return TagToLength(tag); }
   size_t Capacity() const { return TagToLength(tag); }

absl/strings/str_format.h

@@ -536,8 +536,7 @@ using FormatArg = str_format_internal::FormatArgImpl;
 // The arguments are provided in an `absl::Span<const absl::FormatArg>`.
 // The arguments are provided in an `absl::Span<const absl::FormatArg>`.
 // Each `absl::FormatArg` object binds to a single argument and keeps a
 // Each `absl::FormatArg` object binds to a single argument and keeps a
 // reference to it. The values used to create the `FormatArg` objects must
 // reference to it. The values used to create the `FormatArg` objects must
-// outlive this function call. (See `str_format_arg.h` for information on
-// the `FormatArg` class.)_
+// outlive this function call.
 //
 //
 // Example:
 // Example:
 //
 //