Move growth_left to the backing array.

PiperOrigin-RevId: 548794485
Change-Id: Ie82d5f8ad752518ef05b38144ca1e32b21c9def8

absl/container/internal/raw_hash_set.cc

@@ -19,6 +19,7 @@
 #include <cstddef>
 #include <cstddef>
 #include <cstring>
 #include <cstring>
 
 
+#include "absl/base/attributes.h"
 #include "absl/base/config.h"
 #include "absl/base/config.h"
 #include "absl/base/dynamic_annotations.h"
 #include "absl/base/dynamic_annotations.h"
 #include "absl/hash/hash.h"
 #include "absl/hash/hash.h"
@@ -27,9 +28,17 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 ABSL_NAMESPACE_BEGIN
 namespace container_internal {
 namespace container_internal {
 
 
-// A single block of empty control bytes for tables without any slots allocated.
-// This enables removing a branch in the hot path of find().
-alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[16] = {
+// We have space for `growth_left` before a single block of control bytes. A
+// single block of empty control bytes for tables without any slots allocated.
+// This enables removing a branch in the hot path of find(). In order to ensure
+// that the control bytes are aligned to 16, we have 16 bytes before the control
+// bytes even though growth_left only needs 8.
+constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); }
+alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = {
+    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
+    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
+    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
+    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
     ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
     ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
     ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
     ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
     ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
     ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
@@ -239,12 +248,12 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
     c.infoz().RecordStorageChanged(0, c.capacity());
     c.infoz().RecordStorageChanged(0, c.capacity());
   } else {
   } else {
     void* set = &c;
     void* set = &c;
-    (*policy.dealloc)(set, policy, c.control(), c.slots_ptr(), c.capacity());
+    (*policy.dealloc)(set, policy, c.backing_array_start(), c.slots_ptr(),
+                      c.capacity());
     c.set_control(EmptyGroup());
     c.set_control(EmptyGroup());
     c.set_generation_ptr(EmptyGeneration());
     c.set_generation_ptr(EmptyGeneration());
     c.set_slots(nullptr);
     c.set_slots(nullptr);
     c.set_capacity(0);
     c.set_capacity(0);
-    c.set_growth_left(0);
     c.infoz().RecordClearedReservation();
     c.infoz().RecordClearedReservation();
     assert(c.size() == 0);
     assert(c.size() == 0);
     c.infoz().RecordStorageChanged(0, 0);
     c.infoz().RecordStorageChanged(0, 0);

absl/container/internal/raw_hash_set_test.cc

@@ -17,6 +17,7 @@
 #include <algorithm>
 #include <algorithm>
 #include <atomic>
 #include <atomic>
 #include <cmath>
 #include <cmath>
+#include <cstddef>
 #include <cstdint>
 #include <cstdint>
 #include <deque>
 #include <deque>
 #include <functional>
 #include <functional>
@@ -436,6 +437,41 @@ struct CustomAllocIntTable
   using Base::Base;
   using Base::Base;
 };
 };
 
 
+// Tries to allocate memory at the minimum alignment even when the default
+// allocator uses a higher alignment.
+template <typename T>
+struct MinimumAlignmentAlloc : std::allocator<T> {
+  MinimumAlignmentAlloc() = default;
+
+  template <typename U>
+  explicit MinimumAlignmentAlloc(const MinimumAlignmentAlloc<U>& /*other*/) {}
+
+  template <class U>
+  struct rebind {
+    using other = MinimumAlignmentAlloc<U>;
+  };
+
+  T* allocate(size_t n) {
+    T* ptr = std::allocator<T>::allocate(n + 1);
+    char* cptr = reinterpret_cast<char*>(ptr);
+    cptr += alignof(T);
+    return reinterpret_cast<T*>(cptr);
+  }
+
+  void deallocate(T* ptr, size_t n) {
+    char* cptr = reinterpret_cast<char*>(ptr);
+    cptr -= alignof(T);
+    std::allocator<T>::deallocate(reinterpret_cast<T*>(cptr), n + 1);
+  }
+};
+
+struct MinimumAlignmentUint8Table
+    : raw_hash_set<Uint8Policy, container_internal::hash_default_hash<uint8_t>,
+                   std::equal_to<uint8_t>, MinimumAlignmentAlloc<uint8_t>> {
+  using Base = typename MinimumAlignmentUint8Table::raw_hash_set;
+  using Base::Base;
+};
+
 struct BadFastHash {
 struct BadFastHash {
   template <class T>
   template <class T>
   size_t operator()(const T&) const {
   size_t operator()(const T&) const {
@@ -460,14 +496,12 @@ TEST(Table, EmptyFunctorOptimization) {
     void* slots;
     void* slots;
     size_t size;
     size_t size;
     size_t capacity;
     size_t capacity;
-    size_t growth_left;
   };
   };
   struct MockTableInfozDisabled {
   struct MockTableInfozDisabled {
     void* ctrl;
     void* ctrl;
     void* slots;
     void* slots;
     size_t size;
     size_t size;
     size_t capacity;
     size_t capacity;
-    size_t growth_left;
   };
   };
   struct StatelessHash {
   struct StatelessHash {
     size_t operator()(absl::string_view) const { return 0; }
     size_t operator()(absl::string_view) const { return 0; }
@@ -2247,11 +2281,17 @@ TEST(Sanitizer, PoisoningOnErase) {
 }
 }
 #endif  // ABSL_HAVE_ADDRESS_SANITIZER
 #endif  // ABSL_HAVE_ADDRESS_SANITIZER
 
 
-TEST(Table, AlignOne) {
+template <typename T>
+class AlignOneTest : public ::testing::Test {};
+using AlignOneTestTypes =
+    ::testing::Types<Uint8Table, MinimumAlignmentUint8Table>;
+TYPED_TEST_SUITE(AlignOneTest, AlignOneTestTypes);
+
+TYPED_TEST(AlignOneTest, AlignOne) {
   // We previously had a bug in which we were copying a control byte over the
   // We previously had a bug in which we were copying a control byte over the
   // first slot when alignof(value_type) is 1. We test repeated
   // first slot when alignof(value_type) is 1. We test repeated
   // insertions/erases and verify that the behavior is correct.
   // insertions/erases and verify that the behavior is correct.
-  Uint8Table t;
+  TypeParam t;
   std::unordered_set<uint8_t> verifier;  // NOLINT
   std::unordered_set<uint8_t> verifier;  // NOLINT
 
 
   // Do repeated insertions/erases from the table.
   // Do repeated insertions/erases from the table.