Commit 52715dbd by Vitaly Goldshteyn Committed by Copybara-Service

Use GrowthInfo without applying any optimizations based on it.

PiperOrigin-RevId: 619649335
Change-Id: I8b3380816418a363fb6686db7966248cb530c491
parent ff0a0f2d
...@@ -33,11 +33,11 @@ namespace container_internal { ...@@ -33,11 +33,11 @@ namespace container_internal {
// Represents a control byte corresponding to a full slot with arbitrary hash. // Represents a control byte corresponding to a full slot with arbitrary hash.
constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); } constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); }
// We have space for `growth_left` before a single block of control bytes. A // We have space for `growth_info` before a single block of control bytes. A
// single block of empty control bytes for tables without any slots allocated. // single block of empty control bytes for tables without any slots allocated.
// This enables removing a branch in the hot path of find(). In order to ensure // This enables removing a branch in the hot path of find(). In order to ensure
// that the control bytes are aligned to 16, we have 16 bytes before the control // that the control bytes are aligned to 16, we have 16 bytes before the control
// bytes even though growth_left only needs 8. // bytes even though growth_info only needs 8.
alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = { alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = {
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(), ZeroCtrlT(),
...@@ -133,7 +133,7 @@ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size, ...@@ -133,7 +133,7 @@ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size,
assert(HashSetResizeHelper::SooSlotIndex() == 1); assert(HashSetResizeHelper::SooSlotIndex() == 1);
PrepareInsertCommon(common); PrepareInsertCommon(common);
const size_t offset = H1(hash, common.control()) & 2; const size_t offset = H1(hash, common.control()) & 2;
common.set_growth_left(common.growth_left() - 1); common.growth_info().OverwriteEmptyAsFull();
SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size); SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size);
common.infoz().RecordInsert(hash, /*distance_from_desired=*/0); common.infoz().RecordInsert(hash, /*distance_from_desired=*/0);
return offset; return offset;
...@@ -274,10 +274,11 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) { ...@@ -274,10 +274,11 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {
if (WasNeverFull(c, index)) { if (WasNeverFull(c, index)) {
SetCtrl(c, index, ctrl_t::kEmpty, slot_size); SetCtrl(c, index, ctrl_t::kEmpty, slot_size);
c.set_growth_left(c.growth_left() + 1); c.growth_info().OverwriteFullAsEmpty();
return; return;
} }
c.growth_info().OverwriteFullAsDeleted();
SetCtrl(c, index, ctrl_t::kDeleted, slot_size); SetCtrl(c, index, ctrl_t::kDeleted, slot_size);
} }
......
...@@ -1154,9 +1154,9 @@ constexpr size_t NumControlBytes(size_t capacity) { ...@@ -1154,9 +1154,9 @@ constexpr size_t NumControlBytes(size_t capacity) {
} }
// Computes the offset from the start of the backing allocation of control. // Computes the offset from the start of the backing allocation of control.
// infoz and growth_left are stored at the beginning of the backing array. // infoz and growth_info are stored at the beginning of the backing array.
inline static size_t ControlOffset(bool has_infoz) { inline static size_t ControlOffset(bool has_infoz) {
return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t); return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo);
} }
// Helper class for computing offsets and allocation size of hash set fields. // Helper class for computing offsets and allocation size of hash set fields.
...@@ -1249,7 +1249,7 @@ struct HeapPtrs { ...@@ -1249,7 +1249,7 @@ struct HeapPtrs {
// This contains `capacity + 1 + NumClonedBytes()` entries, even // This contains `capacity + 1 + NumClonedBytes()` entries, even
// when the table is empty (hence EmptyGroup). // when the table is empty (hence EmptyGroup).
// //
// Note that growth_left is stored immediately before this pointer. // Note that growth_info is stored immediately before this pointer.
// May be uninitialized for SOO tables. // May be uninitialized for SOO tables.
ctrl_t* control; ctrl_t* control;
...@@ -1321,7 +1321,7 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1321,7 +1321,7 @@ class CommonFields : public CommonFieldsGenerationInfo {
ctrl_t* control() const { return heap_or_soo_.control(); } ctrl_t* control() const { return heap_or_soo_.control(); }
void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } void set_control(ctrl_t* c) { heap_or_soo_.control() = c; }
void* backing_array_start() const { void* backing_array_start() const {
// growth_left (and maybe infoz) is stored before control bytes. // growth_info (and maybe infoz) is stored before control bytes.
assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0);
return control() - ControlOffset(has_infoz()); return control() - ControlOffset(has_infoz());
} }
...@@ -1362,17 +1362,15 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1362,17 +1362,15 @@ class CommonFields : public CommonFieldsGenerationInfo {
// The number of slots we can still fill without needing to rehash. // The number of slots we can still fill without needing to rehash.
// This is stored in the heap allocation before the control bytes. // This is stored in the heap allocation before the control bytes.
// TODO(b/289225379): experiment with moving growth_left back inline to // TODO(b/289225379): experiment with moving growth_info back inline to
// increase room for SOO. // increase room for SOO.
size_t growth_left() const { GrowthInfo& growth_info() {
const size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1;
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0); assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0);
return *gl_ptr; return *gl_ptr;
} }
void set_growth_left(size_t gl) { GrowthInfo growth_info() const {
size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1; return const_cast<CommonFields*>(this)->growth_info();
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
*gl_ptr = gl;
} }
bool has_infoz() const { bool has_infoz() const {
...@@ -1759,7 +1757,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t); ...@@ -1759,7 +1757,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t);
FindInfo find_first_non_full_outofline(const CommonFields&, size_t); FindInfo find_first_non_full_outofline(const CommonFields&, size_t);
inline void ResetGrowthLeft(CommonFields& common) { inline void ResetGrowthLeft(CommonFields& common) {
common.set_growth_left(CapacityToGrowth(common.capacity()) - common.size()); common.growth_info().InitGrowthLeftNoDeleted(
CapacityToGrowth(common.capacity()) - common.size());
} }
// Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire // Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire
...@@ -1818,9 +1817,9 @@ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, ...@@ -1818,9 +1817,9 @@ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h,
SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size); SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size);
} }
// growth_left (which is a size_t) is stored with the backing array. // growth_info (which is a size_t) is stored with the backing array.
constexpr size_t BackingArrayAlignment(size_t align_of_slot) { constexpr size_t BackingArrayAlignment(size_t align_of_slot) {
return (std::max)(align_of_slot, alignof(size_t)); return (std::max)(align_of_slot, alignof(GrowthInfo));
} }
// Returns the address of the ith slot in slots where each slot occupies // Returns the address of the ith slot in slots where each slot occupies
...@@ -2717,7 +2716,7 @@ class raw_hash_set { ...@@ -2717,7 +2716,7 @@ class raw_hash_set {
infoz().RecordStorageChanged(size, cap); infoz().RecordStorageChanged(size, cap);
} }
common().set_size(size); common().set_size(size);
set_growth_left(growth_left() - size); growth_info().OverwriteManyEmptyAsFull(size);
} }
ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept(
...@@ -3847,7 +3846,8 @@ class raw_hash_set { ...@@ -3847,7 +3846,8 @@ class raw_hash_set {
resize(growth_left() > 0 ? cap : NextCapacity(cap)); resize(growth_left() > 0 ? cap : NextCapacity(cap));
} }
FindInfo target; FindInfo target;
if (!rehash_for_bug_detection && ABSL_PREDICT_FALSE(growth_left() == 0)) { if (!rehash_for_bug_detection &&
ABSL_PREDICT_FALSE(growth_left() == 0)) {
const size_t old_capacity = capacity(); const size_t old_capacity = capacity();
rehash_and_grow_if_necessary(); rehash_and_grow_if_necessary();
// NOTE: It is safe to use `FindFirstNonFullAfterResize` after // NOTE: It is safe to use `FindFirstNonFullAfterResize` after
...@@ -3863,7 +3863,7 @@ class raw_hash_set { ...@@ -3863,7 +3863,7 @@ class raw_hash_set {
target = find_first_non_full(common(), hash); target = find_first_non_full(common(), hash);
} }
PrepareInsertCommon(common()); PrepareInsertCommon(common());
set_growth_left(growth_left() - IsEmpty(control()[target.offset])); growth_info().OverwriteControlAsFull(control()[target.offset]);
SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type)); SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type));
infoz().RecordInsert(hash, target.probe_length); infoz().RecordInsert(hash, target.probe_length);
return target.offset; return target.offset;
...@@ -3909,11 +3909,16 @@ class raw_hash_set { ...@@ -3909,11 +3909,16 @@ class raw_hash_set {
// See `CapacityToGrowth()`. // See `CapacityToGrowth()`.
size_t growth_left() const { size_t growth_left() const {
assert(!is_soo()); assert(!is_soo());
return common().growth_left(); return growth_info().GetGrowthLeft();
}
GrowthInfo& growth_info() {
assert(!is_soo());
return common().growth_info();
} }
void set_growth_left(size_t gl) { GrowthInfo growth_info() const {
assert(!is_soo()); assert(!is_soo());
return common().set_growth_left(gl); return common().growth_info();
} }
// Prefetch the heap-allocated memory region to resolve potential TLB and // Prefetch the heap-allocated memory region to resolve potential TLB and
......
...@@ -1825,6 +1825,27 @@ TEST(Table, EraseInsertProbing) { ...@@ -1825,6 +1825,27 @@ TEST(Table, EraseInsertProbing) {
EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12)); EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12));
} }
TEST(Table, GrowthInfoDeletedBit) {
BadTable t;
EXPECT_TRUE(
RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted());
int64_t init_count = static_cast<int64_t>(
CapacityToGrowth(NormalizeCapacity(Group::kWidth + 1)));
for (int64_t i = 0; i < init_count; ++i) {
t.insert(i);
}
EXPECT_TRUE(
RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted());
t.erase(0);
EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 1);
EXPECT_FALSE(
RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted());
t.rehash(0);
EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 0);
EXPECT_TRUE(
RawHashSetTestOnlyAccess::GetCommon(t).growth_info().HasNoDeleted());
}
TYPED_TEST_P(SooTest, Clear) { TYPED_TEST_P(SooTest, Clear) {
TypeParam t; TypeParam t;
EXPECT_TRUE(t.find(0) == t.end()); EXPECT_TRUE(t.find(0) == t.end());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment