Commit 79e41467 by Evan Brown Committed by Copybara-Service

Add suggestions to enable sanitizers for asserts when doing so may be helpful.

PiperOrigin-RevId: 684499936
Change-Id: Id5901d9d526abdf269f097a293bab6d08850d432
parent 485f2be7
...@@ -255,6 +255,15 @@ namespace container_internal { ...@@ -255,6 +255,15 @@ namespace container_internal {
#define ABSL_SWISSTABLE_ENABLE_GENERATIONS #define ABSL_SWISSTABLE_ENABLE_GENERATIONS
#endif #endif
#ifdef ABSL_SWISSTABLE_ASSERT
#error ABSL_SWISSTABLE_ASSERT cannot be directly set
#else
// We use this macro for assertions that users may see when the table is in an
// invalid state that sanitizers may help diagnose.
#define ABSL_SWISSTABLE_ASSERT(CONDITION) \
assert((CONDITION) && "Try enabling sanitizers.")
#endif
// We use uint8_t so we don't need to worry about padding. // We use uint8_t so we don't need to worry about padding.
using GenerationType = uint8_t; using GenerationType = uint8_t;
...@@ -325,7 +334,7 @@ class probe_seq { ...@@ -325,7 +334,7 @@ class probe_seq {
// sequence and `mask` (usually the capacity of the table) as the mask to // sequence and `mask` (usually the capacity of the table) as the mask to
// apply to each value in the progression. // apply to each value in the progression.
probe_seq(size_t hash, size_t mask) { probe_seq(size_t hash, size_t mask) {
assert(((mask + 1) & mask) == 0 && "not a mask"); ABSL_SWISSTABLE_ASSERT(((mask + 1) & mask) == 0 && "not a mask");
mask_ = mask; mask_ = mask;
offset_ = hash & mask_; offset_ = hash & mask_;
} }
...@@ -455,7 +464,7 @@ class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> { ...@@ -455,7 +464,7 @@ class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> {
public: public:
explicit BitMask(T mask) : Base(mask) { explicit BitMask(T mask) : Base(mask) {
if (Shift == 3 && !NullifyBitsOnIteration) { if (Shift == 3 && !NullifyBitsOnIteration) {
assert(this->mask_ == (this->mask_ & kMsbs8Bytes)); ABSL_SWISSTABLE_ASSERT(this->mask_ == (this->mask_ & kMsbs8Bytes));
} }
} }
// BitMask is an iterator over the indices of its abstract bits. // BitMask is an iterator over the indices of its abstract bits.
...@@ -1118,19 +1127,20 @@ class GrowthInfo { ...@@ -1118,19 +1127,20 @@ class GrowthInfo {
// Overwrites single empty slot with a full slot. // Overwrites single empty slot with a full slot.
void OverwriteEmptyAsFull() { void OverwriteEmptyAsFull() {
assert(GetGrowthLeft() > 0); ABSL_SWISSTABLE_ASSERT(GetGrowthLeft() > 0);
--growth_left_info_; --growth_left_info_;
} }
// Overwrites several empty slots with full slots. // Overwrites several empty slots with full slots.
void OverwriteManyEmptyAsFull(size_t cnt) { void OverwriteManyEmptyAsFull(size_t cnt) {
assert(GetGrowthLeft() >= cnt); ABSL_SWISSTABLE_ASSERT(GetGrowthLeft() >= cnt);
growth_left_info_ -= cnt; growth_left_info_ -= cnt;
} }
// Overwrites specified control element with full slot. // Overwrites specified control element with full slot.
void OverwriteControlAsFull(ctrl_t ctrl) { void OverwriteControlAsFull(ctrl_t ctrl) {
assert(GetGrowthLeft() >= static_cast<size_t>(IsEmpty(ctrl))); ABSL_SWISSTABLE_ASSERT(GetGrowthLeft() >=
static_cast<size_t>(IsEmpty(ctrl)));
growth_left_info_ -= static_cast<size_t>(IsEmpty(ctrl)); growth_left_info_ -= static_cast<size_t>(IsEmpty(ctrl));
} }
...@@ -1200,7 +1210,7 @@ class RawHashSetLayout { ...@@ -1200,7 +1210,7 @@ class RawHashSetLayout {
slot_offset_( slot_offset_(
(generation_offset_ + NumGenerationBytes() + slot_align - 1) & (generation_offset_ + NumGenerationBytes() + slot_align - 1) &
(~slot_align + 1)) { (~slot_align + 1)) {
assert(IsValidCapacity(capacity)); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity));
} }
// Returns the capacity of a table. // Returns the capacity of a table.
...@@ -1363,7 +1373,8 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1363,7 +1373,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
void set_control(ctrl_t* c) { heap_or_soo_.control() = c; } void set_control(ctrl_t* c) { heap_or_soo_.control() = c; }
void* backing_array_start() const { void* backing_array_start() const {
// growth_info (and maybe infoz) is stored before control bytes. // growth_info (and maybe infoz) is stored before control bytes.
assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); ABSL_SWISSTABLE_ASSERT(
reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0);
return control() - ControlOffset(has_infoz()); return control() - ControlOffset(has_infoz());
} }
...@@ -1386,11 +1397,11 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1386,11 +1397,11 @@ class CommonFields : public CommonFieldsGenerationInfo {
size_ = size_t{1} << HasInfozShift(); size_ = size_t{1} << HasInfozShift();
} }
void increment_size() { void increment_size() {
assert(size() < capacity()); ABSL_SWISSTABLE_ASSERT(size() < capacity());
size_ += size_t{1} << HasInfozShift(); size_ += size_t{1} << HasInfozShift();
} }
void decrement_size() { void decrement_size() {
assert(size() > 0); ABSL_SWISSTABLE_ASSERT(size() > 0);
size_ -= size_t{1} << HasInfozShift(); size_ -= size_t{1} << HasInfozShift();
} }
...@@ -1398,7 +1409,8 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1398,7 +1409,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
size_t capacity() const { return capacity_; } size_t capacity() const { return capacity_; }
void set_capacity(size_t c) { void set_capacity(size_t c) {
// We allow setting above the max valid capacity for debugging purposes. // We allow setting above the max valid capacity for debugging purposes.
assert(c == 0 || IsValidCapacity(c) || c > kAboveMaxValidCapacity); ABSL_SWISSTABLE_ASSERT(c == 0 || IsValidCapacity(c) ||
c > kAboveMaxValidCapacity);
capacity_ = c; capacity_ = c;
} }
...@@ -1410,7 +1422,8 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1410,7 +1422,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
GrowthInfo& growth_info() { GrowthInfo& growth_info() {
auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1; auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1;
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0); ABSL_SWISSTABLE_ASSERT(
reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0);
return *gl_ptr; return *gl_ptr;
} }
GrowthInfo growth_info() const { GrowthInfo growth_info() const {
...@@ -1430,7 +1443,7 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1430,7 +1443,7 @@ class CommonFields : public CommonFieldsGenerationInfo {
: HashtablezInfoHandle(); : HashtablezInfoHandle();
} }
void set_infoz(HashtablezInfoHandle infoz) { void set_infoz(HashtablezInfoHandle infoz) {
assert(has_infoz()); ABSL_SWISSTABLE_ASSERT(has_infoz());
*reinterpret_cast<HashtablezInfoHandle*>(backing_array_start()) = infoz; *reinterpret_cast<HashtablezInfoHandle*>(backing_array_start()) = infoz;
} }
...@@ -1490,8 +1503,8 @@ class CommonFields : public CommonFieldsGenerationInfo { ...@@ -1490,8 +1503,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
// We can't assert that SOO is enabled because we don't have SooEnabled(), but // We can't assert that SOO is enabled because we don't have SooEnabled(), but
// we assert what we can. // we assert what we can.
void AssertInSooMode() const { void AssertInSooMode() const {
assert(capacity() == SooCapacity()); ABSL_SWISSTABLE_ASSERT(capacity() == SooCapacity());
assert(!has_infoz()); ABSL_SWISSTABLE_ASSERT(!has_infoz());
} }
// The number of slots in the backing array. This is always 2^N-1 for an // The number of slots in the backing array. This is always 2^N-1 for an
...@@ -1517,7 +1530,7 @@ class raw_hash_set; ...@@ -1517,7 +1530,7 @@ class raw_hash_set;
// Returns the next valid capacity after `n`. // Returns the next valid capacity after `n`.
inline size_t NextCapacity(size_t n) { inline size_t NextCapacity(size_t n) {
assert(IsValidCapacity(n) || n == 0); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(n) || n == 0);
return n * 2 + 1; return n * 2 + 1;
} }
...@@ -1547,7 +1560,7 @@ inline size_t NormalizeCapacity(size_t n) { ...@@ -1547,7 +1560,7 @@ inline size_t NormalizeCapacity(size_t n) {
// Given `capacity`, applies the load factor; i.e., it returns the maximum // Given `capacity`, applies the load factor; i.e., it returns the maximum
// number of values we should put into the table before a resizing rehash. // number of values we should put into the table before a resizing rehash.
inline size_t CapacityToGrowth(size_t capacity) { inline size_t CapacityToGrowth(size_t capacity) {
assert(IsValidCapacity(capacity)); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity));
// `capacity*7/8` // `capacity*7/8`
if (Group::kWidth == 8 && capacity == 7) { if (Group::kWidth == 8 && capacity == 7) {
// x-x/8 does not work when x==7. // x-x/8 does not work when x==7.
...@@ -1805,7 +1818,7 @@ inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) { ...@@ -1805,7 +1818,7 @@ inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
seq.index()}; seq.index()};
} }
seq.next(); seq.next();
assert(seq.index() <= common.capacity() && "full table!"); ABSL_SWISSTABLE_ASSERT(seq.index() <= common.capacity() && "full table!");
} }
} }
...@@ -1837,7 +1850,7 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) { ...@@ -1837,7 +1850,7 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) {
// Sets sanitizer poisoning for slot corresponding to control byte being set. // Sets sanitizer poisoning for slot corresponding to control byte being set.
inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h, inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h,
size_t slot_size) { size_t slot_size) {
assert(i < c.capacity()); ABSL_SWISSTABLE_ASSERT(i < c.capacity());
auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size; auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size;
if (IsFull(h)) { if (IsFull(h)) {
SanitizerUnpoisonMemoryRegion(slot_i, slot_size); SanitizerUnpoisonMemoryRegion(slot_i, slot_size);
...@@ -1867,7 +1880,7 @@ inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) { ...@@ -1867,7 +1880,7 @@ inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) {
// setting the cloned control byte. // setting the cloned control byte.
inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h, inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h,
size_t slot_size) { size_t slot_size) {
assert(is_single_group(c.capacity())); ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity()));
DoSanitizeOnSetCtrl(c, i, h, slot_size); DoSanitizeOnSetCtrl(c, i, h, slot_size);
ctrl_t* ctrl = c.control(); ctrl_t* ctrl = c.control();
ctrl[i] = h; ctrl[i] = h;
...@@ -1907,8 +1920,8 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( ...@@ -1907,8 +1920,8 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots(
// Small tables capacity fits into portable group, where // Small tables capacity fits into portable group, where
// GroupPortableImpl::MaskFull is more efficient for the // GroupPortableImpl::MaskFull is more efficient for the
// capacity <= GroupPortableImpl::kWidth. // capacity <= GroupPortableImpl::kWidth.
assert(cap <= GroupPortableImpl::kWidth && ABSL_SWISSTABLE_ASSERT(cap <= GroupPortableImpl::kWidth &&
"unexpectedly large small capacity"); "unexpectedly large small capacity");
static_assert(Group::kWidth >= GroupPortableImpl::kWidth, static_assert(Group::kWidth >= GroupPortableImpl::kWidth,
"unexpected group width"); "unexpected group width");
// Group starts from kSentinel slot, so indices in the mask will // Group starts from kSentinel slot, so indices in the mask will
...@@ -1925,19 +1938,21 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots( ...@@ -1925,19 +1938,21 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots(
ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining; ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining;
while (remaining != 0) { while (remaining != 0) {
for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) { for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) {
assert(IsFull(ctrl[i]) && "hash table was modified unexpectedly"); ABSL_SWISSTABLE_ASSERT(IsFull(ctrl[i]) &&
"hash table was modified unexpectedly");
cb(ctrl + i, slot + i); cb(ctrl + i, slot + i);
--remaining; --remaining;
} }
ctrl += Group::kWidth; ctrl += Group::kWidth;
slot += Group::kWidth; slot += Group::kWidth;
assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) && ABSL_SWISSTABLE_ASSERT(
"hash table was modified unexpectedly"); (remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) &&
"hash table was modified unexpectedly");
} }
// NOTE: erasure of the current element is allowed in callback for // NOTE: erasure of the current element is allowed in callback for
// absl::erase_if specialization. So we use `>=`. // absl::erase_if specialization. So we use `>=`.
assert(original_size_for_assert >= c.size() && ABSL_SWISSTABLE_ASSERT(original_size_for_assert >= c.size() &&
"hash table was modified unexpectedly"); "hash table was modified unexpectedly");
} }
template <typename CharAlloc> template <typename CharAlloc>
...@@ -2008,18 +2023,18 @@ class HashSetResizeHelper { ...@@ -2008,18 +2023,18 @@ class HashSetResizeHelper {
// Offset fall on kSentinel or into the mostly occupied first half. // Offset fall on kSentinel or into the mostly occupied first half.
offset = old_capacity / 2; offset = old_capacity / 2;
} }
assert(IsEmpty(c.control()[offset])); ABSL_SWISSTABLE_ASSERT(IsEmpty(c.control()[offset]));
return FindInfo{offset, 0}; return FindInfo{offset, 0};
} }
HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; } HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; }
void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); } void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); }
ctrl_t* old_ctrl() const { ctrl_t* old_ctrl() const {
assert(!was_soo_); ABSL_SWISSTABLE_ASSERT(!was_soo_);
return old_heap_or_soo_.control(); return old_heap_or_soo_.control();
} }
void* old_slots() const { void* old_slots() const {
assert(!was_soo_); ABSL_SWISSTABLE_ASSERT(!was_soo_);
return old_heap_or_soo_.slot_array().get(); return old_heap_or_soo_.slot_array().get();
} }
size_t old_capacity() const { return old_capacity_; } size_t old_capacity() const { return old_capacity_; }
...@@ -2070,7 +2085,7 @@ class HashSetResizeHelper { ...@@ -2070,7 +2085,7 @@ class HashSetResizeHelper {
ctrl_t soo_slot_h2, ctrl_t soo_slot_h2,
size_t key_size, size_t key_size,
size_t value_size) { size_t value_size) {
assert(c.capacity()); ABSL_SWISSTABLE_ASSERT(c.capacity());
HashtablezInfoHandle infoz = HashtablezInfoHandle infoz =
ShouldSampleHashtablezInfo<Alloc>() ShouldSampleHashtablezInfo<Alloc>()
? SampleHashtablezInfo<SooEnabled>(SizeOfSlot, key_size, value_size, ? SampleHashtablezInfo<SooEnabled>(SizeOfSlot, key_size, value_size,
...@@ -2127,10 +2142,11 @@ class HashSetResizeHelper { ...@@ -2127,10 +2142,11 @@ class HashSetResizeHelper {
// 1. GrowIntoSingleGroupShuffleControlBytes was already called. // 1. GrowIntoSingleGroupShuffleControlBytes was already called.
template <class PolicyTraits, class Alloc> template <class PolicyTraits, class Alloc>
void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) { void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) {
assert(old_capacity_ < Group::kWidth / 2); ABSL_SWISSTABLE_ASSERT(old_capacity_ < Group::kWidth / 2);
assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity())); ABSL_SWISSTABLE_ASSERT(
IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
using slot_type = typename PolicyTraits::slot_type; using slot_type = typename PolicyTraits::slot_type;
assert(is_single_group(c.capacity())); ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity()));
auto* new_slots = static_cast<slot_type*>(c.slot_array()); auto* new_slots = static_cast<slot_type*>(c.slot_array());
auto* old_slots_ptr = static_cast<slot_type*>(old_slots()); auto* old_slots_ptr = static_cast<slot_type*>(old_slots());
...@@ -2764,14 +2780,14 @@ class raw_hash_set { ...@@ -2764,14 +2780,14 @@ class raw_hash_set {
// We don't use `that.is_soo()` here because `that` can have non-SOO // We don't use `that.is_soo()` here because `that` can have non-SOO
// capacity but have a size that fits into SOO capacity. // capacity but have a size that fits into SOO capacity.
if (fits_in_soo(size)) { if (fits_in_soo(size)) {
assert(size == 1); ABSL_SWISSTABLE_ASSERT(size == 1);
common().set_full_soo(); common().set_full_soo();
emplace_at(soo_iterator(), *that.begin()); emplace_at(soo_iterator(), *that.begin());
const HashtablezInfoHandle infoz = try_sample_soo(); const HashtablezInfoHandle infoz = try_sample_soo();
if (infoz.IsSampled()) resize_with_soo_infoz(infoz); if (infoz.IsSampled()) resize_with_soo_infoz(infoz);
return; return;
} }
assert(!that.is_soo()); ABSL_SWISSTABLE_ASSERT(!that.is_soo());
const size_t cap = capacity(); const size_t cap = capacity();
// Note about single group tables: // Note about single group tables:
// 1. It is correct to have any order of elements. // 1. It is correct to have any order of elements.
...@@ -2803,7 +2819,8 @@ class raw_hash_set { ...@@ -2803,7 +2819,8 @@ class raw_hash_set {
offset = (offset + shift) & cap; offset = (offset + shift) & cap;
} }
const h2_t h2 = static_cast<h2_t>(*that_ctrl); const h2_t h2 = static_cast<h2_t>(*that_ctrl);
assert( // We rely that hash is not changed for small tables. ABSL_SWISSTABLE_ASSERT( // We rely that hash is not changed for small
// tables.
H2(PolicyTraits::apply(HashElement{hash_ref()}, H2(PolicyTraits::apply(HashElement{hash_ref()},
PolicyTraits::element(that_slot))) == h2 && PolicyTraits::element(that_slot))) == h2 &&
"hash function value changed unexpectedly during the copy"); "hash function value changed unexpectedly during the copy");
...@@ -2889,7 +2906,7 @@ class raw_hash_set { ...@@ -2889,7 +2906,7 @@ class raw_hash_set {
iterator it = {control(), common().slots_union(), iterator it = {control(), common().slots_union(),
common().generation_ptr()}; common().generation_ptr()};
it.skip_empty_or_deleted(); it.skip_empty_or_deleted();
assert(IsFull(*it.control())); ABSL_SWISSTABLE_ASSERT(IsFull(*it.control()));
return it; return it;
} }
iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND { iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND {
...@@ -3123,7 +3140,7 @@ class raw_hash_set { ...@@ -3123,7 +3140,7 @@ class raw_hash_set {
public: public:
template <class... Args> template <class... Args>
void operator()(Args&&... args) const { void operator()(Args&&... args) const {
assert(*slot_); ABSL_SWISSTABLE_ASSERT(*slot_);
PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...);
*slot_ = nullptr; *slot_ = nullptr;
} }
...@@ -3142,7 +3159,7 @@ class raw_hash_set { ...@@ -3142,7 +3159,7 @@ class raw_hash_set {
if (res.second) { if (res.second) {
slot_type* slot = res.first.slot(); slot_type* slot = res.first.slot();
std::forward<F>(f)(constructor(&alloc_ref(), &slot)); std::forward<F>(f)(constructor(&alloc_ref(), &slot));
assert(!slot); ABSL_SWISSTABLE_ASSERT(!slot);
} }
return res.first; return res.first;
} }
...@@ -3303,7 +3320,7 @@ class raw_hash_set { ...@@ -3303,7 +3320,7 @@ class raw_hash_set {
resize(kInitialSampledCapacity); resize(kInitialSampledCapacity);
} }
// This asserts that we didn't lose sampling coverage in `resize`. // This asserts that we didn't lose sampling coverage in `resize`.
assert(infoz().IsSampled()); ABSL_SWISSTABLE_ASSERT(infoz().IsSampled());
return; return;
} }
alignas(slot_type) unsigned char slot_space[sizeof(slot_type)]; alignas(slot_type) unsigned char slot_space[sizeof(slot_type)];
...@@ -3557,7 +3574,7 @@ class raw_hash_set { ...@@ -3557,7 +3574,7 @@ class raw_hash_set {
// SOO functionality. // SOO functionality.
template <class K = key_type> template <class K = key_type>
iterator find_soo(const key_arg<K>& key) { iterator find_soo(const key_arg<K>& key) {
assert(is_soo()); ABSL_SWISSTABLE_ASSERT(is_soo());
return empty() || !PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, return empty() || !PolicyTraits::apply(EqualElement<K>{key, eq_ref()},
PolicyTraits::element(soo_slot())) PolicyTraits::element(soo_slot()))
? end() ? end()
...@@ -3566,7 +3583,7 @@ class raw_hash_set { ...@@ -3566,7 +3583,7 @@ class raw_hash_set {
template <class K = key_type> template <class K = key_type>
iterator find_non_soo(const key_arg<K>& key, size_t hash) { iterator find_non_soo(const key_arg<K>& key, size_t hash) {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
auto seq = probe(common(), hash); auto seq = probe(common(), hash);
const ctrl_t* ctrl = control(); const ctrl_t* ctrl = control();
while (true) { while (true) {
...@@ -3579,7 +3596,7 @@ class raw_hash_set { ...@@ -3579,7 +3596,7 @@ class raw_hash_set {
} }
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
seq.next(); seq.next();
assert(seq.index() <= capacity() && "full table!"); ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity() && "full table!");
} }
} }
...@@ -3587,14 +3604,14 @@ class raw_hash_set { ...@@ -3587,14 +3604,14 @@ class raw_hash_set {
// insertion into an empty SOO table and in copy construction when the size // insertion into an empty SOO table and in copy construction when the size
// can fit in SOO capacity. // can fit in SOO capacity.
inline HashtablezInfoHandle try_sample_soo() { inline HashtablezInfoHandle try_sample_soo() {
assert(is_soo()); ABSL_SWISSTABLE_ASSERT(is_soo());
if (!ShouldSampleHashtablezInfo<CharAlloc>()) return HashtablezInfoHandle{}; if (!ShouldSampleHashtablezInfo<CharAlloc>()) return HashtablezInfoHandle{};
return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type), return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type),
SooCapacity()); SooCapacity());
} }
inline void destroy_slots() { inline void destroy_slots() {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
if (PolicyTraits::template destroy_is_trivial<Alloc>()) return; if (PolicyTraits::template destroy_is_trivial<Alloc>()) return;
IterateOverFullSlots( IterateOverFullSlots(
common(), slot_array(), common(), slot_array(),
...@@ -3603,7 +3620,7 @@ class raw_hash_set { ...@@ -3603,7 +3620,7 @@ class raw_hash_set {
} }
inline void dealloc() { inline void dealloc() {
assert(capacity() != 0); ABSL_SWISSTABLE_ASSERT(capacity() != 0);
// Unpoison before returning the memory to the allocator. // Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity()); SanitizerUnpoisonMemoryRegion(slot_array(), sizeof(slot_type) * capacity());
infoz().Unregister(); infoz().Unregister();
...@@ -3633,7 +3650,7 @@ class raw_hash_set { ...@@ -3633,7 +3650,7 @@ class raw_hash_set {
// This merely updates the pertinent control byte. This can be used in // This merely updates the pertinent control byte. This can be used in
// conjunction with Policy::transfer to move the object to another place. // conjunction with Policy::transfer to move the object to another place.
void erase_meta_only(const_iterator it) { void erase_meta_only(const_iterator it) {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()), EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()),
sizeof(slot_type)); sizeof(slot_type));
} }
...@@ -3659,7 +3676,7 @@ class raw_hash_set { ...@@ -3659,7 +3676,7 @@ class raw_hash_set {
// SOO tables, since they need to switch from SOO to heap in order to // SOO tables, since they need to switch from SOO to heap in order to
// store the infoz. // store the infoz.
void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) { void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) {
assert(forced_infoz.IsSampled()); ABSL_SWISSTABLE_ASSERT(forced_infoz.IsSampled());
raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()), raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()),
forced_infoz); forced_infoz);
} }
...@@ -3670,8 +3687,8 @@ class raw_hash_set { ...@@ -3670,8 +3687,8 @@ class raw_hash_set {
CommonFields& common, size_t new_capacity, CommonFields& common, size_t new_capacity,
HashtablezInfoHandle forced_infoz) { HashtablezInfoHandle forced_infoz) {
raw_hash_set* set = reinterpret_cast<raw_hash_set*>(&common); raw_hash_set* set = reinterpret_cast<raw_hash_set*>(&common);
assert(IsValidCapacity(new_capacity)); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity));
assert(!set->fits_in_soo(new_capacity)); ABSL_SWISSTABLE_ASSERT(!set->fits_in_soo(new_capacity));
const bool was_soo = set->is_soo(); const bool was_soo = set->is_soo();
const bool had_soo_slot = was_soo && !set->empty(); const bool had_soo_slot = was_soo && !set->empty();
const ctrl_t soo_slot_h2 = const ctrl_t soo_slot_h2 =
...@@ -3706,7 +3723,7 @@ class raw_hash_set { ...@@ -3706,7 +3723,7 @@ class raw_hash_set {
// InitializeSlots did all the work including infoz().RecordRehash(). // InitializeSlots did all the work including infoz().RecordRehash().
return; return;
} }
assert(resize_helper.old_capacity() > 0); ABSL_SWISSTABLE_ASSERT(resize_helper.old_capacity() > 0);
// Nothing more to do in this case. // Nothing more to do in this case.
if (was_soo && !had_soo_slot) return; if (was_soo && !had_soo_slot) return;
...@@ -3882,7 +3899,7 @@ class raw_hash_set { ...@@ -3882,7 +3899,7 @@ class raw_hash_set {
template <class K> template <class K>
std::pair<iterator, bool> find_or_prepare_insert_non_soo(const K& key) { std::pair<iterator, bool> find_or_prepare_insert_non_soo(const K& key) {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
prefetch_heap_block(); prefetch_heap_block();
auto hash = hash_ref()(key); auto hash = hash_ref()(key);
auto seq = probe(common(), hash); auto seq = probe(common(), hash);
...@@ -3905,7 +3922,7 @@ class raw_hash_set { ...@@ -3905,7 +3922,7 @@ class raw_hash_set {
true}; true};
} }
seq.next(); seq.next();
assert(seq.index() <= capacity() && "full table!"); ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity() && "full table!");
} }
} }
...@@ -4032,16 +4049,16 @@ class raw_hash_set { ...@@ -4032,16 +4049,16 @@ class raw_hash_set {
// //
// See `CapacityToGrowth()`. // See `CapacityToGrowth()`.
size_t growth_left() const { size_t growth_left() const {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return common().growth_left(); return common().growth_left();
} }
GrowthInfo& growth_info() { GrowthInfo& growth_info() {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return common().growth_info(); return common().growth_info();
} }
GrowthInfo growth_info() const { GrowthInfo growth_info() const {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return common().growth_info(); return common().growth_info();
} }
...@@ -4049,7 +4066,7 @@ class raw_hash_set { ...@@ -4049,7 +4066,7 @@ class raw_hash_set {
// cache misses. This is intended to overlap with execution of calculating the // cache misses. This is intended to overlap with execution of calculating the
// hash for a key. // hash for a key.
void prefetch_heap_block() const { void prefetch_heap_block() const {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
#if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
__builtin_prefetch(control(), 0, 1); __builtin_prefetch(control(), 0, 1);
#endif #endif
...@@ -4059,15 +4076,15 @@ class raw_hash_set { ...@@ -4059,15 +4076,15 @@ class raw_hash_set {
const CommonFields& common() const { return settings_.template get<0>(); } const CommonFields& common() const { return settings_.template get<0>(); }
ctrl_t* control() const { ctrl_t* control() const {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return common().control(); return common().control();
} }
slot_type* slot_array() const { slot_type* slot_array() const {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return static_cast<slot_type*>(common().slot_array()); return static_cast<slot_type*>(common().slot_array());
} }
slot_type* soo_slot() { slot_type* soo_slot() {
assert(is_soo()); ABSL_SWISSTABLE_ASSERT(is_soo());
return static_cast<slot_type*>(common().soo_data()); return static_cast<slot_type*>(common().soo_data());
} }
const slot_type* soo_slot() const { const slot_type* soo_slot() const {
...@@ -4080,7 +4097,7 @@ class raw_hash_set { ...@@ -4080,7 +4097,7 @@ class raw_hash_set {
return const_cast<raw_hash_set*>(this)->soo_iterator(); return const_cast<raw_hash_set*>(this)->soo_iterator();
} }
HashtablezInfoHandle infoz() { HashtablezInfoHandle infoz() {
assert(!is_soo()); ABSL_SWISSTABLE_ASSERT(!is_soo());
return common().infoz(); return common().infoz();
} }
...@@ -4153,7 +4170,8 @@ struct HashtableFreeFunctionsAccess { ...@@ -4153,7 +4170,8 @@ struct HashtableFreeFunctionsAccess {
if (c->is_soo()) { if (c->is_soo()) {
auto it = c->soo_iterator(); auto it = c->soo_iterator();
if (!pred(*it)) { if (!pred(*it)) {
assert(c->size() == 1 && "hash table was modified unexpectedly"); ABSL_SWISSTABLE_ASSERT(c->size() == 1 &&
"hash table was modified unexpectedly");
return 0; return 0;
} }
c->destroy(it.slot()); c->destroy(it.slot());
...@@ -4173,8 +4191,9 @@ struct HashtableFreeFunctionsAccess { ...@@ -4173,8 +4191,9 @@ struct HashtableFreeFunctionsAccess {
}); });
// NOTE: IterateOverFullSlots allow removal of the current element, so we // NOTE: IterateOverFullSlots allow removal of the current element, so we
// verify the size additionally here. // verify the size additionally here.
assert(original_size_for_assert - num_deleted == c->size() && ABSL_SWISSTABLE_ASSERT(original_size_for_assert - num_deleted ==
"hash table was modified unexpectedly"); c->size() &&
"hash table was modified unexpectedly");
return num_deleted; return num_deleted;
} }
...@@ -4268,5 +4287,6 @@ ABSL_NAMESPACE_END ...@@ -4268,5 +4287,6 @@ ABSL_NAMESPACE_END
#undef ABSL_SWISSTABLE_ENABLE_GENERATIONS #undef ABSL_SWISSTABLE_ENABLE_GENERATIONS
#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED #undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED
#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN #undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN
#undef ABSL_SWISSTABLE_ASSERT
#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment