Skip to content

A
abseil-cpp
Commit eb46a63d by Gennadiy Rozental Committed by Copybara-Service
Options

Optimize the absl::GetFlag cost for most non built-in flag types (including string). 

PiperOrigin-RevId: 649200175
Change-Id: Ic6741d9fe5e0b1853ed8bb37b585d38b51d15581
parent 6e701508
Showing with 470 additions and 82 deletions
absl/flags/BUILD.bazel
...@@ -390,6 +390,7 @@ cc_test( ...@@ -390,6 +390,7 @@ cc_test(
":flag", ":flag",
":flag_internal", ":flag_internal",
":marshalling", ":marshalling",
":parse",
":reflection", ":reflection",
"//absl/base:core_headers", "//absl/base:core_headers",
"//absl/base:malloc_internal", "//absl/base:malloc_internal",
......
absl/flags/CMakeLists.txt
...@@ -340,6 +340,7 @@ absl_cc_test( ...@@ -340,6 +340,7 @@ absl_cc_test(
absl::flags_config absl::flags_config
absl::flags_internal absl::flags_internal
absl::flags_marshalling absl::flags_marshalling
absl::flags_parse
absl::flags_reflection absl::flags_reflection
absl::int128 absl::int128
absl::optional absl::optional
......
absl/flags/flag_test.cc
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include "absl/flags/declare.h" #include "absl/flags/declare.h"
#include "absl/flags/internal/flag.h" #include "absl/flags/internal/flag.h"
#include "absl/flags/marshalling.h" #include "absl/flags/marshalling.h"
#include "absl/flags/parse.h"
#include "absl/flags/reflection.h" #include "absl/flags/reflection.h"
#include "absl/flags/usage_config.h" #include "absl/flags/usage_config.h"
#include "absl/numeric/int128.h" #include "absl/numeric/int128.h"
...@@ -126,9 +127,9 @@ TEST_F(FlagTest, Traits) { ...@@ -126,9 +127,9 @@ TEST_F(FlagTest, Traits) {
#endif #endif
EXPECT_EQ(flags::StorageKind<std::string>(), EXPECT_EQ(flags::StorageKind<std::string>(),
flags::FlagValueStorageKind::kAlignedBuffer); flags::FlagValueStorageKind::kHeapAllocated);
EXPECT_EQ(flags::StorageKind<std::vector<std::string>>(), EXPECT_EQ(flags::StorageKind<std::vector<std::string>>(),
flags::FlagValueStorageKind::kAlignedBuffer); flags::FlagValueStorageKind::kHeapAllocated);
EXPECT_EQ(flags::StorageKind<absl::int128>(), EXPECT_EQ(flags::StorageKind<absl::int128>(),
flags::FlagValueStorageKind::kSequenceLocked); flags::FlagValueStorageKind::kSequenceLocked);
...@@ -267,7 +268,7 @@ TEST_F(FlagTest, TestFlagDeclaration) { ...@@ -267,7 +268,7 @@ TEST_F(FlagTest, TestFlagDeclaration) {
#if ABSL_FLAGS_STRIP_NAMES #if ABSL_FLAGS_STRIP_NAMES
// The intent of this helper struct and an expression below is to make sure that // The intent of this helper struct and an expression below is to make sure that
// in the configuration where ABSL_FLAGS_STRIP_NAMES=1 registrar construction // in the configuration where ABSL_FLAGS_STRIP_NAMES=1 registrar construction
// (in cases of of no Tail calls like OnUpdate) is constexpr and thus can and // (in cases of no Tail calls like OnUpdate) is constexpr and thus can and
// should be completely optimized away, thus avoiding the cost/overhead of // should be completely optimized away, thus avoiding the cost/overhead of
// static initializers. // static initializers.
struct VerifyConsteval { struct VerifyConsteval {
...@@ -515,8 +516,10 @@ TEST_F(FlagTest, TestDefault) { ...@@ -515,8 +516,10 @@ TEST_F(FlagTest, TestDefault) {
struct NonTriviallyCopyableAggregate { struct NonTriviallyCopyableAggregate {
NonTriviallyCopyableAggregate() = default; NonTriviallyCopyableAggregate() = default;
// NOLINTNEXTLINE
NonTriviallyCopyableAggregate(const NonTriviallyCopyableAggregate& rhs) NonTriviallyCopyableAggregate(const NonTriviallyCopyableAggregate& rhs)
: value(rhs.value) {} : value(rhs.value) {}
// NOLINTNEXTLINE
NonTriviallyCopyableAggregate& operator=( NonTriviallyCopyableAggregate& operator=(
const NonTriviallyCopyableAggregate& rhs) { const NonTriviallyCopyableAggregate& rhs) {
value = rhs.value; value = rhs.value;
...@@ -975,51 +978,194 @@ bool AbslParseFlag(absl::string_view, SmallAlignUDT*, std::string*) { ...@@ -975,51 +978,194 @@ bool AbslParseFlag(absl::string_view, SmallAlignUDT*, std::string*) {
} }
std::string AbslUnparseFlag(const SmallAlignUDT&) { return ""; } std::string AbslUnparseFlag(const SmallAlignUDT&) { return ""; }
// User-defined type with small size, but not trivially copyable. } // namespace
ABSL_FLAG(SmallAlignUDT, test_flag_sa_udt, {}, "help");
namespace {
TEST_F(FlagTest, TestSmallAlignUDT) {
EXPECT_EQ(flags::StorageKind<SmallAlignUDT>(),
flags::FlagValueStorageKind::kSequenceLocked);
SmallAlignUDT value = absl::GetFlag(FLAGS_test_flag_sa_udt);
EXPECT_EQ(value.c, 'A');
EXPECT_EQ(value.s, 12);
value.c = 'B';
value.s = 45;
absl::SetFlag(&FLAGS_test_flag_sa_udt, value);
value = absl::GetFlag(FLAGS_test_flag_sa_udt);
EXPECT_EQ(value.c, 'B');
EXPECT_EQ(value.s, 45);
}
} // namespace
// --------------------------------------------------------------------
namespace {
// User-defined not trivially copyable type.
template <int id>
struct NonTriviallyCopyableUDT { struct NonTriviallyCopyableUDT {
NonTriviallyCopyableUDT() : c('A') {} NonTriviallyCopyableUDT() : c('A') { s_num_instance++; }
NonTriviallyCopyableUDT(const NonTriviallyCopyableUDT& rhs) : c(rhs.c) {} NonTriviallyCopyableUDT(const NonTriviallyCopyableUDT& rhs) : c(rhs.c) {
s_num_instance++;
}
NonTriviallyCopyableUDT& operator=(const NonTriviallyCopyableUDT& rhs) { NonTriviallyCopyableUDT& operator=(const NonTriviallyCopyableUDT& rhs) {
c = rhs.c; c = rhs.c;
return *this; return *this;
} }
~NonTriviallyCopyableUDT() { s_num_instance--; }
static uint64_t s_num_instance;
char c; char c;
}; };
bool AbslParseFlag(absl::string_view, NonTriviallyCopyableUDT*, std::string*) { template <int id>
uint64_t NonTriviallyCopyableUDT<id>::s_num_instance = 0;
template <int id>
bool AbslParseFlag(absl::string_view txt, NonTriviallyCopyableUDT<id>* f,
std::string*) {
f->c = txt.empty() ? '\0' : txt[0];
return true; return true;
} }
std::string AbslUnparseFlag(const NonTriviallyCopyableUDT&) { return ""; } template <int id>
std::string AbslUnparseFlag(const NonTriviallyCopyableUDT<id>&) {
return "";
}
template <int id, typename F>
void TestExpectedLeaks(
F&& f, uint64_t num_leaks,
absl::optional<uint64_t> num_new_instances = absl::nullopt) {
if (!num_new_instances.has_value()) num_new_instances = num_leaks;
auto num_leaked_before = flags::NumLeakedFlagValues();
auto num_instances_before = NonTriviallyCopyableUDT<id>::s_num_instance;
f();
EXPECT_EQ(num_leaked_before + num_leaks, flags::NumLeakedFlagValues());
EXPECT_EQ(num_instances_before + num_new_instances.value(),
NonTriviallyCopyableUDT<id>::s_num_instance);
}
} // namespace } // namespace
ABSL_FLAG(SmallAlignUDT, test_flag_sa_udt, {}, "help"); ABSL_FLAG(NonTriviallyCopyableUDT<1>, test_flag_ntc_udt1, {}, "help");
ABSL_FLAG(NonTriviallyCopyableUDT, test_flag_ntc_udt, {}, "help"); ABSL_FLAG(NonTriviallyCopyableUDT<2>, test_flag_ntc_udt2, {}, "help");
ABSL_FLAG(NonTriviallyCopyableUDT<3>, test_flag_ntc_udt3, {}, "help");
ABSL_FLAG(NonTriviallyCopyableUDT<4>, test_flag_ntc_udt4, {}, "help");
ABSL_FLAG(NonTriviallyCopyableUDT<5>, test_flag_ntc_udt5, {}, "help");
namespace { namespace {
TEST_F(FlagTest, TestSmallAlignUDT) { TEST_F(FlagTest, TestNonTriviallyCopyableGetSetSet) {
SmallAlignUDT value = absl::GetFlag(FLAGS_test_flag_sa_udt); EXPECT_EQ(flags::StorageKind<NonTriviallyCopyableUDT<1>>(),
EXPECT_EQ(value.c, 'A'); flags::FlagValueStorageKind::kHeapAllocated);
EXPECT_EQ(value.s, 12);
TestExpectedLeaks<1>(
[&] {
NonTriviallyCopyableUDT<1> value =
absl::GetFlag(FLAGS_test_flag_ntc_udt1);
EXPECT_EQ(value.c, 'A');
},
0);
TestExpectedLeaks<1>(
[&] {
NonTriviallyCopyableUDT<1> value;
value.c = 'B';
absl::SetFlag(&FLAGS_test_flag_ntc_udt1, value);
EXPECT_EQ(value.c, 'B');
},
1);
TestExpectedLeaks<1>(
[&] {
NonTriviallyCopyableUDT<1> value;
value.c = 'C';
absl::SetFlag(&FLAGS_test_flag_ntc_udt1, value);
},
0);
}
value.c = 'B'; TEST_F(FlagTest, TestNonTriviallyCopyableParseSet) {
value.s = 45; TestExpectedLeaks<2>(
absl::SetFlag(&FLAGS_test_flag_sa_udt, value); [&] {
value = absl::GetFlag(FLAGS_test_flag_sa_udt); const char* in_argv[] = {"testbin", "--test_flag_ntc_udt2=A"};
EXPECT_EQ(value.c, 'B'); absl::ParseCommandLine(2, const_cast<char**>(in_argv));
EXPECT_EQ(value.s, 45); },
0);
TestExpectedLeaks<2>(
[&] {
NonTriviallyCopyableUDT<2> value;
value.c = 'B';
absl::SetFlag(&FLAGS_test_flag_ntc_udt2, value);
EXPECT_EQ(value.c, 'B');
},
0);
} }
TEST_F(FlagTest, TestNonTriviallyCopyableUDT) { TEST_F(FlagTest, TestNonTriviallyCopyableSet) {
NonTriviallyCopyableUDT value = absl::GetFlag(FLAGS_test_flag_ntc_udt); TestExpectedLeaks<3>(
EXPECT_EQ(value.c, 'A'); [&] {
NonTriviallyCopyableUDT<3> value;
value.c = 'B';
absl::SetFlag(&FLAGS_test_flag_ntc_udt3, value);
EXPECT_EQ(value.c, 'B');
},
0);
}
value.c = 'B'; // One new instance created during initialization and stored in the flag.
absl::SetFlag(&FLAGS_test_flag_ntc_udt, value); auto premain_utd4_get =
value = absl::GetFlag(FLAGS_test_flag_ntc_udt); (TestExpectedLeaks<4>([] { (void)absl::GetFlag(FLAGS_test_flag_ntc_udt4); },
EXPECT_EQ(value.c, 'B'); 0, 1),
false);
TEST_F(FlagTest, TestNonTriviallyCopyableGetBeforeMainParseGet) {
TestExpectedLeaks<4>(
[&] {
const char* in_argv[] = {"testbin", "--test_flag_ntc_udt4=C"};
absl::ParseCommandLine(2, const_cast<char**>(in_argv));
},
1);
TestExpectedLeaks<4>(
[&] {
NonTriviallyCopyableUDT<4> value =
absl::GetFlag(FLAGS_test_flag_ntc_udt4);
EXPECT_EQ(value.c, 'C');
},
0);
}
// One new instance created during initialization, which is reused since it was
// never read.
auto premain_utd5_set = (TestExpectedLeaks<5>(
[] {
NonTriviallyCopyableUDT<5> value;
value.c = 'B';
absl::SetFlag(&FLAGS_test_flag_ntc_udt5, value);
},
0, 1),
false);
TEST_F(FlagTest, TestNonTriviallyCopyableSetParseGet) {
TestExpectedLeaks<5>(
[&] {
const char* in_argv[] = {"testbin", "--test_flag_ntc_udt5=C"};
absl::ParseCommandLine(2, const_cast<char**>(in_argv));
},
0);
TestExpectedLeaks<5>(
[&] {
NonTriviallyCopyableUDT<5> value =
absl::GetFlag(FLAGS_test_flag_ntc_udt5);
EXPECT_EQ(value.c, 'C');
},
0);
} }
} // namespace } // namespace
......
absl/flags/internal/flag.cc
...@@ -22,14 +22,17 @@ ...@@ -22,14 +22,17 @@
#include <array> #include <array>
#include <atomic> #include <atomic>
#include <cstring>
#include <memory> #include <memory>
#include <new>
#include <string> #include <string>
#include <typeinfo> #include <typeinfo>
#include <vector>
#include "absl/base/attributes.h"
#include "absl/base/call_once.h" #include "absl/base/call_once.h"
#include "absl/base/casts.h" #include "absl/base/casts.h"
#include "absl/base/config.h" #include "absl/base/config.h"
#include "absl/base/const_init.h"
#include "absl/base/dynamic_annotations.h" #include "absl/base/dynamic_annotations.h"
#include "absl/base/optimization.h" #include "absl/base/optimization.h"
#include "absl/flags/config.h" #include "absl/flags/config.h"
...@@ -44,10 +47,9 @@ namespace absl { ...@@ -44,10 +47,9 @@ namespace absl {
ABSL_NAMESPACE_BEGIN ABSL_NAMESPACE_BEGIN
namespace flags_internal { namespace flags_internal {
// The help message indicating that the commandline flag has been // The help message indicating that the commandline flag has been stripped. It
// 'stripped'. It will not show up when doing "-help" and its // will not show up when doing "-help" and its variants. The flag is stripped
// variants. The flag is stripped if ABSL_FLAGS_STRIP_HELP is set to 1 // if ABSL_FLAGS_STRIP_HELP is set to 1 before including absl/flags/flag.h
// before including absl/flags/flag.h
const char kStrippedFlagHelp[] = "\001\002\003\004 (unknown) \004\003\002\001"; const char kStrippedFlagHelp[] = "\001\002\003\004 (unknown) \004\003\002\001";
namespace { namespace {
...@@ -78,9 +80,32 @@ class MutexRelock { ...@@ -78,9 +80,32 @@ class MutexRelock {
absl::Mutex& mu_; absl::Mutex& mu_;
}; };
// This is a freelist of leaked flag values and guard for its access.
// When we can't guarantee it is safe to reuse the memory for flag values,
// we move the memory to the freelist where it lives indefinitely, so it can
// still be safely accessed. This also prevents leak checkers from complaining
// about the leaked memory that can no longer be accessed through any pointer.
ABSL_CONST_INIT absl::Mutex s_freelist_guard(absl::kConstInit);
ABSL_CONST_INIT std::vector<void*>* s_freelist = nullptr;
void AddToFreelist(void* p) {
absl::MutexLock l(&s_freelist_guard);
if (!s_freelist) {
s_freelist = new std::vector<void*>;
}
s_freelist->push_back(p);
}
} // namespace } // namespace
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
uint64_t NumLeakedFlagValues() {
absl::MutexLock l(&s_freelist_guard);
return s_freelist == nullptr ? 0u : s_freelist->size();
}
///////////////////////////////////////////////////////////////////////////////
// Persistent state of the flag data. // Persistent state of the flag data.
class FlagImpl; class FlagImpl;
...@@ -97,7 +122,7 @@ class FlagState : public flags_internal::FlagStateInterface { ...@@ -97,7 +122,7 @@ class FlagState : public flags_internal::FlagStateInterface {
counter_(counter) {} counter_(counter) {}
~FlagState() override { ~FlagState() override {
if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kAlignedBuffer && if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kHeapAllocated &&
flag_impl_.ValueStorageKind() != FlagValueStorageKind::kSequenceLocked) flag_impl_.ValueStorageKind() != FlagValueStorageKind::kSequenceLocked)
return; return;
flags_internal::Delete(flag_impl_.op_, value_.heap_allocated); flags_internal::Delete(flag_impl_.op_, value_.heap_allocated);
...@@ -140,6 +165,33 @@ void DynValueDeleter::operator()(void* ptr) const { ...@@ -140,6 +165,33 @@ void DynValueDeleter::operator()(void* ptr) const {
Delete(op, ptr); Delete(op, ptr);
} }
MaskedPointer::MaskedPointer(ptr_t rhs, bool is_candidate) : ptr_(rhs) {
if (is_candidate) {
ApplyMask(kUnprotectedReadCandidate);
}
}
bool MaskedPointer::IsUnprotectedReadCandidate() const {
return CheckMask(kUnprotectedReadCandidate);
}
bool MaskedPointer::HasBeenRead() const { return CheckMask(kHasBeenRead); }
void MaskedPointer::Set(FlagOpFn op, const void* src, bool is_candidate) {
flags_internal::Copy(op, src, Ptr());
if (is_candidate) {
ApplyMask(kUnprotectedReadCandidate);
}
}
void MaskedPointer::MarkAsRead() { ApplyMask(kHasBeenRead); }
void MaskedPointer::ApplyMask(mask_t mask) {
ptr_ = reinterpret_cast<ptr_t>(reinterpret_cast<mask_t>(ptr_) | mask);
}
bool MaskedPointer::CheckMask(mask_t mask) const {
return (reinterpret_cast<mask_t>(ptr_) & mask) != 0;
}
void FlagImpl::Init() { void FlagImpl::Init() {
new (&data_guard_) absl::Mutex; new (&data_guard_) absl::Mutex;
...@@ -174,11 +226,16 @@ void FlagImpl::Init() { ...@@ -174,11 +226,16 @@ void FlagImpl::Init() {
(*default_value_.gen_func)(AtomicBufferValue()); (*default_value_.gen_func)(AtomicBufferValue());
break; break;
} }
case FlagValueStorageKind::kAlignedBuffer: case FlagValueStorageKind::kHeapAllocated:
// For this storage kind the default_value_ always points to gen_func // For this storage kind the default_value_ always points to gen_func
// during initialization. // during initialization.
assert(def_kind == FlagDefaultKind::kGenFunc); assert(def_kind == FlagDefaultKind::kGenFunc);
(*default_value_.gen_func)(AlignedBufferValue()); // Flag value initially points to the internal buffer.
MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire);
(*default_value_.gen_func)(ptr_value.Ptr());
// Default value is a candidate for an unprotected read.
PtrStorage().store(MaskedPointer(ptr_value.Ptr(), true),
std::memory_order_release);
break; break;
} }
seq_lock_.MarkInitialized(); seq_lock_.MarkInitialized();
...@@ -234,7 +291,7 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const { ...@@ -234,7 +291,7 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const {
return {res, DynValueDeleter{op_}}; return {res, DynValueDeleter{op_}};
} }
void FlagImpl::StoreValue(const void* src) { void FlagImpl::StoreValue(const void* src, ValueSource source) {
switch (ValueStorageKind()) { switch (ValueStorageKind()) {
case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kValueAndInitBit:
case FlagValueStorageKind::kOneWordAtomic: { case FlagValueStorageKind::kOneWordAtomic: {
...@@ -249,8 +306,27 @@ void FlagImpl::StoreValue(const void* src) { ...@@ -249,8 +306,27 @@ void FlagImpl::StoreValue(const void* src) {
seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_)); seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_));
break; break;
} }
case FlagValueStorageKind::kAlignedBuffer: case FlagValueStorageKind::kHeapAllocated:
Copy(op_, src, AlignedBufferValue()); MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire);
if (ptr_value.IsUnprotectedReadCandidate() && ptr_value.HasBeenRead()) {
// If current value is a candidate for an unprotected read and if it was
// already read at least once, follow up reads (if any) are done without
// mutex protection. We can't guarantee it is safe to reuse this memory
// since it may have been accessed by another thread concurrently, so
// instead we move the memory to a freelist so it can still be safely
// accessed, and allocate a new one for the new value.
AddToFreelist(ptr_value.Ptr());
ptr_value = MaskedPointer(Clone(op_, src), source == kCommandLine);
} else {
// Current value either was set programmatically or was never read.
// We can reuse the memory since all accesses to this value (if any)
// were protected by mutex. That said, if a new value comes from command
// line it now becomes a candidate for an unprotected read.
ptr_value.Set(op_, src, source == kCommandLine);
}
PtrStorage().store(ptr_value, std::memory_order_release);
seq_lock_.IncrementModificationCount(); seq_lock_.IncrementModificationCount();
break; break;
} }
...@@ -305,9 +381,10 @@ std::string FlagImpl::CurrentValue() const { ...@@ -305,9 +381,10 @@ std::string FlagImpl::CurrentValue() const {
ReadSequenceLockedData(cloned.get()); ReadSequenceLockedData(cloned.get());
return flags_internal::Unparse(op_, cloned.get()); return flags_internal::Unparse(op_, cloned.get());
} }
case FlagValueStorageKind::kAlignedBuffer: { case FlagValueStorageKind::kHeapAllocated: {
absl::MutexLock l(guard); absl::MutexLock l(guard);
return flags_internal::Unparse(op_, AlignedBufferValue()); return flags_internal::Unparse(
op_, PtrStorage().load(std::memory_order_acquire).Ptr());
} }
} }
...@@ -370,10 +447,12 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { ...@@ -370,10 +447,12 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() {
return absl::make_unique<FlagState>(*this, cloned, modified, return absl::make_unique<FlagState>(*this, cloned, modified,
on_command_line, ModificationCount()); on_command_line, ModificationCount());
} }
case FlagValueStorageKind::kAlignedBuffer: { case FlagValueStorageKind::kHeapAllocated: {
return absl::make_unique<FlagState>( return absl::make_unique<FlagState>(
*this, flags_internal::Clone(op_, AlignedBufferValue()), modified, *this,
on_command_line, ModificationCount()); flags_internal::Clone(
op_, PtrStorage().load(std::memory_order_acquire).Ptr()),
modified, on_command_line, ModificationCount());
} }
} }
return nullptr; return nullptr;
...@@ -388,11 +467,11 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) { ...@@ -388,11 +467,11 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) {
switch (ValueStorageKind()) { switch (ValueStorageKind()) {
case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kValueAndInitBit:
case FlagValueStorageKind::kOneWordAtomic: case FlagValueStorageKind::kOneWordAtomic:
StoreValue(&flag_state.value_.one_word); StoreValue(&flag_state.value_.one_word, kProgrammaticChange);
break; break;
case FlagValueStorageKind::kSequenceLocked: case FlagValueStorageKind::kSequenceLocked:
case FlagValueStorageKind::kAlignedBuffer: case FlagValueStorageKind::kHeapAllocated:
StoreValue(flag_state.value_.heap_allocated); StoreValue(flag_state.value_.heap_allocated, kProgrammaticChange);
break; break;
} }
...@@ -411,11 +490,6 @@ StorageT* FlagImpl::OffsetValue() const { ...@@ -411,11 +490,6 @@ StorageT* FlagImpl::OffsetValue() const {
return reinterpret_cast<StorageT*>(p + offset); return reinterpret_cast<StorageT*>(p + offset);
} }
void* FlagImpl::AlignedBufferValue() const {
assert(ValueStorageKind() == FlagValueStorageKind::kAlignedBuffer);
return OffsetValue<void>();
}
std::atomic<uint64_t>* FlagImpl::AtomicBufferValue() const { std::atomic<uint64_t>* FlagImpl::AtomicBufferValue() const {
assert(ValueStorageKind() == FlagValueStorageKind::kSequenceLocked); assert(ValueStorageKind() == FlagValueStorageKind::kSequenceLocked);
return OffsetValue<std::atomic<uint64_t>>(); return OffsetValue<std::atomic<uint64_t>>();
...@@ -427,6 +501,11 @@ std::atomic<int64_t>& FlagImpl::OneWordValue() const { ...@@ -427,6 +501,11 @@ std::atomic<int64_t>& FlagImpl::OneWordValue() const {
return OffsetValue<FlagOneWordValue>()->value; return OffsetValue<FlagOneWordValue>()->value;
} }
std::atomic<MaskedPointer>& FlagImpl::PtrStorage() const {
assert(ValueStorageKind() == FlagValueStorageKind::kHeapAllocated);
return OffsetValue<FlagMaskedPointerValue>()->value;
}
// Attempts to parse supplied `value` string using parsing routine in the `flag` // Attempts to parse supplied `value` string using parsing routine in the `flag`
// argument. If parsing successful, this function replaces the dst with newly // argument. If parsing successful, this function replaces the dst with newly
// parsed value. In case if any error is encountered in either step, the error // parsed value. In case if any error is encountered in either step, the error
...@@ -460,9 +539,17 @@ void FlagImpl::Read(void* dst) const { ...@@ -460,9 +539,17 @@ void FlagImpl::Read(void* dst) const {
ReadSequenceLockedData(dst); ReadSequenceLockedData(dst);
break; break;
} }
case FlagValueStorageKind::kAlignedBuffer: { case FlagValueStorageKind::kHeapAllocated: {
absl::MutexLock l(guard); absl::MutexLock l(guard);
flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst); MaskedPointer ptr_value = PtrStorage().load(std::memory_order_acquire);
flags_internal::CopyConstruct(op_, ptr_value.Ptr(), dst);
// For unprotected read candidates, mark that the value as has been read.
if (ptr_value.IsUnprotectedReadCandidate() && !ptr_value.HasBeenRead()) {
ptr_value.MarkAsRead();
PtrStorage().store(ptr_value, std::memory_order_release);
}
break; break;
} }
} }
...@@ -513,7 +600,7 @@ void FlagImpl::Write(const void* src) { ...@@ -513,7 +600,7 @@ void FlagImpl::Write(const void* src) {
} }
} }
StoreValue(src); StoreValue(src, kProgrammaticChange);
} }
// Sets the value of the flag based on specified string `value`. If the flag // Sets the value of the flag based on specified string `value`. If the flag
...@@ -534,7 +621,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, ...@@ -534,7 +621,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode,
auto tentative_value = TryParse(value, err); auto tentative_value = TryParse(value, err);
if (!tentative_value) return false; if (!tentative_value) return false;
StoreValue(tentative_value.get()); StoreValue(tentative_value.get(), source);
if (source == kCommandLine) { if (source == kCommandLine) {
on_command_line_ = true; on_command_line_ = true;
...@@ -555,7 +642,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, ...@@ -555,7 +642,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode,
auto tentative_value = TryParse(value, err); auto tentative_value = TryParse(value, err);
if (!tentative_value) return false; if (!tentative_value) return false;
StoreValue(tentative_value.get()); StoreValue(tentative_value.get(), source);
break; break;
} }
case SET_FLAGS_DEFAULT: { case SET_FLAGS_DEFAULT: {
...@@ -573,7 +660,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, ...@@ -573,7 +660,7 @@ bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode,
if (!modified_) { if (!modified_) {
// Need to set both default value *and* current, in this case. // Need to set both default value *and* current, in this case.
StoreValue(default_value_.dynamic_value); StoreValue(default_value_.dynamic_value, source);
modified_ = false; modified_ = false;
} }
break; break;
......
absl/flags/internal/flag.h
...@@ -295,11 +295,8 @@ constexpr FlagDefaultArg DefaultArg(char) { ...@@ -295,11 +295,8 @@ constexpr FlagDefaultArg DefaultArg(char) {
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Flag current value auxiliary structs. // Flag storage selector traits. Each trait indicates what kind of storage kind
// to use for the flag value.
constexpr int64_t UninitializedFlagValue() {
return static_cast<int64_t>(0xababababababababll);
}
template <typename T> template <typename T>
using FlagUseValueAndInitBitStorage = using FlagUseValueAndInitBitStorage =
...@@ -321,9 +318,11 @@ enum class FlagValueStorageKind : uint8_t { ...@@ -321,9 +318,11 @@ enum class FlagValueStorageKind : uint8_t {
kValueAndInitBit = 0, kValueAndInitBit = 0,
kOneWordAtomic = 1, kOneWordAtomic = 1,
kSequenceLocked = 2, kSequenceLocked = 2,
kAlignedBuffer = 3, kHeapAllocated = 3,
}; };
// This constexpr function returns the storage kind for the given flag value
// type.
template <typename T> template <typename T>
static constexpr FlagValueStorageKind StorageKind() { static constexpr FlagValueStorageKind StorageKind() {
return FlagUseValueAndInitBitStorage<T>::value return FlagUseValueAndInitBitStorage<T>::value
...@@ -332,14 +331,24 @@ static constexpr FlagValueStorageKind StorageKind() { ...@@ -332,14 +331,24 @@ static constexpr FlagValueStorageKind StorageKind() {
? FlagValueStorageKind::kOneWordAtomic ? FlagValueStorageKind::kOneWordAtomic
: FlagUseSequenceLockStorage<T>::value : FlagUseSequenceLockStorage<T>::value
? FlagValueStorageKind::kSequenceLocked ? FlagValueStorageKind::kSequenceLocked
: FlagValueStorageKind::kAlignedBuffer; : FlagValueStorageKind::kHeapAllocated;
} }
// This is a base class for the storage classes used by kOneWordAtomic and
// kValueAndInitBit storage kinds. It literally just stores the one word value
// as an atomic. By default, it is initialized to a magic value that is unlikely
// a valid value for the flag value type.
struct FlagOneWordValue { struct FlagOneWordValue {
constexpr static int64_t Uninitialized() {
return static_cast<int64_t>(0xababababababababll);
}
constexpr FlagOneWordValue() : value(Uninitialized()) {}
constexpr explicit FlagOneWordValue(int64_t v) : value(v) {} constexpr explicit FlagOneWordValue(int64_t v) : value(v) {}
std::atomic<int64_t> value; std::atomic<int64_t> value;
}; };
// This class represents a memory layout used by kValueAndInitBit storage kind.
template <typename T> template <typename T>
struct alignas(8) FlagValueAndInitBit { struct alignas(8) FlagValueAndInitBit {
T value; T value;
...@@ -348,16 +357,91 @@ struct alignas(8) FlagValueAndInitBit { ...@@ -348,16 +357,91 @@ struct alignas(8) FlagValueAndInitBit {
uint8_t init; uint8_t init;
}; };
// This class implements an aligned pointer with two options stored via masks
// in unused bits of the pointer value (due to alignment requirement).
// - IsUnprotectedReadCandidate - indicates that the value can be switched to
// unprotected read without a lock.
// - HasBeenRead - indicates that the value has been read at least once.
// - AllowsUnprotectedRead - combination of the two options above and indicates
// that the value can now be read without a lock.
// Further details of these options and their use is covered in the description
// of the FlagValue<T, FlagValueStorageKind::kHeapAllocated> specialization.
class MaskedPointer {
public:
using mask_t = uintptr_t;
using ptr_t = void*;
static constexpr int RequiredAlignment() { return 4; }
constexpr explicit MaskedPointer(ptr_t rhs) : ptr_(rhs) {}
MaskedPointer(ptr_t rhs, bool is_candidate);
void* Ptr() const {
return reinterpret_cast<void*>(reinterpret_cast<mask_t>(ptr_) &
kPtrValueMask);
}
bool AllowsUnprotectedRead() const {
return (reinterpret_cast<mask_t>(ptr_) & kAllowsUnprotectedRead) ==
kAllowsUnprotectedRead;
}
bool IsUnprotectedReadCandidate() const;
bool HasBeenRead() const;
void Set(FlagOpFn op, const void* src, bool is_candidate);
void MarkAsRead();
private:
// Masks
// Indicates that the flag value either default or originated from command
// line.
static constexpr mask_t kUnprotectedReadCandidate = 0x1u;
// Indicates that flag has been read.
static constexpr mask_t kHasBeenRead = 0x2u;
static constexpr mask_t kAllowsUnprotectedRead =
kUnprotectedReadCandidate | kHasBeenRead;
static constexpr mask_t kPtrValueMask = ~kAllowsUnprotectedRead;
void ApplyMask(mask_t mask);
bool CheckMask(mask_t mask) const;
ptr_t ptr_;
};
// This class implements a type erased storage of the heap allocated flag value.
// It is used as a base class for the storage class for kHeapAllocated storage
// kind. The initial_buffer is expected to have an alignment of at least
// MaskedPointer::RequiredAlignment(), so that the bits used by the
// MaskedPointer to store masks are set to 0. This guarantees that value starts
// in an uninitialized state.
struct FlagMaskedPointerValue {
constexpr explicit FlagMaskedPointerValue(MaskedPointer::ptr_t initial_buffer)
: value(MaskedPointer(initial_buffer)) {}
std::atomic<MaskedPointer> value;
};
// This is the forward declaration for the template that represents a storage
// for the flag values. This template is expected to be explicitly specialized
// for each storage kind and it does not have a generic default
// implementation.
template <typename T, template <typename T,
FlagValueStorageKind Kind = flags_internal::StorageKind<T>()> FlagValueStorageKind Kind = flags_internal::StorageKind<T>()>
struct FlagValue; struct FlagValue;
// This specialization represents the storage of flag values types with the
// kValueAndInitBit storage kind. It is based on the FlagOneWordValue class
// and relies on memory layout in FlagValueAndInitBit<T> to indicate that the
// value has been initialized or not.
template <typename T> template <typename T>
struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue {
constexpr FlagValue() : FlagOneWordValue(0) {} constexpr FlagValue() : FlagOneWordValue(0) {}
bool Get(const SequenceLock&, T& dst) const { bool Get(const SequenceLock&, T& dst) const {
int64_t storage = value.load(std::memory_order_acquire); int64_t storage = value.load(std::memory_order_acquire);
if (ABSL_PREDICT_FALSE(storage == 0)) { if (ABSL_PREDICT_FALSE(storage == 0)) {
// This assert is to ensure that the initialization inside FlagImpl::Init
// is able to set init member correctly.
static_assert(offsetof(FlagValueAndInitBit<T>, init) == sizeof(T),
"Unexpected memory layout of FlagValueAndInitBit");
return false; return false;
} }
dst = absl::bit_cast<FlagValueAndInitBit<T>>(storage).value; dst = absl::bit_cast<FlagValueAndInitBit<T>>(storage).value;
...@@ -365,12 +449,16 @@ struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { ...@@ -365,12 +449,16 @@ struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue {
} }
}; };
// This specialization represents the storage of flag values types with the
// kOneWordAtomic storage kind. It is based on the FlagOneWordValue class
// and relies on the magic uninitialized state of default constructed instead of
// FlagOneWordValue to indicate that the value has been initialized or not.
template <typename T> template <typename T>
struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue {
constexpr FlagValue() : FlagOneWordValue(UninitializedFlagValue()) {} constexpr FlagValue() : FlagOneWordValue() {}
bool Get(const SequenceLock&, T& dst) const { bool Get(const SequenceLock&, T& dst) const {
int64_t one_word_val = value.load(std::memory_order_acquire); int64_t one_word_val = value.load(std::memory_order_acquire);
if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) { if (ABSL_PREDICT_FALSE(one_word_val == FlagOneWordValue::Uninitialized())) {
return false; return false;
} }
std::memcpy(&dst, static_cast<const void*>(&one_word_val), sizeof(T)); std::memcpy(&dst, static_cast<const void*>(&one_word_val), sizeof(T));
...@@ -378,6 +466,12 @@ struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { ...@@ -378,6 +466,12 @@ struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue {
} }
}; };
// This specialization represents the storage of flag values types with the
// kSequenceLocked storage kind. This storage is used by trivially copyable
// types with size greater than 8 bytes. This storage relies on uninitialized
// state of the SequenceLock to indicate that the value has been initialized or
// not. This storage also provides lock-free read access to the underlying
// value once it is initialized.
template <typename T> template <typename T>
struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> {
bool Get(const SequenceLock& lock, T& dst) const { bool Get(const SequenceLock& lock, T& dst) const {
...@@ -391,11 +485,62 @@ struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { ...@@ -391,11 +485,62 @@ struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> {
std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords]; std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords];
}; };
// This specialization represents the storage of flag values types with the
// kHeapAllocated storage kind. This is a storage of last resort and is used
// if none of other storage kinds are applicable.
//
// Generally speaking the values with this storage kind can't be accessed
// atomically and thus can't be read without holding a lock. If we would ever
// want to avoid the lock, we'd need to leak the old value every time new flag
// value is being set (since we are in danger of having a race condition
// otherwise).
//
// Instead of doing that, this implementation attempts to cater to some common
// use cases by allowing at most 2 values to be leaked - default value and
// value set from the command line.
//
// This specialization provides an initial buffer for the first flag value. This
// is where the default value is going to be stored. We attempt to reuse this
// buffer if possible, including storing the value set from the command line
// there.
//
// As long as we only read this value, we can access it without a lock (in
// practice we still use the lock for the very first read to be able set
// "has been read" option on this flag).
//
// If flag is specified on the command line we store the parsed value either
// in the internal buffer (if the default value never been read) or we leak the
// default value and allocate the new storage for the parse value. This value is
// also a candidate for an unprotected read. If flag is set programmatically
// after the command line is parsed, the storage for this value is going to be
// leaked. Note that in both scenarios we are not going to have a real leak.
// Instead we'll store the leaked value pointers in the internal freelist to
// avoid triggering the memory leak checker complains.
//
// If the flag is ever set programmatically, it stops being the candidate for an
// unprotected read, and any follow up access to the flag value requires a lock.
// Note that if the value if set programmatically before the command line is
// parsed, we can switch back to enabling unprotected reads for that value.
template <typename T> template <typename T>
struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> { struct FlagValue<T, FlagValueStorageKind::kHeapAllocated>
bool Get(const SequenceLock&, T&) const { return false; } : FlagMaskedPointerValue {
// We const initialize the value with unmasked pointer to the internal buffer,
// making sure it is not a candidate for unprotected read. This way we can
// ensure Init is done before any access to the flag value.
constexpr FlagValue() : FlagMaskedPointerValue(&buffer[0]) {}
bool Get(const SequenceLock&, T& dst) const {
MaskedPointer ptr_value = value.load(std::memory_order_acquire);
if (ABSL_PREDICT_TRUE(ptr_value.AllowsUnprotectedRead())) {
::new (static_cast<void*>(&dst)) T(*static_cast<T*>(ptr_value.Ptr()));
return true;
}
return false;
}
alignas(T) char value[sizeof(T)]; alignas(MaskedPointer::RequiredAlignment()) alignas(
T) char buffer[sizeof(T)]{};
}; };
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
...@@ -483,7 +628,7 @@ class FlagImpl final : public CommandLineFlag { ...@@ -483,7 +628,7 @@ class FlagImpl final : public CommandLineFlag {
// Used in read/write operations to validate source/target has correct type. // Used in read/write operations to validate source/target has correct type.
// For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to // For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to
// absl::GetFlag(FLAGS_foo) validates that the type of FLAGS_foo is indeed // absl::GetFlag(FLAGS_foo) validates that the type of FLAGS_foo is indeed
// int. To do that we pass the "assumed" type id (which is deduced from type // int. To do that we pass the assumed type id (which is deduced from type
// int) as an argument `type_id`, which is in turn is validated against the // int) as an argument `type_id`, which is in turn is validated against the
// type id stored in flag object by flag definition statement. // type id stored in flag object by flag definition statement.
void AssertValidType(FlagFastTypeId type_id, void AssertValidType(FlagFastTypeId type_id,
...@@ -504,17 +649,13 @@ class FlagImpl final : public CommandLineFlag { ...@@ -504,17 +649,13 @@ class FlagImpl final : public CommandLineFlag {
void Init(); void Init();
// Offset value access methods. One per storage kind. These methods to not // Offset value access methods. One per storage kind. These methods to not
// respect const correctness, so be very carefull using them. // respect const correctness, so be very careful using them.
// This is a shared helper routine which encapsulates most of the magic. Since // This is a shared helper routine which encapsulates most of the magic. Since
// it is only used inside the three routines below, which are defined in // it is only used inside the three routines below, which are defined in
// flag.cc, we can define it in that file as well. // flag.cc, we can define it in that file as well.
template <typename StorageT> template <typename StorageT>
StorageT* OffsetValue() const; StorageT* OffsetValue() const;
// This is an accessor for a value stored in an aligned buffer storage
// used for non-trivially-copyable data types.
// Returns a mutable pointer to the start of a buffer.
void* AlignedBufferValue() const;
// The same as above, but used for sequencelock-protected storage. // The same as above, but used for sequencelock-protected storage.
std::atomic<uint64_t>* AtomicBufferValue() const; std::atomic<uint64_t>* AtomicBufferValue() const;
...@@ -523,13 +664,16 @@ class FlagImpl final : public CommandLineFlag { ...@@ -523,13 +664,16 @@ class FlagImpl final : public CommandLineFlag {
// mutable reference to an atomic value. // mutable reference to an atomic value.
std::atomic<int64_t>& OneWordValue() const; std::atomic<int64_t>& OneWordValue() const;
std::atomic<MaskedPointer>& PtrStorage() const;
// Attempts to parse supplied `value` string. If parsing is successful, // Attempts to parse supplied `value` string. If parsing is successful,
// returns new value. Otherwise returns nullptr. // returns new value. Otherwise returns nullptr.
std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value, std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value,
std::string& err) const std::string& err) const
ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
// Stores the flag value based on the pointer to the source. // Stores the flag value based on the pointer to the source.
void StoreValue(const void* src) ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); void StoreValue(const void* src, ValueSource source)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard());
// Copy the flag data, protected by `seq_lock_` into `dst`. // Copy the flag data, protected by `seq_lock_` into `dst`.
// //
...@@ -585,7 +729,7 @@ class FlagImpl final : public CommandLineFlag { ...@@ -585,7 +729,7 @@ class FlagImpl final : public CommandLineFlag {
const char* const name_; const char* const name_;
// The file name where ABSL_FLAG resides. // The file name where ABSL_FLAG resides.
const char* const filename_; const char* const filename_;
// Type-specific operations "vtable". // Type-specific operations vtable.
const FlagOpFn op_; const FlagOpFn op_;
// Help message literal or function to generate it. // Help message literal or function to generate it.
const FlagHelpMsg help_; const FlagHelpMsg help_;
...@@ -720,16 +864,21 @@ class FlagImplPeer { ...@@ -720,16 +864,21 @@ class FlagImplPeer {
// Implementation of Flag value specific operations routine. // Implementation of Flag value specific operations routine.
template <typename T> template <typename T>
void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) {
struct AlignedSpace {
alignas(MaskedPointer::RequiredAlignment()) alignas(T) char buf[sizeof(T)];
};
using Allocator = std::allocator<AlignedSpace>;
switch (op) { switch (op) {
case FlagOp::kAlloc: { case FlagOp::kAlloc: {
std::allocator<T> alloc; Allocator alloc;
return std::allocator_traits<std::allocator<T>>::allocate(alloc, 1); return std::allocator_traits<Allocator>::allocate(alloc, 1);
} }
case FlagOp::kDelete: { case FlagOp::kDelete: {
T* p = static_cast<T*>(v2); T* p = static_cast<T*>(v2);
p->~T(); p->~T();
std::allocator<T> alloc; Allocator alloc;
std::allocator_traits<std::allocator<T>>::deallocate(alloc, p, 1); std::allocator_traits<Allocator>::deallocate(
alloc, reinterpret_cast<AlignedSpace*>(p), 1);
return nullptr; return nullptr;
} }
case FlagOp::kCopy: case FlagOp::kCopy:
...@@ -789,7 +938,7 @@ class FlagRegistrar { ...@@ -789,7 +938,7 @@ class FlagRegistrar {
return *this; return *this;
} }
// Make the registrar "die" gracefully as an empty struct on a line where // Makes the registrar die gracefully as an empty struct on a line where
// registration happens. Registrar objects are intended to live only as // registration happens. Registrar objects are intended to live only as
// temporary. // temporary.
constexpr operator FlagRegistrarEmpty() const { return {}; } // NOLINT constexpr operator FlagRegistrarEmpty() const { return {}; } // NOLINT
...@@ -798,6 +947,10 @@ class FlagRegistrar { ...@@ -798,6 +947,10 @@ class FlagRegistrar {
Flag<T>& flag_; // Flag being registered (not owned). Flag<T>& flag_; // Flag being registered (not owned).
}; };
///////////////////////////////////////////////////////////////////////////////
// Test only API
uint64_t NumLeakedFlagValues();
} // namespace flags_internal } // namespace flags_internal
ABSL_NAMESPACE_END ABSL_NAMESPACE_END
} // namespace absl } // namespace absl
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment