absl: inline and de-dup Mutex::Await/LockWhen/CondVar::Wait

Mutex::Await/LockWhen/CondVar::Wait duplicate code, and cause additional
calls at runtime and code bloat.
Inline thin wrappers that just convert argument types and
add a single de-duped implementation for these methods.

This reduces code size, shaves off 55K from the mutex_test in release build,
and should make things marginally faster.

$ nm -nS mutex_test | egrep "(_ZN4absl5Mutex.*(Await|LockWhen))|(_ZN4absl7CondVar.*Wait)"

before:
00000000000912c0 00000000000001a8 T _ZN4absl7CondVar4WaitEPNS_5MutexE
00000000000988c0 0000000000000c36 T _ZN4absl7CondVar16WaitWithDeadlineEPNS_5MutexENS_4TimeE
000000000009a6e0 0000000000000041 T _ZN4absl5Mutex19LockWhenWithTimeoutERKNS_9ConditionENS_8DurationE
00000000000a28c0 0000000000000779 T _ZN4absl5Mutex17AwaitWithDeadlineERKNS_9ConditionENS_4TimeE
00000000000cf4e0 0000000000000011 T _ZN4absl5Mutex8LockWhenERKNS_9ConditionE
00000000000cf500 0000000000000041 T _ZN4absl5Mutex20LockWhenWithDeadlineERKNS_9ConditionENS_4TimeE
00000000000cf560 0000000000000011 T _ZN4absl5Mutex14ReaderLockWhenERKNS_9ConditionE
00000000000cf580 0000000000000041 T _ZN4absl5Mutex26ReaderLockWhenWithDeadlineERKNS_9ConditionENS_4TimeE
00000000000cf5e0 0000000000000766 T _ZN4absl5Mutex5AwaitERKNS_9ConditionE
00000000000cfd60 00000000000007b5 T _ZN4absl5Mutex16AwaitWithTimeoutERKNS_9ConditionENS_8DurationE
00000000000d0700 00000000000003cf T _ZN4absl7CondVar15WaitWithTimeoutEPNS_5MutexENS_8DurationE
000000000011c280 0000000000000041 T _ZN4absl5Mutex25ReaderLockWhenWithTimeoutERKNS_9ConditionENS_8DurationE

after:
000000000009c300 00000000000007ed T _ZN4absl7CondVar10WaitCommonEPNS_5MutexENS_24synchronization_internal13KernelTimeoutE
00000000000a03c0 00000000000006fe T _ZN4absl5Mutex11AwaitCommonERKNS_9ConditionENS_24synchronization_internal13KernelTimeoutE
000000000011ae00 0000000000000025 T _ZN4absl5Mutex14LockWhenCommonERKNS_9ConditionENS_24synchronization_internal13KernelTimeoutEb
PiperOrigin-RevId: 563729364
Change-Id: Ic6b43761f76719c01e03d43cc0e0c419e41a85c1

absl/synchronization/mutex.cc

@@ -1522,104 +1522,25 @@ void Mutex::ReaderLock() {
   ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
   ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
 }
 }
 
 
-void Mutex::LockWhen(const Condition& cond) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, 0);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  this->LockSlow(kExclusive, &cond, 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0);
-}
-
-bool Mutex::LockWhenWithTimeout(const Condition& cond, absl::Duration timeout) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, 0);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  bool res = LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(timeout), 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0);
-  return res;
-}
-
-bool Mutex::LockWhenWithDeadline(const Condition& cond, absl::Time deadline) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, 0);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  bool res =
-      LockSlowWithDeadline(kExclusive, &cond, KernelTimeout(deadline), 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0);
-  return res;
-}
-
-void Mutex::ReaderLockWhen(const Condition& cond) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  this->LockSlow(kShared, &cond, 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
-}
-
-bool Mutex::ReaderLockWhenWithTimeout(const Condition& cond,
-                                      absl::Duration timeout) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock);
-  GraphId id = DebugOnlyDeadlockCheck(this);
-  bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(timeout), 0);
-  DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
-  return res;
-}
-
-bool Mutex::ReaderLockWhenWithDeadline(const Condition& cond,
-                                       absl::Time deadline) {
-  ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock);
+bool Mutex::LockWhenCommon(const Condition& cond,
+                           synchronization_internal::KernelTimeout t,
+                           bool write) {
+  MuHow how = write ? kExclusive : kShared;
+  ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how));
   GraphId id = DebugOnlyDeadlockCheck(this);
   GraphId id = DebugOnlyDeadlockCheck(this);
-  bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(deadline), 0);
+  bool res = LockSlowWithDeadline(how, &cond, t, 0);
   DebugOnlyLockEnter(this, id);
   DebugOnlyLockEnter(this, id);
-  ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0);
+  ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0);
   return res;
   return res;
 }
 }
 
 
-void Mutex::Await(const Condition& cond) {
-  if (cond.Eval()) {  // condition already true; nothing to do
-    if (kDebugMode) {
-      this->AssertReaderHeld();
-    }
-  } else {  // normal case
-    ABSL_RAW_CHECK(this->AwaitCommon(cond, KernelTimeout::Never()),
-                   "condition untrue on return from Await");
-  }
-}
-
-bool Mutex::AwaitWithTimeout(const Condition& cond, absl::Duration timeout) {
-  if (cond.Eval()) {  // condition already true; nothing to do
-    if (kDebugMode) {
-      this->AssertReaderHeld();
-    }
-    return true;
+bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) {
+  if (kDebugMode) {
+    this->AssertReaderHeld();
   }
   }
-
-  KernelTimeout t{timeout};
-  bool res = this->AwaitCommon(cond, t);
-  ABSL_RAW_CHECK(res || t.has_timeout(),
-                 "condition untrue on return from Await");
-  return res;
-}
-
-bool Mutex::AwaitWithDeadline(const Condition& cond, absl::Time deadline) {
   if (cond.Eval()) {  // condition already true; nothing to do
   if (cond.Eval()) {  // condition already true; nothing to do
-    if (kDebugMode) {
-      this->AssertReaderHeld();
-    }
     return true;
     return true;
   }
   }
-
-  KernelTimeout t{deadline};
-  bool res = this->AwaitCommon(cond, t);
-  ABSL_RAW_CHECK(res || t.has_timeout(),
-                 "condition untrue on return from Await");
-  return res;
-}
-
-bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) {
-  this->AssertReaderHeld();
   MuHow how =
   MuHow how =
       (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared;
       (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared;
   ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how));
   ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how));
@@ -1634,6 +1555,8 @@ bool Mutex::AwaitCommon(const Condition& cond, KernelTimeout t) {
   bool res = waitp.cond != nullptr ||  // => cond known true from LockSlowLoop
   bool res = waitp.cond != nullptr ||  // => cond known true from LockSlowLoop
              EvalConditionAnnotated(&cond, this, true, false, how == kShared);
              EvalConditionAnnotated(&cond, this, true, false, how == kShared);
   ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0);
   ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0);
+  ABSL_RAW_CHECK(res || t.has_timeout(),
+                 "condition untrue on return from Await");
   return res;
   return res;
 }
 }
 
 
@@ -2654,16 +2577,6 @@ bool CondVar::WaitCommon(Mutex* mutex, KernelTimeout t) {
   return rc;
   return rc;
 }
 }
 
 
-bool CondVar::WaitWithTimeout(Mutex* mu, absl::Duration timeout) {
-  return WaitCommon(mu, KernelTimeout(timeout));
-}
-
-bool CondVar::WaitWithDeadline(Mutex* mu, absl::Time deadline) {
-  return WaitCommon(mu, KernelTimeout(deadline));
-}
-
-void CondVar::Wait(Mutex* mu) { WaitCommon(mu, KernelTimeout::Never()); }
-
 // Wake thread w
 // Wake thread w
 // If it was a timed wait, w will be waiting on w->cv
 // If it was a timed wait, w will be waiting on w->cv
 // Otherwise, if it was not a Mutex mutex, w will be waiting on w->sem
 // Otherwise, if it was not a Mutex mutex, w will be waiting on w->sem

absl/synchronization/mutex.h

@@ -324,7 +324,9 @@ class ABSL_LOCKABLE Mutex {
   // `true`, `Await()` *may* skip the release/re-acquire step.
   // `true`, `Await()` *may* skip the release/re-acquire step.
   //
   //
   // `Await()` requires that this thread holds this `Mutex` in some mode.
   // `Await()` requires that this thread holds this `Mutex` in some mode.
-  void Await(const Condition& cond);
+  void Await(const Condition& cond) {
+    AwaitCommon(cond, synchronization_internal::KernelTimeout::Never());
+  }
 
 
   // Mutex::LockWhen()
   // Mutex::LockWhen()
   // Mutex::ReaderLockWhen()
   // Mutex::ReaderLockWhen()
@@ -334,9 +336,15 @@ class ABSL_LOCKABLE Mutex {
   // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is
   // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is
   // logically equivalent to `*Lock(); Await();` though they may have different
   // logically equivalent to `*Lock(); Await();` though they may have different
   // performance characteristics.
   // performance characteristics.
-  void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION();
+  void LockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() {
+    LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(),
+                   true);
+  }
 
 
-  void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION();
+  void ReaderLockWhen(const Condition& cond) ABSL_SHARED_LOCK_FUNCTION() {
+    LockWhenCommon(cond, synchronization_internal::KernelTimeout::Never(),
+                   false);
+  }
 
 
   void WriterLockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() {
   void WriterLockWhen(const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() {
     this->LockWhen(cond);
     this->LockWhen(cond);
@@ -363,9 +371,13 @@ class ABSL_LOCKABLE Mutex {
   // Negative timeouts are equivalent to a zero timeout.
   // Negative timeouts are equivalent to a zero timeout.
   //
   //
   // This method requires that this thread holds this `Mutex` in some mode.
   // This method requires that this thread holds this `Mutex` in some mode.
-  bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout);
+  bool AwaitWithTimeout(const Condition& cond, absl::Duration timeout) {
+    return AwaitCommon(cond, synchronization_internal::KernelTimeout{timeout});
+  }
 
 
-  bool AwaitWithDeadline(const Condition& cond, absl::Time deadline);
+  bool AwaitWithDeadline(const Condition& cond, absl::Time deadline) {
+    return AwaitCommon(cond, synchronization_internal::KernelTimeout{deadline});
+  }
 
 
   // Mutex::LockWhenWithTimeout()
   // Mutex::LockWhenWithTimeout()
   // Mutex::ReaderLockWhenWithTimeout()
   // Mutex::ReaderLockWhenWithTimeout()
@@ -379,9 +391,15 @@ class ABSL_LOCKABLE Mutex {
   //
   //
   // Negative timeouts are equivalent to a zero timeout.
   // Negative timeouts are equivalent to a zero timeout.
   bool LockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
   bool LockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
-      ABSL_EXCLUSIVE_LOCK_FUNCTION();
+      ABSL_EXCLUSIVE_LOCK_FUNCTION() {
+    return LockWhenCommon(
+        cond, synchronization_internal::KernelTimeout{timeout}, true);
+  }
   bool ReaderLockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
   bool ReaderLockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
-      ABSL_SHARED_LOCK_FUNCTION();
+      ABSL_SHARED_LOCK_FUNCTION() {
+    return LockWhenCommon(
+        cond, synchronization_internal::KernelTimeout{timeout}, false);
+  }
   bool WriterLockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
   bool WriterLockWhenWithTimeout(const Condition& cond, absl::Duration timeout)
       ABSL_EXCLUSIVE_LOCK_FUNCTION() {
       ABSL_EXCLUSIVE_LOCK_FUNCTION() {
     return this->LockWhenWithTimeout(cond, timeout);
     return this->LockWhenWithTimeout(cond, timeout);
@@ -399,9 +417,15 @@ class ABSL_LOCKABLE Mutex {
   //
   //
   // Deadlines in the past are equivalent to an immediate deadline.
   // Deadlines in the past are equivalent to an immediate deadline.
   bool LockWhenWithDeadline(const Condition& cond, absl::Time deadline)
   bool LockWhenWithDeadline(const Condition& cond, absl::Time deadline)
-      ABSL_EXCLUSIVE_LOCK_FUNCTION();
+      ABSL_EXCLUSIVE_LOCK_FUNCTION() {
+    return LockWhenCommon(
+        cond, synchronization_internal::KernelTimeout{deadline}, true);
+  }
   bool ReaderLockWhenWithDeadline(const Condition& cond, absl::Time deadline)
   bool ReaderLockWhenWithDeadline(const Condition& cond, absl::Time deadline)
-      ABSL_SHARED_LOCK_FUNCTION();
+      ABSL_SHARED_LOCK_FUNCTION() {
+    return LockWhenCommon(
+        cond, synchronization_internal::KernelTimeout{deadline}, false);
+  }
   bool WriterLockWhenWithDeadline(const Condition& cond, absl::Time deadline)
   bool WriterLockWhenWithDeadline(const Condition& cond, absl::Time deadline)
       ABSL_EXCLUSIVE_LOCK_FUNCTION() {
       ABSL_EXCLUSIVE_LOCK_FUNCTION() {
     return this->LockWhenWithDeadline(cond, deadline);
     return this->LockWhenWithDeadline(cond, deadline);
@@ -500,6 +524,8 @@ class ABSL_LOCKABLE Mutex {
   // Common code between Await() and AwaitWithTimeout/Deadline()
   // Common code between Await() and AwaitWithTimeout/Deadline()
   bool AwaitCommon(const Condition& cond,
   bool AwaitCommon(const Condition& cond,
                    synchronization_internal::KernelTimeout t);
                    synchronization_internal::KernelTimeout t);
+  bool LockWhenCommon(const Condition& cond,
+                      synchronization_internal::KernelTimeout t, bool write);
   // Attempt to remove thread s from queue.
   // Attempt to remove thread s from queue.
   void TryRemove(base_internal::PerThreadSynch* s);
   void TryRemove(base_internal::PerThreadSynch* s);
   // Block a thread on mutex.
   // Block a thread on mutex.
@@ -888,7 +914,9 @@ class CondVar {
   // spurious wakeup), then reacquires the `Mutex` and returns.
   // spurious wakeup), then reacquires the `Mutex` and returns.
   //
   //
   // Requires and ensures that the current thread holds the `Mutex`.
   // Requires and ensures that the current thread holds the `Mutex`.
-  void Wait(Mutex* mu);
+  void Wait(Mutex* mu) {
+    WaitCommon(mu, synchronization_internal::KernelTimeout::Never());
+  }
 
 
   // CondVar::WaitWithTimeout()
   // CondVar::WaitWithTimeout()
   //
   //
@@ -903,7 +931,9 @@ class CondVar {
   // to return `true` or `false`.
   // to return `true` or `false`.
   //
   //
   // Requires and ensures that the current thread holds the `Mutex`.
   // Requires and ensures that the current thread holds the `Mutex`.
-  bool WaitWithTimeout(Mutex* mu, absl::Duration timeout);
+  bool WaitWithTimeout(Mutex* mu, absl::Duration timeout) {
+    return WaitCommon(mu, synchronization_internal::KernelTimeout(timeout));
+  }
 
 
   // CondVar::WaitWithDeadline()
   // CondVar::WaitWithDeadline()
   //
   //
@@ -920,7 +950,9 @@ class CondVar {
   // to return `true` or `false`.
   // to return `true` or `false`.
   //
   //
   // Requires and ensures that the current thread holds the `Mutex`.
   // Requires and ensures that the current thread holds the `Mutex`.
-  bool WaitWithDeadline(Mutex* mu, absl::Time deadline);
+  bool WaitWithDeadline(Mutex* mu, absl::Time deadline) {
+    return WaitCommon(mu, synchronization_internal::KernelTimeout(deadline));
+  }
 
 
   // CondVar::Signal()
   // CondVar::Signal()
   //
   //