11.2.0/bits/atomic_futex.h

// -*- C++ -*- header.

// Copyright (C) 2015-2021 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/** @file bits/atomic_futex.h
 *  This is an internal header file, included by other library headers.
 *  Do not attempt to use it directly.
 */

#ifndef _GLIBCXX_ATOMIC_FUTEX_H
#define _GLIBCXX_ATOMIC_FUTEX_H 1

#pragma GCC system_header

#include <bits/c++config.h>
#include <atomic>
#include <chrono>
#if ! (defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1)
#include <mutex>
#include <condition_variable>
#endif

#ifndef _GLIBCXX_ALWAYS_INLINE
#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
#endif

namespace std _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION

#ifdef _GLIBCXX_HAS_GTHREADS
#if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
  struct __atomic_futex_unsigned_base
  {
    // __s and __ns are measured against CLOCK_REALTIME. Returns false
    // iff a timeout occurred.
    bool
    _M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout,
        chrono::seconds __s, chrono::nanoseconds __ns);

    // __s and __ns are measured against CLOCK_MONOTONIC. Returns
    // false iff a timeout occurred.
    bool
    _M_futex_wait_until_steady(unsigned *__addr, unsigned __val,
        bool __has_timeout, chrono::seconds __s, chrono::nanoseconds __ns);

    // This can be executed after the object has been destroyed.
    static void _M_futex_notify_all(unsigned* __addr);
  };

  template <unsigned _Waiter_bit = 0x80000000>
  class __atomic_futex_unsigned : __atomic_futex_unsigned_base
  {
    typedef chrono::steady_clock __clock_t;

    // This must be lock-free and at offset 0.
    atomic<unsigned> _M_data;

  public:
    explicit
    __atomic_futex_unsigned(unsigned __data) : _M_data(__data)
    { }

    _GLIBCXX_ALWAYS_INLINE unsigned
    _M_load(memory_order __mo)
    {
      return _M_data.load(__mo) & ~_Waiter_bit;
    }

  private:
    // If a timeout occurs, returns a current value after the timeout;
    // otherwise, returns the operand's value if equal is true or a different
    // value if equal is false.
    // The assumed value is the caller's assumption about the current value
    // when making the call.
    // __s and __ns are measured against CLOCK_REALTIME.
    unsigned
    _M_load_and_test_until(unsigned __assumed, unsigned __operand,
        bool __equal, memory_order __mo, bool __has_timeout,
        chrono::seconds __s, chrono::nanoseconds __ns)
    {
      for (;;)
        {
          // Don't bother checking the value again because we expect the caller
          // to have done it recently.
          // memory_order_relaxed is sufficient because we can rely on just the
          // modification order (store_notify uses an atomic RMW operation too),
          // and the futex syscalls synchronize between themselves.
          _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
          bool __ret = _M_futex_wait_until((unsigned*)(void*)&_M_data,
                                           __assumed | _Waiter_bit,
                                           __has_timeout, __s, __ns);
          // Fetch the current value after waiting (clears _Waiter_bit).
          __assumed = _M_load(__mo);
          if (!__ret || ((__operand == __assumed) == __equal))
            return __assumed;
          // TODO adapt wait time
        }
    }

    // If a timeout occurs, returns a current value after the timeout;
    // otherwise, returns the operand's value if equal is true or a different
    // value if equal is false.
    // The assumed value is the caller's assumption about the current value
    // when making the call.
    // __s and __ns are measured against CLOCK_MONOTONIC.
    unsigned
    _M_load_and_test_until_steady(unsigned __assumed, unsigned __operand,
        bool __equal, memory_order __mo, bool __has_timeout,
        chrono::seconds __s, chrono::nanoseconds __ns)
    {
      for (;;)
        {
          // Don't bother checking the value again because we expect the caller
          // to have done it recently.
          // memory_order_relaxed is sufficient because we can rely on just the
          // modification order (store_notify uses an atomic RMW operation too),
          // and the futex syscalls synchronize between themselves.
          _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
          bool __ret = _M_futex_wait_until_steady((unsigned*)(void*)&_M_data,
                                           __assumed | _Waiter_bit,
                                           __has_timeout, __s, __ns);
          // Fetch the current value after waiting (clears _Waiter_bit).
          __assumed = _M_load(__mo);
          if (!__ret || ((__operand == __assumed) == __equal))
            return __assumed;
          // TODO adapt wait time
        }
    }

    // Returns the operand's value if equal is true or a different value if
    // equal is false.
    // The assumed value is the caller's assumption about the current value
    // when making the call.
    unsigned
    _M_load_and_test(unsigned __assumed, unsigned __operand,
        bool __equal, memory_order __mo)
    {
      return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
                                    false, {}, {});
    }

    // If a timeout occurs, returns a current value after the timeout;
    // otherwise, returns the operand's value if equal is true or a different
    // value if equal is false.
    // The assumed value is the caller's assumption about the current value
    // when making the call.
    template<typename _Dur>
    unsigned
    _M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
        bool __equal, memory_order __mo,
        const chrono::time_point<std::chrono::system_clock, _Dur>& __atime)
    {
      auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
      auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
      // XXX correct?
      return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
          true, __s.time_since_epoch(), __ns);
    }

    template<typename _Dur>
    unsigned
    _M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
        bool __equal, memory_order __mo,
        const chrono::time_point<std::chrono::steady_clock, _Dur>& __atime)
    {
      auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
      auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
      // XXX correct?
      return _M_load_and_test_until_steady(__assumed, __operand, __equal, __mo,
          true, __s.time_since_epoch(), __ns);
    }

  public:

    _GLIBCXX_ALWAYS_INLINE unsigned
    _M_load_when_not_equal(unsigned __val, memory_order __mo)
    {
      unsigned __i = _M_load(__mo);
      if ((__i & ~_Waiter_bit) != __val)
        return (__i & ~_Waiter_bit);
      // TODO Spin-wait first.
      return _M_load_and_test(__i, __val, false, __mo);
    }

    _GLIBCXX_ALWAYS_INLINE void
    _M_load_when_equal(unsigned __val, memory_order __mo)
    {
      unsigned __i = _M_load(__mo);
      if ((__i & ~_Waiter_bit) == __val)
        return;
      // TODO Spin-wait first.
      _M_load_and_test(__i, __val, true, __mo);
    }

    // Returns false iff a timeout occurred.
    template<typename _Rep, typename _Period>
      _GLIBCXX_ALWAYS_INLINE bool
      _M_load_when_equal_for(unsigned __val, memory_order __mo,
          const chrono::duration<_Rep, _Period>& __rtime)
      {
        using __dur = typename __clock_t::duration;
        return _M_load_when_equal_until(__val, __mo,
                    __clock_t::now() + chrono::__detail::ceil<__dur>(__rtime));
      }

    // Returns false iff a timeout occurred.
    template<typename _Clock, typename _Duration>
      _GLIBCXX_ALWAYS_INLINE bool
      _M_load_when_equal_until(unsigned __val, memory_order __mo,
          const chrono::time_point<_Clock, _Duration>& __atime)
      {
        typename _Clock::time_point __c_entry = _Clock::now();
        do {
          const __clock_t::time_point __s_entry = __clock_t::now();
          const auto __delta = __atime - __c_entry;
          const auto __s_atime = __s_entry +
              chrono::__detail::ceil<__clock_t::duration>(__delta);
          if (_M_load_when_equal_until(__val, __mo, __s_atime))
            return true;
          __c_entry = _Clock::now();
        } while (__c_entry < __atime);
        return false;
      }

    // Returns false iff a timeout occurred.
    template<typename _Duration>
    _GLIBCXX_ALWAYS_INLINE bool
    _M_load_when_equal_until(unsigned __val, memory_order __mo,
        const chrono::time_point<std::chrono::system_clock, _Duration>& __atime)
    {
      unsigned __i = _M_load(__mo);
      if ((__i & ~_Waiter_bit) == __val)
        return true;
      // TODO Spin-wait first.  Ignore effect on timeout.
      __i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
      return (__i & ~_Waiter_bit) == __val;
    }

    // Returns false iff a timeout occurred.
    template<typename _Duration>
    _GLIBCXX_ALWAYS_INLINE bool
    _M_load_when_equal_until(unsigned __val, memory_order __mo,
        const chrono::time_point<std::chrono::steady_clock, _Duration>& __atime)
    {
      unsigned __i = _M_load(__mo);
      if ((__i & ~_Waiter_bit) == __val)
        return true;
      // TODO Spin-wait first.  Ignore effect on timeout.
      __i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
      return (__i & ~_Waiter_bit) == __val;
    }

    _GLIBCXX_ALWAYS_INLINE void
    _M_store_notify_all(unsigned __val, memory_order __mo)
    {
      unsigned* __futex = (unsigned *)(void *)&_M_data;
      if (_M_data.exchange(__val, __mo) & _Waiter_bit)
        _M_futex_notify_all(__futex);
    }
  };

#else // ! (_GLIBCXX_HAVE_LINUX_FUTEX && ATOMIC_INT_LOCK_FREE > 1)

  // If futexes are not available, use a mutex and a condvar to wait.
  // Because we access the data only within critical sections, all accesses
  // are sequentially consistent; thus, we satisfy any provided memory_order.
  template <unsigned _Waiter_bit = 0x80000000>
  class __atomic_futex_unsigned
  {
    typedef chrono::system_clock __clock_t;

    unsigned _M_data;
    mutex _M_mutex;
    condition_variable _M_condvar;

  public:
    explicit
    __atomic_futex_unsigned(unsigned __data) : _M_data(__data)
    { }

    _GLIBCXX_ALWAYS_INLINE unsigned
    _M_load(memory_order __mo)
    {
      unique_lock<mutex> __lock(_M_mutex);
      return _M_data;
    }

    _GLIBCXX_ALWAYS_INLINE unsigned
    _M_load_when_not_equal(unsigned __val, memory_order __mo)
    {
      unique_lock<mutex> __lock(_M_mutex);
      while (_M_data == __val)
        _M_condvar.wait(__lock);
      return _M_data;
    }

    _GLIBCXX_ALWAYS_INLINE void
    _M_load_when_equal(unsigned __val, memory_order __mo)
    {
      unique_lock<mutex> __lock(_M_mutex);
      while (_M_data != __val)
        _M_condvar.wait(__lock);
    }

    template<typename _Rep, typename _Period>
      _GLIBCXX_ALWAYS_INLINE bool
      _M_load_when_equal_for(unsigned __val, memory_order __mo,
          const chrono::duration<_Rep, _Period>& __rtime)
      {
        unique_lock<mutex> __lock(_M_mutex);
        return _M_condvar.wait_for(__lock, __rtime,
                                   [&] { return _M_data == __val;});
      }

    template<typename _Clock, typename _Duration>
      _GLIBCXX_ALWAYS_INLINE bool
      _M_load_when_equal_until(unsigned __val, memory_order __mo,
          const chrono::time_point<_Clock, _Duration>& __atime)
      {
        unique_lock<mutex> __lock(_M_mutex);
        return _M_condvar.wait_until(__lock, __atime,
                                     [&] { return _M_data == __val;});
      }

    _GLIBCXX_ALWAYS_INLINE void
    _M_store_notify_all(unsigned __val, memory_order __mo)
    {
      unique_lock<mutex> __lock(_M_mutex);
      _M_data = __val;
      _M_condvar.notify_all();
    }
  };

#endif // _GLIBCXX_HAVE_LINUX_FUTEX && ATOMIC_INT_LOCK_FREE > 1
#endif // _GLIBCXX_HAS_GTHREADS

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace std

#endif
Revision:	1166
Committed:	Tue Oct 26 14:22:36 2021 UTC (4 years ago) by rossy
Content type:	text/x-chdr
File size:	12394 byte(s)
Log Message:	Daodan: Replace MinGW build env with an up-to-date MSYS2 env
#	Content
1	// -- C++ -- header.
2
3	// Copyright (C) 2015-2021 Free Software Foundation, Inc.
4	//
5	// This file is part of the GNU ISO C++ Library. This library is free
6	// software; you can redistribute it and/or modify it under the
7	// terms of the GNU General Public License as published by the
8	// Free Software Foundation; either version 3, or (at your option)
9	// any later version.
10
11	// This library is distributed in the hope that it will be useful,
12	// but WITHOUT ANY WARRANTY; without even the implied warranty of
13	// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	// GNU General Public License for more details.
15
16	// Under Section 7 of GPL version 3, you are granted additional
17	// permissions described in the GCC Runtime Library Exception, version
18	// 3.1, as published by the Free Software Foundation.
19
20	// You should have received a copy of the GNU General Public License and
21	// a copy of the GCC Runtime Library Exception along with this program;
22	// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23	// <http://www.gnu.org/licenses/>.
24
25	/** @file bits/atomic_futex.h
26	* This is an internal header file, included by other library headers.
27	* Do not attempt to use it directly.
28	*/
29
30	#ifndef _GLIBCXX_ATOMIC_FUTEX_H
31	#define _GLIBCXX_ATOMIC_FUTEX_H 1
32
33	#pragma GCC system_header
34
35	#include <bits/c++config.h>
36	#include <atomic>
37	#include <chrono>
38	#if ! (defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1)
39	#include <mutex>
40	#include <condition_variable>
41	#endif
42
43	#ifndef _GLIBCXX_ALWAYS_INLINE
44	#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
45	#endif
46
47	namespace std _GLIBCXX_VISIBILITY(default)
48	{
49	_GLIBCXX_BEGIN_NAMESPACE_VERSION
50
51	#ifdef _GLIBCXX_HAS_GTHREADS
52	#if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
53	struct __atomic_futex_unsigned_base
54	{
55	// __s and __ns are measured against CLOCK_REALTIME. Returns false
56	// iff a timeout occurred.
57	bool
58	_M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout,
59	chrono::seconds __s, chrono::nanoseconds __ns);
60
61	// __s and __ns are measured against CLOCK_MONOTONIC. Returns
62	// false iff a timeout occurred.
63	bool
64	_M_futex_wait_until_steady(unsigned *__addr, unsigned __val,
65	bool __has_timeout, chrono::seconds __s, chrono::nanoseconds __ns);
66
67	// This can be executed after the object has been destroyed.
68	static void _M_futex_notify_all(unsigned* __addr);
69	};
70
71	template <unsigned _Waiter_bit = 0x80000000>
72	class __atomic_futex_unsigned : __atomic_futex_unsigned_base
73	{
74	typedef chrono::steady_clock __clock_t;
75
76	// This must be lock-free and at offset 0.
77	atomic<unsigned> _M_data;
78
79	public:
80	explicit
81	__atomic_futex_unsigned(unsigned __data) : _M_data(__data)
82	{ }
83
84	_GLIBCXX_ALWAYS_INLINE unsigned
85	_M_load(memory_order __mo)
86	{
87	return _M_data.load(__mo) & ~_Waiter_bit;
88	}
89
90	private:
91	// If a timeout occurs, returns a current value after the timeout;
92	// otherwise, returns the operand's value if equal is true or a different
93	// value if equal is false.
94	// The assumed value is the caller's assumption about the current value
95	// when making the call.
96	// __s and __ns are measured against CLOCK_REALTIME.
97	unsigned
98	_M_load_and_test_until(unsigned __assumed, unsigned __operand,
99	bool __equal, memory_order __mo, bool __has_timeout,
100	chrono::seconds __s, chrono::nanoseconds __ns)
101	{
102	for (;;)
103	{
104	// Don't bother checking the value again because we expect the caller
105	// to have done it recently.
106	// memory_order_relaxed is sufficient because we can rely on just the
107	// modification order (store_notify uses an atomic RMW operation too),
108	// and the futex syscalls synchronize between themselves.
109	_M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
110	bool __ret = _M_futex_wait_until((unsigned)(void)&_M_data,
111	__assumed \| _Waiter_bit,
112	__has_timeout, __s, __ns);
113	// Fetch the current value after waiting (clears _Waiter_bit).
114	__assumed = _M_load(__mo);
115	if (!__ret \|\| ((__operand == __assumed) == __equal))
116	return __assumed;
117	// TODO adapt wait time
118	}
119	}
120
121	// If a timeout occurs, returns a current value after the timeout;
122	// otherwise, returns the operand's value if equal is true or a different
123	// value if equal is false.
124	// The assumed value is the caller's assumption about the current value
125	// when making the call.
126	// __s and __ns are measured against CLOCK_MONOTONIC.
127	unsigned
128	_M_load_and_test_until_steady(unsigned __assumed, unsigned __operand,
129	bool __equal, memory_order __mo, bool __has_timeout,
130	chrono::seconds __s, chrono::nanoseconds __ns)
131	{
132	for (;;)
133	{
134	// Don't bother checking the value again because we expect the caller
135	// to have done it recently.
136	// memory_order_relaxed is sufficient because we can rely on just the
137	// modification order (store_notify uses an atomic RMW operation too),
138	// and the futex syscalls synchronize between themselves.
139	_M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
140	bool __ret = _M_futex_wait_until_steady((unsigned)(void)&_M_data,
141	__assumed \| _Waiter_bit,
142	__has_timeout, __s, __ns);
143	// Fetch the current value after waiting (clears _Waiter_bit).
144	__assumed = _M_load(__mo);
145	if (!__ret \|\| ((__operand == __assumed) == __equal))
146	return __assumed;
147	// TODO adapt wait time
148	}
149	}
150
151	// Returns the operand's value if equal is true or a different value if
152	// equal is false.
153	// The assumed value is the caller's assumption about the current value
154	// when making the call.
155	unsigned
156	_M_load_and_test(unsigned __assumed, unsigned __operand,
157	bool __equal, memory_order __mo)
158	{
159	return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
160	false, {}, {});
161	}
162
163	// If a timeout occurs, returns a current value after the timeout;
164	// otherwise, returns the operand's value if equal is true or a different
165	// value if equal is false.
166	// The assumed value is the caller's assumption about the current value
167	// when making the call.
168	template<typename _Dur>
169	unsigned
170	_M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
171	bool __equal, memory_order __mo,
172	const chrono::time_point<std::chrono::system_clock, _Dur>& __atime)
173	{
174	auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
175	auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
176	// XXX correct?
177	return _M_load_and_test_until(__assumed, __operand, __equal, __mo,
178	true, __s.time_since_epoch(), __ns);
179	}
180
181	template<typename _Dur>
182	unsigned
183	_M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
184	bool __equal, memory_order __mo,
185	const chrono::time_point<std::chrono::steady_clock, _Dur>& __atime)
186	{
187	auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
188	auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
189	// XXX correct?
190	return _M_load_and_test_until_steady(__assumed, __operand, __equal, __mo,
191	true, __s.time_since_epoch(), __ns);
192	}
193
194	public:
195
196	_GLIBCXX_ALWAYS_INLINE unsigned
197	_M_load_when_not_equal(unsigned __val, memory_order __mo)
198	{
199	unsigned __i = _M_load(__mo);
200	if ((__i & ~_Waiter_bit) != __val)
201	return (__i & ~_Waiter_bit);
202	// TODO Spin-wait first.
203	return _M_load_and_test(__i, __val, false, __mo);
204	}
205
206	_GLIBCXX_ALWAYS_INLINE void
207	_M_load_when_equal(unsigned __val, memory_order __mo)
208	{
209	unsigned __i = _M_load(__mo);
210	if ((__i & ~_Waiter_bit) == __val)
211	return;
212	// TODO Spin-wait first.
213	_M_load_and_test(__i, __val, true, __mo);
214	}
215
216	// Returns false iff a timeout occurred.
217	template<typename _Rep, typename _Period>
218	_GLIBCXX_ALWAYS_INLINE bool
219	_M_load_when_equal_for(unsigned __val, memory_order __mo,
220	const chrono::duration<_Rep, _Period>& __rtime)
221	{
222	using __dur = typename __clock_t::duration;
223	return _M_load_when_equal_until(__val, __mo,
224	__clock_t::now() + chrono::__detail::ceil<__dur>(__rtime));
225	}
226
227	// Returns false iff a timeout occurred.
228	template<typename _Clock, typename _Duration>
229	_GLIBCXX_ALWAYS_INLINE bool
230	_M_load_when_equal_until(unsigned __val, memory_order __mo,
231	const chrono::time_point<_Clock, _Duration>& __atime)
232	{
233	typename _Clock::time_point __c_entry = _Clock::now();
234	do {
235	const __clock_t::time_point __s_entry = __clock_t::now();
236	const auto __delta = __atime - __c_entry;
237	const auto __s_atime = __s_entry +
238	chrono::__detail::ceil<__clock_t::duration>(__delta);
239	if (_M_load_when_equal_until(__val, __mo, __s_atime))
240	return true;
241	__c_entry = _Clock::now();
242	} while (__c_entry < __atime);
243	return false;
244	}
245
246	// Returns false iff a timeout occurred.
247	template<typename _Duration>
248	_GLIBCXX_ALWAYS_INLINE bool
249	_M_load_when_equal_until(unsigned __val, memory_order __mo,
250	const chrono::time_point<std::chrono::system_clock, _Duration>& __atime)
251	{
252	unsigned __i = _M_load(__mo);
253	if ((__i & ~_Waiter_bit) == __val)
254	return true;
255	// TODO Spin-wait first. Ignore effect on timeout.
256	__i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
257	return (__i & ~_Waiter_bit) == __val;
258	}
259
260	// Returns false iff a timeout occurred.
261	template<typename _Duration>
262	_GLIBCXX_ALWAYS_INLINE bool
263	_M_load_when_equal_until(unsigned __val, memory_order __mo,
264	const chrono::time_point<std::chrono::steady_clock, _Duration>& __atime)
265	{
266	unsigned __i = _M_load(__mo);
267	if ((__i & ~_Waiter_bit) == __val)
268	return true;
269	// TODO Spin-wait first. Ignore effect on timeout.
270	__i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
271	return (__i & ~_Waiter_bit) == __val;
272	}
273
274	_GLIBCXX_ALWAYS_INLINE void
275	_M_store_notify_all(unsigned __val, memory_order __mo)
276	{
277	unsigned* __futex = (unsigned )(void )&_M_data;
278	if (_M_data.exchange(__val, __mo) & _Waiter_bit)
279	_M_futex_notify_all(__futex);
280	}
281	};
282
283	#else // ! (_GLIBCXX_HAVE_LINUX_FUTEX && ATOMIC_INT_LOCK_FREE > 1)
284
285	// If futexes are not available, use a mutex and a condvar to wait.
286	// Because we access the data only within critical sections, all accesses
287	// are sequentially consistent; thus, we satisfy any provided memory_order.
288	template <unsigned _Waiter_bit = 0x80000000>
289	class __atomic_futex_unsigned
290	{
291	typedef chrono::system_clock __clock_t;
292
293	unsigned _M_data;
294	mutex _M_mutex;
295	condition_variable _M_condvar;
296
297	public:
298	explicit
299	__atomic_futex_unsigned(unsigned __data) : _M_data(__data)
300	{ }
301
302	_GLIBCXX_ALWAYS_INLINE unsigned
303	_M_load(memory_order __mo)
304	{
305	unique_lock<mutex> __lock(_M_mutex);
306	return _M_data;
307	}
308
309	_GLIBCXX_ALWAYS_INLINE unsigned
310	_M_load_when_not_equal(unsigned __val, memory_order __mo)
311	{
312	unique_lock<mutex> __lock(_M_mutex);
313	while (_M_data == __val)
314	_M_condvar.wait(__lock);
315	return _M_data;
316	}
317
318	_GLIBCXX_ALWAYS_INLINE void
319	_M_load_when_equal(unsigned __val, memory_order __mo)
320	{
321	unique_lock<mutex> __lock(_M_mutex);
322	while (_M_data != __val)
323	_M_condvar.wait(__lock);
324	}
325
326	template<typename _Rep, typename _Period>
327	_GLIBCXX_ALWAYS_INLINE bool
328	_M_load_when_equal_for(unsigned __val, memory_order __mo,
329	const chrono::duration<_Rep, _Period>& __rtime)
330	{
331	unique_lock<mutex> __lock(_M_mutex);
332	return _M_condvar.wait_for(__lock, __rtime,
333	[&] { return _M_data == __val;});
334	}
335
336	template<typename _Clock, typename _Duration>
337	_GLIBCXX_ALWAYS_INLINE bool
338	_M_load_when_equal_until(unsigned __val, memory_order __mo,
339	const chrono::time_point<_Clock, _Duration>& __atime)
340	{
341	unique_lock<mutex> __lock(_M_mutex);
342	return _M_condvar.wait_until(__lock, __atime,
343	[&] { return _M_data == __val;});
344	}
345
346	_GLIBCXX_ALWAYS_INLINE void
347	_M_store_notify_all(unsigned __val, memory_order __mo)
348	{
349	unique_lock<mutex> __lock(_M_mutex);
350	_M_data = __val;
351	_M_condvar.notify_all();
352	}
353	};
354
355	#endif // _GLIBCXX_HAVE_LINUX_FUTEX && ATOMIC_INT_LOCK_FREE > 1
356	#endif // _GLIBCXX_HAS_GTHREADS
357
358	_GLIBCXX_END_NAMESPACE_VERSION
359	} // namespace std
360
361	#endif