/* Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #ifndef __TBB_detail__utils_H #define __TBB_detail__utils_H #include #include #include #include "_config.h" #include "_assert.h" #include "_machine.h" namespace tbb { namespace detail { inline namespace d0 { //! Utility template function to prevent "unused" warnings by various compilers. template void suppress_unused_warning(T&&...) {} //! Compile-time constant that is upper bound on cache line/sector size. /** It should be used only in situations where having a compile-time upper bound is more useful than a run-time exact answer. @ingroup memory_allocation */ constexpr size_t max_nfs_size = 128; constexpr std::size_t max_nfs_size_exp = 7; static_assert(1 << max_nfs_size_exp == max_nfs_size, "max_nfs_size_exp must be a log2(max_nfs_size)"); //! Class that implements exponential backoff. class atomic_backoff { //! Time delay, in units of "pause" instructions. /** Should be equal to approximately the number of "pause" instructions that take the same time as an context switch. Must be a power of two.*/ static constexpr std::int32_t LOOPS_BEFORE_YIELD = 16; std::int32_t count; public: // In many cases, an object of this type is initialized eagerly on hot path, // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } // For this reason, the construction cost must be very small! atomic_backoff() : count(1) {} // This constructor pauses immediately; do not use on hot paths! atomic_backoff(bool) : count(1) { pause(); } //! No Copy atomic_backoff(const atomic_backoff&) = delete; atomic_backoff& operator=(const atomic_backoff&) = delete; //! Pause for a while. void pause() { if (count <= LOOPS_BEFORE_YIELD) { machine_pause(count); // Pause twice as long the next time. count *= 2; } else { // Pause is so long that we might as well yield CPU to scheduler. yield(); } } //! Pause for a few times and return false if saturated. bool bounded_pause() { machine_pause(count); if (count < LOOPS_BEFORE_YIELD) { // Pause twice as long the next time. count *= 2; return true; } else { return false; } } void reset() { count = 1; } }; //! Spin WHILE the condition is true. /** T and U should be comparable types. */ template T spin_wait_while(const std::atomic& location, C comp, std::memory_order order) { atomic_backoff backoff; T snapshot = location.load(order); while (comp(snapshot)) { backoff.pause(); snapshot = location.load(order); } return snapshot; } //! Spin WHILE the value of the variable is equal to a given value /** T and U should be comparable types. */ template T spin_wait_while_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { return spin_wait_while(location, [&value](T t) { return t == value; }, order); } //! Spin UNTIL the value of the variable is equal to a given value /** T and U should be comparable types. */ template T spin_wait_until_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { return spin_wait_while(location, [&value](T t) { return t != value; }, order); } //! Spin UNTIL the condition returns true or spinning time is up. /** Returns what the passed functor returned last time it was invoked. */ template bool timed_spin_wait_until(Condition condition) { // 32 pauses + 32 yields are meausered as balanced spin time before sleep. bool finish = condition(); for (int i = 1; !finish && i < 32; finish = condition(), i *= 2) { machine_pause(i); } for (int i = 32; !finish && i < 64; finish = condition(), ++i) { yield(); } return finish; } template std::uintptr_t log2(T in) { __TBB_ASSERT(in > 0, "The logarithm of a non-positive value is undefined."); return machine_log2(in); } template T reverse_bits(T src) { return machine_reverse_bits(src); } template T reverse_n_bits(T src, std::size_t n) { __TBB_ASSERT(n != 0, "Reverse for 0 bits is undefined behavior."); return reverse_bits(src) >> (number_of_bits() - n); } // A function to check if passed integer is a power of two template constexpr bool is_power_of_two( IntegerType arg ) { static_assert(std::is_integral::value, "An argument for is_power_of_two should be integral type"); return arg && (0 == (arg & (arg - 1))); } // A function to determine if passed integer is a power of two // at least as big as another power of two, i.e. for strictly positive i and j, // with j being a power of two, determines whether i==j< constexpr bool is_power_of_two_at_least(ArgIntegerType arg, DivisorIntegerType divisor) { // Divisor should be a power of two static_assert(std::is_integral::value, "An argument for is_power_of_two_at_least should be integral type"); return 0 == (arg & (arg - divisor)); } // A function to compute arg modulo divisor where divisor is a power of 2. template inline ArgIntegerType modulo_power_of_two(ArgIntegerType arg, DivisorIntegerType divisor) { __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); return arg & (divisor - 1); } //! A function to check if passed in pointer is aligned on a specific border template constexpr bool is_aligned(T* pointer, std::uintptr_t alignment) { return 0 == ((std::uintptr_t)pointer & (alignment - 1)); } #if TBB_USE_ASSERT static void* const poisoned_ptr = reinterpret_cast(-1); //! Set p to invalid pointer value. template inline void poison_pointer( T* &p ) { p = reinterpret_cast(poisoned_ptr); } template inline void poison_pointer(std::atomic& p) { p.store(reinterpret_cast(poisoned_ptr), std::memory_order_relaxed); } /** Expected to be used in assertions only, thus no empty form is defined. **/ template inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } template inline bool is_poisoned(const std::atomic& p) { return is_poisoned(p.load(std::memory_order_relaxed)); } #else template inline void poison_pointer(T&) {/*do nothing*/} #endif /* !TBB_USE_ASSERT */ template bool assert_pointer_valid(T* p, const char* comment = nullptr) { suppress_unused_warning(p, comment); __TBB_ASSERT(p != nullptr, comment); __TBB_ASSERT(!is_poisoned(p), comment); #if !(_MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER) __TBB_ASSERT(is_aligned(p, alignment == 0 ? alignof(T) : alignment), comment); #endif // Returns something to simplify assert_pointers_valid implementation. return true; } template void assert_pointers_valid(Args*... p) { // suppress_unused_warning is used as an evaluation context for the variadic pack. suppress_unused_warning(assert_pointer_valid(p)...); } //! Base class for types that should not be assigned. class no_assign { public: void operator=(const no_assign&) = delete; no_assign(const no_assign&) = default; no_assign() = default; }; //! Base class for types that should not be copied or assigned. class no_copy: no_assign { public: no_copy(const no_copy&) = delete; no_copy() = default; }; template void swap_atomics_relaxed(std::atomic& lhs, std::atomic& rhs){ T tmp = lhs.load(std::memory_order_relaxed); lhs.store(rhs.load(std::memory_order_relaxed), std::memory_order_relaxed); rhs.store(tmp, std::memory_order_relaxed); } //! One-time initialization states enum class do_once_state { uninitialized = 0, ///< No execution attempts have been undertaken yet pending, ///< A thread is executing associated do-once routine executed, ///< Do-once routine has been executed initialized = executed ///< Convenience alias }; //! One-time initialization function /** /param initializer Pointer to function without arguments The variant that returns bool is used for cases when initialization can fail and it is OK to continue execution, but the state should be reset so that the initialization attempt was repeated the next time. /param state Shared state associated with initializer that specifies its initialization state. Must be initially set to #uninitialized value (e.g. by means of default static zero initialization). **/ template void atomic_do_once( const F& initializer, std::atomic& state ) { // The loop in the implementation is necessary to avoid race when thread T2 // that arrived in the middle of initialization attempt by another thread T1 // has just made initialization possible. // In such a case T2 has to rely on T1 to initialize, but T1 may already be past // the point where it can recognize the changed conditions. do_once_state expected_state; while ( state.load( std::memory_order_acquire ) != do_once_state::executed ) { if( state.load( std::memory_order_relaxed ) == do_once_state::uninitialized ) { expected_state = do_once_state::uninitialized; #if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910 using enum_type = typename std::underlying_type::type; if( ((std::atomic&)state).compare_exchange_strong( (enum_type&)expected_state, (enum_type)do_once_state::pending ) ) { #else if( state.compare_exchange_strong( expected_state, do_once_state::pending ) ) { #endif run_initializer( initializer, state ); break; } } spin_wait_while_eq( state, do_once_state::pending ); } } // Run the initializer which can not fail template void run_initializer(const Functor& f, std::atomic& state ) { f(); state.store(do_once_state::executed, std::memory_order_release); } #if __TBB_CPP20_CONCEPTS_PRESENT template concept boolean_testable_impl = std::convertible_to; template concept boolean_testable = boolean_testable_impl && requires( T&& t ) { { !std::forward(t) } -> boolean_testable_impl; }; #if __TBB_CPP20_COMPARISONS_PRESENT struct synthesized_three_way_comparator { template auto operator()( const T1& lhs, const T2& rhs ) const requires requires { { lhs < rhs } -> boolean_testable; { rhs < lhs } -> boolean_testable; } { if constexpr (std::three_way_comparable_with) { return lhs <=> rhs; } else { if (lhs < rhs) { return std::weak_ordering::less; } if (rhs < lhs) { return std::weak_ordering::greater; } return std::weak_ordering::equivalent; } } }; // struct synthesized_three_way_comparator template using synthesized_three_way_result = decltype(synthesized_three_way_comparator{}(std::declval(), std::declval())); #endif // __TBB_CPP20_COMPARISONS_PRESENT // Check if the type T is implicitly OR explicitly convertible to U template concept relaxed_convertible_to = std::constructible_from; template concept adaptive_same_as = #if __TBB_STRICT_CONSTRAINTS std::same_as; #else std::convertible_to; #endif #endif // __TBB_CPP20_CONCEPTS_PRESENT } // namespace d0 namespace d1 { class delegate_base { public: virtual bool operator()() const = 0; virtual ~delegate_base() {} }; template class delegated_function : public delegate_base { public: delegated_function(FuncType& f) : my_func(f) {} bool operator()() const override { return my_func(); } private: FuncType &my_func; }; } // namespace d1 } // namespace detail } // namespace tbb #endif // __TBB_detail__utils_H