diff --git a/Core/include/Acts/Utilities/Any.hpp b/Core/include/Acts/Utilities/Any.hpp index 12d073fe291..9fafa988a15 100644 --- a/Core/include/Acts/Utilities/Any.hpp +++ b/Core/include/Acts/Utilities/Any.hpp @@ -110,6 +110,11 @@ static _AnyAllocationReporter s_reporter; } while (0) static constexpr bool kAnyNoexcept = true; #endif + +/// Throws @c std::bad_any_cast. Defined out of line in Any.cpp so the throw +/// stays out of the accessors and does not stop them from being inlined. +[[noreturn]] void throwBadAnyCast(); + } // namespace detail /// @addtogroup utilities @@ -236,14 +241,14 @@ class AnyBase : public AnyBaseAll { T& as() { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - if (m_handler == nullptr || m_handler->typeHash != typeHash()) { - throw std::bad_any_cast{}; + if (!holds()) { + detail::throwBadAnyCast(); } _ACTS_ANY_VERBOSE("Get as " << (m_handler->heapAllocated ? "heap" : "local")); - return *std::bit_cast(dataPtr()); + return *std::bit_cast(dataPtrFor()); } /// Get const reference to stored value of specified type @@ -254,14 +259,14 @@ class AnyBase : public AnyBaseAll { const T& as() const { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - if (m_handler == nullptr || m_handler->typeHash != typeHash()) { - throw std::bad_any_cast{}; + if (!holds()) { + detail::throwBadAnyCast(); } _ACTS_ANY_VERBOSE("Get as " << (m_handler->heapAllocated ? "heap" : "local")); - return *std::bit_cast(dataPtr()); + return *std::bit_cast(dataPtrFor()); } /// Get pointer to stored value of specified type @@ -272,10 +277,10 @@ class AnyBase : public AnyBaseAll { T* asPtr() { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - if (m_handler == nullptr || m_handler->typeHash != typeHash()) { + if (!holds()) { return nullptr; } - return std::bit_cast(dataPtr()); + return std::bit_cast(dataPtrFor()); } /// Get const pointer to stored value of specified type @@ -286,10 +291,10 @@ class AnyBase : public AnyBaseAll { const T* asPtr() const { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - if (m_handler == nullptr || m_handler->typeHash != typeHash()) { + if (!holds()) { return nullptr; } - return std::bit_cast(dataPtr()); + return std::bit_cast(dataPtrFor()); } /// Move the stored value out. Leaves this Any empty. @@ -300,10 +305,10 @@ class AnyBase : public AnyBaseAll { T take() { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - if (m_handler == nullptr || m_handler->typeHash != typeHash()) { - throw std::bad_any_cast{}; + if (!holds()) { + detail::throwBadAnyCast(); } - T* ptr = std::bit_cast(dataPtr()); + T* ptr = std::bit_cast(dataPtrFor()); T value = std::move(*ptr); destroy(); return value; @@ -418,8 +423,7 @@ class AnyBase : public AnyBaseAll { // At this point they can't be equal and nullptr, so it's safe to // dereference - if (m_handler == other.m_handler && - m_handler->typeHash == other.m_handler->typeHash) { + if (m_handler == other.m_handler) { // same type, but checked before they're not both nullptr move(std::move(other)); } else { @@ -452,7 +456,7 @@ class AnyBase : public AnyBaseAll { bool is() const { static_assert(std::is_same_v>, "Please pass the raw type, no const or ref"); - return m_handler != nullptr && m_handler->typeHash == typeHash(); + return holds(); } // The base accessors below are member templates on a dummy @c B defaulting to @@ -527,6 +531,37 @@ class AnyBase : public AnyBaseAll { } private: + // The handler is a per-type singleton, so a pointer comparison settles the + // common case. The @c type_info comparison covers handlers duplicated across + // shared objects, where the pointers differ but the type does not. + template + bool holds() const { + if (m_handler == makeHandler()) [[likely]] { + return true; + } + return m_handler != nullptr && *m_handler->typeInfo == typeid(T); + } + + // T is known statically here, so unlike dataPtr() this needs no load of + // m_handler->heapAllocated and no branch on it. + template + void* dataPtrFor() { + if constexpr (heapAllocated()) { + return *std::bit_cast(m_data.data()); + } else { + return std::bit_cast(m_data.data()); + } + } + + template + const void* dataPtrFor() const { + if constexpr (heapAllocated()) { + return *std::bit_cast(m_data.data()); + } else { + return std::bit_cast(m_data.data()); + } + } + void* dataPtr() { if (m_handler->heapAllocated) { return *std::bit_cast(m_data.data()); @@ -559,53 +594,64 @@ class AnyBase : public AnyBaseAll { void* (*copyConstruct)(const void* from, void* to) = nullptr; void (*copy)(const void* from, void* to) = nullptr; bool heapAllocated{false}; - std::uint64_t typeHash{0}; const std::type_info* typeInfo{nullptr}; }; + // Constant so that the singleton below is constant-initialized and needs no + // thread-safe-static guard. template - static const Handler* makeHandler() { - static_assert(!std::is_base_of_v>, - "Cannot wrap Any in Any"); - static const Handler static_handler = []() { - Handler h; - h.heapAllocated = heapAllocated(); - if constexpr (!std::is_trivially_destructible_v || - heapAllocated()) { - h.destroy = &destroyImpl; - } - if constexpr (!heapAllocated() && - !std::is_trivially_move_constructible_v) { - h.moveConstruct = &moveConstructImpl; - } - if constexpr (!heapAllocated() && - !std::is_trivially_move_assignable_v) { - h.move = &moveImpl; - } - if constexpr (std::is_copy_constructible_v && - (!std::is_trivially_copy_constructible_v || - heapAllocated())) { - h.copyConstruct = ©ConstructImpl; - } + static constexpr Handler makeHandlerValue() { + Handler h; + h.heapAllocated = heapAllocated(); + if constexpr (!std::is_trivially_destructible_v || heapAllocated()) { + h.destroy = &destroyImpl; + } + if constexpr (!heapAllocated() && + !std::is_trivially_move_constructible_v) { + h.moveConstruct = &moveConstructImpl; + } + if constexpr (!heapAllocated() && + !std::is_trivially_move_assignable_v) { + h.move = &moveImpl; + } + if constexpr (std::is_copy_constructible_v && + (!std::is_trivially_copy_constructible_v || + heapAllocated())) { + h.copyConstruct = ©ConstructImpl; + } - if constexpr (std::is_copy_assignable_v && - (!std::is_trivially_copy_assignable_v || - heapAllocated())) { - h.copy = ©Impl; - } + if constexpr (std::is_copy_assignable_v && + (!std::is_trivially_copy_assignable_v || + heapAllocated())) { + h.copy = ©Impl; + } - if constexpr (!std::is_void_v) { - h.upcast = [](void* p) -> Base* { - return static_cast(static_cast(p)); - }; - h.upcastConst = [](const void* p) -> const Base* { - return static_cast(static_cast(p)); - }; - } + if constexpr (!std::is_void_v) { + h.upcast = [](void* p) -> Base* { + return static_cast(static_cast(p)); + }; + h.upcastConst = [](const void* p) -> const Base* { + return static_cast(static_cast(p)); + }; + } - h.typeHash = typeHash(); - h.typeInfo = &typeid(T); + h.typeInfo = &typeid(T); + return h; + } + + template + static const Handler* makeHandler() { + static_assert(!std::is_base_of_v>, + "Cannot wrap Any in Any"); + static constexpr Handler static_handler = makeHandlerValue(); + +#if defined(_ACTS_ANY_ENABLE_DEBUG) + // Reporting has to happen here rather than in makeHandlerValue, which is + // constant evaluated. Only compiled in when debug output is enabled, so it + // does not put a guard variable on the hot path. + [[maybe_unused]] static const bool reported = []() { + const Handler& h = static_handler; _ACTS_ANY_DEBUG("Type: " << typeid(T).name()); _ACTS_ANY_DEBUG(" -> destroy: " << h.destroy); _ACTS_ANY_DEBUG(" -> moveConstruct: " << h.moveConstruct); @@ -614,9 +660,10 @@ class AnyBase : public AnyBaseAll { _ACTS_ANY_DEBUG(" -> copy: " << h.copy); _ACTS_ANY_DEBUG( " -> heapAllocated: " << (h.heapAllocated ? "yes" : "no")); - - return h; + return true; }(); +#endif + return &static_handler; } @@ -628,6 +675,14 @@ class AnyBase : public AnyBaseAll { template T* constructValue(Args&&... args) { if constexpr (!heapAllocated()) { + if constexpr (std::is_empty_v) { + // An empty object occupies one byte of the buffer that its constructor + // never writes. Write it before the object's lifetime starts, so the + // trivial copy/move paths, which copy the buffer as a fixed-size + // block, do not read a buffer that was never written at all. Nothing + // is emitted for types that carry state. + m_data[0] = std::byte{0}; + } // construct into local buffer auto* ptr = new (m_data.data()) T(std::forward(args)...); _ACTS_ANY_VERBOSE("Construct local (this=" diff --git a/Core/include/Acts/Utilities/HashedString.hpp b/Core/include/Acts/Utilities/HashedString.hpp index 7588ae6f0f6..ceada806b36 100644 --- a/Core/include/Acts/Utilities/HashedString.hpp +++ b/Core/include/Acts/Utilities/HashedString.hpp @@ -78,6 +78,8 @@ constexpr HashedString operator""_hash(char const* s, std::size_t count) { /// Hash for a type. Since it's not possible to hash a type at compile-time, /// this function returns a runtime hash but caches it in a static variable. +/// The hash is derived from the mangled name, which is fixed by the Itanium +/// ABI and therefore identical between GCC and Clang. /// @tparam T Type to hash /// @return Hashed string representation template diff --git a/Core/src/Utilities/Any.cpp b/Core/src/Utilities/Any.cpp new file mode 100644 index 00000000000..9d727b49d07 --- /dev/null +++ b/Core/src/Utilities/Any.cpp @@ -0,0 +1,19 @@ +// This file is part of the ACTS project. +// +// Copyright (C) 2016 CERN for the benefit of the ACTS project +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "Acts/Utilities/Any.hpp" + +#include + +namespace Acts::detail { + +void throwBadAnyCast() { + throw std::bad_any_cast{}; +} + +} // namespace Acts::detail diff --git a/Core/src/Utilities/CMakeLists.txt b/Core/src/Utilities/CMakeLists.txt index 6ad19700070..e13b9df8204 100644 --- a/Core/src/Utilities/CMakeLists.txt +++ b/Core/src/Utilities/CMakeLists.txt @@ -9,6 +9,7 @@ target_sources( Intersection.cpp IAxis.cpp GraphViz.cpp + Any.cpp ProtoAxis.cpp ScopedTimer.cpp TransformComparator.cpp diff --git a/Tests/Benchmarks/SourceLinkBenchmark.cpp b/Tests/Benchmarks/SourceLinkBenchmark.cpp index 81bf1faf535..e4fea071b7a 100644 --- a/Tests/Benchmarks/SourceLinkBenchmark.cpp +++ b/Tests/Benchmarks/SourceLinkBenchmark.cpp @@ -6,12 +6,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +#include "Acts/EventData/SourceLink.hpp" #include "Acts/EventData/VectorMultiTrajectory.hpp" #include "Acts/Geometry/GeometryIdentifier.hpp" #include "ActsTests/CommonHelpers/BenchmarkTools.hpp" +#include #include #include +#include using namespace Acts; using namespace ActsTests; @@ -93,6 +96,42 @@ int main(int /*argc*/, char** /*argv[]*/) { inputs); std::cout << copyMoveConstructSourceLink << std::endl; + // The track finding unpacks a source link for every measurement candidate, + // in the calibrator, the measurement selector and the surface accessor. + std::cout << "Unpack source link with get" << std::endl; + auto unpackGet = microBenchmark( + [&](const SourceLink& input) { + return input.get().index(); + }, + inputs); + std::cout << unpackGet << std::endl; + + std::cout << "Unpack source link with getPtr" << std::endl; + auto unpackGetPtr = microBenchmark( + [&](const SourceLink& input) { + return input.getPtr()->index(); + }, + inputs); + std::cout << unpackGetPtr << std::endl; + + std::cout << "Unpack geometry id from source link" << std::endl; + auto unpackGeometryId = microBenchmark( + [&](const SourceLink& input) { + return input.get().geometryId(); + }, + inputs); + std::cout << unpackGeometryId << std::endl; + + // Shape of the track finding inner loop: wrap, then immediately unpack. + std::cout << "Construct and unpack source link" << std::endl; + auto constructAndUnpack = microBenchmark( + [&]() { + SourceLink sl{bsl}; + return sl.get().index(); + }, + n); + std::cout << constructAndUnpack << std::endl; + std::cout << "Optional assignment" << std::endl; auto opt_assignment = microBenchmark( [&]() {