perf: Speed up type-erased SourceLink unpacking

Core/include/Acts/Utilities/Any.hpp

@@ -110,6 +110,11 @@
   } while (0)
 static constexpr bool kAnyNoexcept = true;
 #endif
+
+/// Throws @c std::bad_any_cast. Defined out of line in Any.cpp so the throw
+/// stays out of the accessors and does not stop them from being inlined.
+[[noreturn]] void throwBadAnyCast();
+
 }  // namespace detail
 
 /// @addtogroup utilities
@@ -236,14 +241,14 @@
   T& as() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      detail::throwBadAnyCast();
     }
 
     _ACTS_ANY_VERBOSE("Get as "
                       << (m_handler->heapAllocated ? "heap" : "local"));
 
-    return *std::bit_cast<T*>(dataPtr());
+    return *std::bit_cast<T*>(dataPtrFor<T>());
   }
 
   /// Get const reference to stored value of specified type
@@ -254,14 +259,14 @@
   const T& as() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      detail::throwBadAnyCast();
     }
 
     _ACTS_ANY_VERBOSE("Get as "
                       << (m_handler->heapAllocated ? "heap" : "local"));
 
-    return *std::bit_cast<const T*>(dataPtr());
+    return *std::bit_cast<const T*>(dataPtrFor<T>());
   }
 
   /// Get pointer to stored value of specified type
@@ -272,10 +277,10 @@
   T* asPtr() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
+    if (!holds<T>()) {
       return nullptr;
     }
-    return std::bit_cast<T*>(dataPtr());
+    return std::bit_cast<T*>(dataPtrFor<T>());
   }
 
   /// Get const pointer to stored value of specified type
@@ -286,10 +291,10 @@
   const T* asPtr() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
+    if (!holds<T>()) {
       return nullptr;
     }
-    return std::bit_cast<const T*>(dataPtr());
+    return std::bit_cast<const T*>(dataPtrFor<T>());
   }
 
   /// Move the stored value out. Leaves this Any empty.
@@ -300,10 +305,10 @@
   T take() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      detail::throwBadAnyCast();
     }
-    T* ptr = std::bit_cast<T*>(dataPtr());
+    T* ptr = std::bit_cast<T*>(dataPtrFor<T>());
     T value = std::move(*ptr);
     destroy();
     return value;
@@ -418,8 +423,7 @@
 
     // At this point they can't be equal and nullptr, so it's safe to
     // dereference
-    if (m_handler == other.m_handler &&
-        m_handler->typeHash == other.m_handler->typeHash) {
+    if (m_handler == other.m_handler) {
       // same type, but checked before they're not both nullptr
       move(std::move(other));
     } else {
@@ -452,7 +456,7 @@
   bool is() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    return m_handler != nullptr && m_handler->typeHash == typeHash<T>();
+    return holds<T>();
   }
 
   // The base accessors below are member templates on a dummy @c B defaulting to
@@ -527,6 +531,37 @@
   }
 
  private:
+  // The handler is a per-type singleton, so a pointer comparison settles the
+  // common case. The @c type_info comparison covers handlers duplicated across
+  // shared objects, where the pointers differ but the type does not.
+  template <typename T>
+  bool holds() const {
+    if (m_handler == makeHandler<T>()) [[likely]] {
+      return true;
+    }
+    return m_handler != nullptr && *m_handler->typeInfo == typeid(T);
+  }
+
+  // T is known statically here, so unlike dataPtr() this needs no load of
+  // m_handler->heapAllocated and no branch on it.
+  template <typename T>
+  void* dataPtrFor() {
+    if constexpr (heapAllocated<T>()) {
+      return *std::bit_cast<void**>(m_data.data());
+    } else {
+      return std::bit_cast<void*>(m_data.data());
+    }
+  }
+
+  template <typename T>
+  const void* dataPtrFor() const {
+    if constexpr (heapAllocated<T>()) {
+      return *std::bit_cast<void* const*>(m_data.data());
+    } else {
+      return std::bit_cast<const void*>(m_data.data());
+    }
+  }
+
   void* dataPtr() {
     if (m_handler->heapAllocated) {
       return *std::bit_cast<void**>(m_data.data());
@@ -559,53 +594,64 @@
     void* (*copyConstruct)(const void* from, void* to) = nullptr;
     void (*copy)(const void* from, void* to) = nullptr;
     bool heapAllocated{false};
-    std::uint64_t typeHash{0};
     const std::type_info* typeInfo{nullptr};
   };
 
+  // Constant so that the singleton below is constant-initialized and needs no
+  // thread-safe-static guard.
   template <typename T>
-  static const Handler* makeHandler() {
-    static_assert(!std::is_base_of_v<AnyBaseAll, std::decay_t<T>>,
-                  "Cannot wrap Any in Any");
-    static const Handler static_handler = []() {
-      Handler h;
-      h.heapAllocated = heapAllocated<T>();
-      if constexpr (!std::is_trivially_destructible_v<T> ||
-                    heapAllocated<T>()) {
-        h.destroy = &destroyImpl<T>;
-      }
-      if constexpr (!heapAllocated<T>() &&
-                    !std::is_trivially_move_constructible_v<T>) {
-        h.moveConstruct = &moveConstructImpl<T>;
-      }
-      if constexpr (!heapAllocated<T>() &&
-                    !std::is_trivially_move_assignable_v<T>) {
-        h.move = &moveImpl<T>;
-      }
-      if constexpr (std::is_copy_constructible_v<T> &&
-                    (!std::is_trivially_copy_constructible_v<T> ||
-                     heapAllocated<T>())) {
-        h.copyConstruct = &copyConstructImpl<T>;
-      }
+  static constexpr Handler makeHandlerValue() {
+    Handler h;
+    h.heapAllocated = heapAllocated<T>();
+    if constexpr (!std::is_trivially_destructible_v<T> || heapAllocated<T>()) {
+      h.destroy = &destroyImpl<T>;
+    }
+    if constexpr (!heapAllocated<T>() &&
+                  !std::is_trivially_move_constructible_v<T>) {
+      h.moveConstruct = &moveConstructImpl<T>;
+    }
+    if constexpr (!heapAllocated<T>() &&
+                  !std::is_trivially_move_assignable_v<T>) {
+      h.move = &moveImpl<T>;
+    }
+    if constexpr (std::is_copy_constructible_v<T> &&
+                  (!std::is_trivially_copy_constructible_v<T> ||
+                   heapAllocated<T>())) {
+      h.copyConstruct = &copyConstructImpl<T>;
+    }
 
-      if constexpr (std::is_copy_assignable_v<T> &&
-                    (!std::is_trivially_copy_assignable_v<T> ||
-                     heapAllocated<T>())) {
-        h.copy = &copyImpl<T>;
-      }
+    if constexpr (std::is_copy_assignable_v<T> &&
+                  (!std::is_trivially_copy_assignable_v<T> ||
+                   heapAllocated<T>())) {
+      h.copy = &copyImpl<T>;
+    }
 
-      if constexpr (!std::is_void_v<Base>) {
-        h.upcast = [](void* p) -> Base* {
-          return static_cast<Base*>(static_cast<T*>(p));
-        };
-        h.upcastConst = [](const void* p) -> const Base* {
-          return static_cast<const Base*>(static_cast<const T*>(p));
-        };
-      }
+    if constexpr (!std::is_void_v<Base>) {
+      h.upcast = [](void* p) -> Base* {
+        return static_cast<Base*>(static_cast<T*>(p));
+      };
+      h.upcastConst = [](const void* p) -> const Base* {
+        return static_cast<const Base*>(static_cast<const T*>(p));
+      };
+    }
 
-      h.typeHash = typeHash<T>();
-      h.typeInfo = &typeid(T);
+    h.typeInfo = &typeid(T);
 
+    return h;
+  }
+
+  template <typename T>
+  static const Handler* makeHandler() {
+    static_assert(!std::is_base_of_v<AnyBaseAll, std::decay_t<T>>,
+                  "Cannot wrap Any in Any");
+    static constexpr Handler static_handler = makeHandlerValue<T>();
+
+#if defined(_ACTS_ANY_ENABLE_DEBUG)
+    // Reporting has to happen here rather than in makeHandlerValue, which is
+    // constant evaluated. Only compiled in when debug output is enabled, so it
+    // does not put a guard variable on the hot path.
+    [[maybe_unused]] static const bool reported = []() {
+      const Handler& h = static_handler;
       _ACTS_ANY_DEBUG("Type: " << typeid(T).name());
       _ACTS_ANY_DEBUG(" -> destroy: " << h.destroy);
       _ACTS_ANY_DEBUG(" -> moveConstruct: " << h.moveConstruct);
@@ -614,9 +660,10 @@
       _ACTS_ANY_DEBUG(" -> copy: " << h.copy);
       _ACTS_ANY_DEBUG(
           " -> heapAllocated: " << (h.heapAllocated ? "yes" : "no"));
-
-      return h;
+      return true;
     }();
+#endif
+
     return &static_handler;
   }
 
@@ -628,6 +675,14 @@
   template <typename T, typename... Args>
   T* constructValue(Args&&... args) {
     if constexpr (!heapAllocated<T>()) {
+      if constexpr (std::is_empty_v<T>) {
+        // An empty object occupies one byte of the buffer that its constructor
+        // never writes. Write it before the object's lifetime starts, so the
+        // trivial copy/move paths, which copy the buffer as a fixed-size
+        // block, do not read a buffer that was never written at all. Nothing
+        // is emitted for types that carry state.
+        m_data[0] = std::byte{0};
+      }
       // construct into local buffer
       auto* ptr = new (m_data.data()) T(std::forward<Args>(args)...);
       _ACTS_ANY_VERBOSE("Construct local (this="

Core/include/Acts/Utilities/HashedString.hpp

@@ -78,6 +78,8 @@ constexpr HashedString operator""_hash(char const* s, std::size_t count) {
 
 /// Hash for a type. Since it's not possible to hash a type at compile-time,
 /// this function returns a runtime hash but caches it in a static variable.
+/// The hash is derived from the mangled name, which is fixed by the Itanium
+/// ABI and therefore identical between GCC and Clang.
 /// @tparam T Type to hash
 /// @return Hashed string representation
 template <typename T>

Core/src/Utilities/Any.cpp

@@ -0,0 +1,19 @@
+// This file is part of the ACTS project.
+//
+// Copyright (C) 2016 CERN for the benefit of the ACTS project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#include "Acts/Utilities/Any.hpp"
+
+#include <any>
+
+namespace Acts::detail {
+
+void throwBadAnyCast() {
+  throw std::bad_any_cast{};
+}
+
+}  // namespace Acts::detail

Core/src/Utilities/CMakeLists.txt

@@ -9,6 +9,7 @@ target_sources(
         Intersection.cpp
         IAxis.cpp
         GraphViz.cpp
+        Any.cpp
         ProtoAxis.cpp
         ScopedTimer.cpp
         TransformComparator.cpp

Tests/Benchmarks/SourceLinkBenchmark.cpp

@@ -6,12 +6,15 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+#include "Acts/EventData/SourceLink.hpp"
 #include "Acts/EventData/VectorMultiTrajectory.hpp"
 #include "Acts/Geometry/GeometryIdentifier.hpp"
 #include "ActsTests/CommonHelpers/BenchmarkTools.hpp"
 
+#include <cstdint>
 #include <iostream>
 #include <type_traits>
+#include <vector>
 
 using namespace Acts;
 using namespace ActsTests;
@@ -93,6 +96,42 @@ int main(int /*argc*/, char** /*argv[]*/) {
       inputs);
   std::cout << copyMoveConstructSourceLink << std::endl;
 
+  // The track finding unpacks a source link for every measurement candidate,
+  // in the calibrator, the measurement selector and the surface accessor.
+  std::cout << "Unpack source link with get<T>" << std::endl;
+  auto unpackGet = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.get<BenchmarkSourceLink>().index();
+      },
+      inputs);
+  std::cout << unpackGet << std::endl;
+
+  std::cout << "Unpack source link with getPtr<T>" << std::endl;
+  auto unpackGetPtr = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.getPtr<BenchmarkSourceLink>()->index();
+      },
+      inputs);
+  std::cout << unpackGetPtr << std::endl;
+
+  std::cout << "Unpack geometry id from source link" << std::endl;
+  auto unpackGeometryId = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.get<BenchmarkSourceLink>().geometryId();
+      },
+      inputs);
+  std::cout << unpackGeometryId << std::endl;
+
+  // Shape of the track finding inner loop: wrap, then immediately unpack.
+  std::cout << "Construct and unpack source link" << std::endl;
+  auto constructAndUnpack = microBenchmark(
+      [&]() {
+        SourceLink sl{bsl};
+        return sl.get<BenchmarkSourceLink>().index();
+      },
+      n);
+  std::cout << constructAndUnpack << std::endl;
+
   std::cout << "Optional assignment" << std::endl;
   auto opt_assignment = microBenchmark(
       [&]() {