perf: Speed up type-erased SourceLink unpacking

Core/include/Acts/Utilities/Any.hpp

@@ -236,14 +236,14 @@
   T& as() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      throwBadAnyCast();
     }
 
     _ACTS_ANY_VERBOSE("Get as "
                       << (m_handler->heapAllocated ? "heap" : "local"));
 
-    return *std::bit_cast<T*>(dataPtr());
+    return *std::bit_cast<T*>(dataPtrFor<T>());
   }
 
   /// Get const reference to stored value of specified type
@@ -254,14 +254,14 @@
   const T& as() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      throwBadAnyCast();
     }
 
     _ACTS_ANY_VERBOSE("Get as "
                       << (m_handler->heapAllocated ? "heap" : "local"));
 
-    return *std::bit_cast<const T*>(dataPtr());
+    return *std::bit_cast<const T*>(dataPtrFor<T>());
   }
 
   /// Get pointer to stored value of specified type
@@ -272,10 +272,10 @@
   T* asPtr() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
+    if (!holds<T>()) {
       return nullptr;
     }
-    return std::bit_cast<T*>(dataPtr());
+    return std::bit_cast<T*>(dataPtrFor<T>());
   }
 
   /// Get const pointer to stored value of specified type
@@ -286,10 +286,10 @@
   const T* asPtr() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
+    if (!holds<T>()) {
       return nullptr;
     }
-    return std::bit_cast<const T*>(dataPtr());
+    return std::bit_cast<const T*>(dataPtrFor<T>());
   }
 
   /// Move the stored value out. Leaves this Any empty.
@@ -300,10 +300,10 @@
   T take() {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    if (m_handler == nullptr || m_handler->typeHash != typeHash<T>()) {
-      throw std::bad_any_cast{};
+    if (!holds<T>()) {
+      throwBadAnyCast();
     }
-    T* ptr = std::bit_cast<T*>(dataPtr());
+    T* ptr = std::bit_cast<T*>(dataPtrFor<T>());
     T value = std::move(*ptr);
     destroy();
     return value;
@@ -452,7 +452,7 @@
   bool is() const {
     static_assert(std::is_same_v<T, std::decay_t<T>>,
                   "Please pass the raw type, no const or ref");
-    return m_handler != nullptr && m_handler->typeHash == typeHash<T>();
+    return holds<T>();
   }
 
   // The base accessors below are member templates on a dummy @c B defaulting to
@@ -527,6 +527,42 @@
   }
 
  private:
+  // The handler is a per-type singleton, so a pointer comparison settles the
+  // common case. The hash comparison covers handlers duplicated across shared
+  // objects, where the pointers differ but the type does not.
+  template <typename T>
+  bool holds() const {
+    if (m_handler == makeHandler<T>()) [[likely]] {
+      return true;
+    }
+    return m_handler != nullptr && m_handler->typeHash == typeHash<T>();
+  }
+
+  // Cold and out-of-line to keep the accessors inlinable.
+  [[noreturn]] [[gnu::noinline, gnu::cold]] static void throwBadAnyCast() {
+    throw std::bad_any_cast{};
+  }
+
+  // T is known statically here, so unlike dataPtr() this needs no load of
+  // m_handler->heapAllocated and no branch on it.
+  template <typename T>
+  void* dataPtrFor() {
+    if constexpr (heapAllocated<T>()) {
+      return *std::bit_cast<void**>(m_data.data());
+    } else {
+      return std::bit_cast<void*>(m_data.data());
+    }
+  }
+
+  template <typename T>
+  const void* dataPtrFor() const {
+    if constexpr (heapAllocated<T>()) {
+      return *std::bit_cast<void* const*>(m_data.data());
+    } else {
+      return std::bit_cast<const void*>(m_data.data());
+    }
+  }
+
   void* dataPtr() {
     if (m_handler->heapAllocated) {
       return *std::bit_cast<void**>(m_data.data());
@@ -563,60 +599,55 @@
     const std::type_info* typeInfo{nullptr};
   };
 
+  // Constant so that the singleton below is constant-initialized and needs no
+  // thread-safe-static guard.
   template <typename T>
-  static const Handler* makeHandler() {
-    static_assert(!std::is_base_of_v<AnyBaseAll, std::decay_t<T>>,
-                  "Cannot wrap Any in Any");
-    static const Handler static_handler = []() {
-      Handler h;
-      h.heapAllocated = heapAllocated<T>();
-      if constexpr (!std::is_trivially_destructible_v<T> ||
-                    heapAllocated<T>()) {
-        h.destroy = &destroyImpl<T>;
-      }
-      if constexpr (!heapAllocated<T>() &&
-                    !std::is_trivially_move_constructible_v<T>) {
-        h.moveConstruct = &moveConstructImpl<T>;
-      }
-      if constexpr (!heapAllocated<T>() &&
-                    !std::is_trivially_move_assignable_v<T>) {
-        h.move = &moveImpl<T>;
-      }
-      if constexpr (std::is_copy_constructible_v<T> &&
-                    (!std::is_trivially_copy_constructible_v<T> ||
-                     heapAllocated<T>())) {
-        h.copyConstruct = &copyConstructImpl<T>;
-      }
+  static constexpr Handler makeHandlerValue() {
+    Handler h;
+    h.heapAllocated = heapAllocated<T>();
+    if constexpr (!std::is_trivially_destructible_v<T> || heapAllocated<T>()) {
+      h.destroy = &destroyImpl<T>;
+    }
+    if constexpr (!heapAllocated<T>() &&
+                  !std::is_trivially_move_constructible_v<T>) {
+      h.moveConstruct = &moveConstructImpl<T>;
+    }
+    if constexpr (!heapAllocated<T>() &&
+                  !std::is_trivially_move_assignable_v<T>) {
+      h.move = &moveImpl<T>;
+    }
+    if constexpr (std::is_copy_constructible_v<T> &&
+                  (!std::is_trivially_copy_constructible_v<T> ||
+                   heapAllocated<T>())) {
+      h.copyConstruct = &copyConstructImpl<T>;
+    }
 
-      if constexpr (std::is_copy_assignable_v<T> &&
-                    (!std::is_trivially_copy_assignable_v<T> ||
-                     heapAllocated<T>())) {
-        h.copy = &copyImpl<T>;
-      }
+    if constexpr (std::is_copy_assignable_v<T> &&
+                  (!std::is_trivially_copy_assignable_v<T> ||
+                   heapAllocated<T>())) {
+      h.copy = &copyImpl<T>;
+    }
 
-      if constexpr (!std::is_void_v<Base>) {
-        h.upcast = [](void* p) -> Base* {
-          return static_cast<Base*>(static_cast<T*>(p));
-        };
-        h.upcastConst = [](const void* p) -> const Base* {
-          return static_cast<const Base*>(static_cast<const T*>(p));
-        };
-      }
+    if constexpr (!std::is_void_v<Base>) {
+      h.upcast = [](void* p) -> Base* {
+        return static_cast<Base*>(static_cast<T*>(p));
+      };
+      h.upcastConst = [](const void* p) -> const Base* {
+        return static_cast<const Base*>(static_cast<const T*>(p));
+      };
+    }
 
-      h.typeHash = typeHash<T>();
-      h.typeInfo = &typeid(T);
+    h.typeHash = typeHash<T>();
+    h.typeInfo = &typeid(T);
 
-      _ACTS_ANY_DEBUG("Type: " << typeid(T).name());
-      _ACTS_ANY_DEBUG(" -> destroy: " << h.destroy);
-      _ACTS_ANY_DEBUG(" -> moveConstruct: " << h.moveConstruct);
-      _ACTS_ANY_DEBUG(" -> move: " << h.move);
-      _ACTS_ANY_DEBUG(" -> copyConstruct: " << h.copyConstruct);
-      _ACTS_ANY_DEBUG(" -> copy: " << h.copy);
-      _ACTS_ANY_DEBUG(
-          " -> heapAllocated: " << (h.heapAllocated ? "yes" : "no"));
+    return h;
+  }
 
-      return h;
-    }();
+  template <typename T>
+  static const Handler* makeHandler() {
+    static_assert(!std::is_base_of_v<AnyBaseAll, std::decay_t<T>>,
+                  "Cannot wrap Any in Any");
+    static constexpr Handler static_handler = makeHandlerValue<T>();
     return &static_handler;
   }
 

Core/include/Acts/Utilities/HashedString.hpp

@@ -11,6 +11,7 @@
 #include <csignal>
 #include <cstddef>
 #include <cstdint>
+#include <source_location>
 #include <string_view>
 #include <typeinfo>
 
@@ -76,14 +77,23 @@ constexpr HashedString operator""_hash(char const* s, std::size_t count) {
 
 }  // namespace HashedStringLiteral
 
-/// Hash for a type. Since it's not possible to hash a type at compile-time,
-/// this function returns a runtime hash but caches it in a static variable.
+namespace detail {
+// Per-type unique string. GCC and Clang both put the template arguments into
+// the function name, so this needs no compiler-specific macro. Checked in
+// HashedString.cpp.
+template <typename T>
+constexpr std::string_view typeIdentity() {
+  return std::source_location::current().function_name();
+}
+}  // namespace detail
+
+/// Compile-time hash for a type. Only ever compares type identity within one
+/// process, so the value itself carries no meaning.
 /// @tparam T Type to hash
 /// @return Hashed string representation
 template <typename T>
-std::uint64_t typeHash() {
-  const static std::uint64_t value = detail::fnv1a_64(typeid(T).name());
-  return value;
+constexpr std::uint64_t typeHash() {
+  return detail::fnv1a_64(detail::typeIdentity<T>());
 }
 
 }  // namespace Acts

Core/src/Utilities/CMakeLists.txt

@@ -9,6 +9,7 @@ target_sources(
         Intersection.cpp
         IAxis.cpp
         GraphViz.cpp
+        HashedString.cpp
         ProtoAxis.cpp
         ScopedTimer.cpp
         TransformComparator.cpp

Core/src/Utilities/HashedString.cpp

@@ -0,0 +1,24 @@
+// This file is part of the ACTS project.
+//
+// Copyright (C) 2016 CERN for the benefit of the ACTS project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#include "Acts/Utilities/HashedString.hpp"
+
+namespace Acts::detail {
+namespace {
+struct TypeHashProbeA {};
+struct TypeHashProbeB {};
+}  // namespace
+
+// Fail the build on a compiler that omits the template arguments from
+// std::source_location::function_name(), rather than silently collide hashes.
+static_assert(typeHash<TypeHashProbeA>() != typeHash<TypeHashProbeB>(),
+              "typeHash<T> does not distinguish types on this compiler: "
+              "std::source_location::function_name() apparently omits template "
+              "arguments. Fall back to __PRETTY_FUNCTION__ / __FUNCSIG__.");
+
+}  // namespace Acts::detail

Tests/Benchmarks/SourceLinkBenchmark.cpp

@@ -6,12 +6,15 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+#include "Acts/EventData/SourceLink.hpp"
 #include "Acts/EventData/VectorMultiTrajectory.hpp"
 #include "Acts/Geometry/GeometryIdentifier.hpp"
 #include "ActsTests/CommonHelpers/BenchmarkTools.hpp"
 
+#include <cstdint>
 #include <iostream>
 #include <type_traits>
+#include <vector>
 
 using namespace Acts;
 using namespace ActsTests;
@@ -93,6 +96,42 @@ int main(int /*argc*/, char** /*argv[]*/) {
       inputs);
   std::cout << copyMoveConstructSourceLink << std::endl;
 
+  // The track finding unpacks a source link for every measurement candidate,
+  // in the calibrator, the measurement selector and the surface accessor.
+  std::cout << "Unpack source link with get<T>" << std::endl;
+  auto unpackGet = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.get<BenchmarkSourceLink>().index();
+      },
+      inputs);
+  std::cout << unpackGet << std::endl;
+
+  std::cout << "Unpack source link with getPtr<T>" << std::endl;
+  auto unpackGetPtr = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.getPtr<BenchmarkSourceLink>()->index();
+      },
+      inputs);
+  std::cout << unpackGetPtr << std::endl;
+
+  std::cout << "Unpack geometry id from source link" << std::endl;
+  auto unpackGeometryId = microBenchmark(
+      [&](const SourceLink& input) {
+        return input.get<BenchmarkSourceLink>().geometryId();
+      },
+      inputs);
+  std::cout << unpackGeometryId << std::endl;
+
+  // Shape of the track finding inner loop: wrap, then immediately unpack.
+  std::cout << "Construct and unpack source link" << std::endl;
+  auto constructAndUnpack = microBenchmark(
+      [&]() {
+        SourceLink sl{bsl};
+        return sl.get<BenchmarkSourceLink>().index();
+      },
+      n);
+  std::cout << constructAndUnpack << std::endl;
+
   std::cout << "Optional assignment" << std::endl;
   auto opt_assignment = microBenchmark(
       [&]() {