[interop] Ship resource-dir Clang headers for self-contained install (compiler-research#33)

This is the second part of the patch that ships the required runtime
clang headers. With this change, a build/install of cppjit is
self-contained and with the LLVM that is statically linked into
libClangCppInterOp.so, we drop the LLVM dependency on a target
machine. The wheels build PR will exercise this on clean images
without LLVM.

The bundle is produced by the CMake install rules from the LLVM used at build
time, so it stays identical to the headers libClangCppInterOp was compiled against
under every install channel (pip, conda recipes, distro tooling, bare cmake --install)
and for development builds against any provided LLVM_DIR. Only include/ ships, 
the JIT reads nothing else from a resource directory. This adds ~0.8 MB to the wheel
(the headers compress well) and 8 MB to the installed package on-disk.

Setups without the bundled headers such as raw build trees, or distributions that
strip vendored copies, fall back to a single Cpp::DetectResourceDir("clang-<major>") probe and bare clang detection at CppInterOp's interpreter creation time.

Author: aaronj0

CMakeLists.txt

@@ -125,6 +125,7 @@ target_compile_definitions(cppjit PRIVATE
     CPPINTEROP_LIBRARY="cppjit_backend/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
     CPPINTEROP_INCLUDE_DIR="cppjit_backend/include"
     CPPJIT_CLANG_MAJOR="${LLVM_VERSION_MAJOR}"
+    CPPJIT_CLANG_INCLUDE_DIR="cppjit_backend/lib/clang/${LLVM_VERSION_MAJOR}"
 )
 
 target_include_directories(cppjit PRIVATE
@@ -175,6 +176,18 @@ install(CODE "
     file(INSTALL \"${CPPINTEROP_INSTALL_DIR}/include/\" DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit_backend/include)
 ")
 
+# ship the builtin headers of the build clang, laid out as a headers-only
+# resource dir: only include/ ships
+set(_clang_resource_dir "${LLVM_LIBRARY_DIR}/clang/${LLVM_VERSION_MAJOR}")
+if(NOT EXISTS "${_clang_resource_dir}/include")
+    message(FATAL_ERROR
+        "No builtin headers at ${_clang_resource_dir}/include; the LLVM at "
+        "${LLVM_DIR} carries no clang resource directory")
+endif()
+install(DIRECTORY "${_clang_resource_dir}/include/"
+    DESTINATION "cppjit_backend/lib/clang/${LLVM_VERSION_MAJOR}/include"
+)
+
 # the public cpyrt API headers keep their installed cpyrt/ prefix
 install(FILES
     src/cpyrt/API.h

src/interop/interop_wrapper.cxx

@@ -78,6 +78,7 @@ static inline bool is_integral(std::string& s) {
 struct InterOpPaths {
   std::string Library;
   std::string IncludeDir;
+  std::string ClangIncludeDir; // empty when the bundled headers are absent
 };
 
 // One relative layout, two anchors: prefer CppInterOp next to our own load
@@ -94,8 +95,17 @@ static InterOpPaths cppinterop_paths() {
       anchor = here;
   }
 #endif
-  return {(anchor / CPPINTEROP_LIBRARY).string(),
-          (anchor / CPPINTEROP_INCLUDE_DIR).string()};
+  InterOpPaths Paths{(anchor / CPPINTEROP_LIBRARY).string(),
+                     (anchor / CPPINTEROP_INCLUDE_DIR).string(),
+                     {}};
+  // The builtin headers of the build clang ship with every installed
+  // package (see the CMake install rule); a raw build tree has none and
+  // falls back to resource-dir detection.
+  const std::filesystem::path bundled = anchor / CPPJIT_CLANG_INCLUDE_DIR;
+  std::error_code ec;
+  if (std::filesystem::exists(bundled / "include", ec))
+    Paths.ClangIncludeDir = bundled.string();
+  return Paths;
 }
 
 // The one place libclangCppInterOp is dlopen'd.
@@ -109,7 +119,8 @@ static bool loadDispatchAPI(const InterOpPaths& Paths) {
 
 // CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside
 // CreateInterpreter, so nothing needs to be forwarded from here.
-static interop::TInterp_t acquireOrCreateInterpreter() {
+static interop::TInterp_t
+acquireOrCreateInterpreter(const InterOpPaths& Paths) {
   if (auto existingInterp = Cpp::GetInterpreter())
     return existingInterp;
 
@@ -119,10 +130,11 @@ static interop::TInterp_t acquireOrCreateInterpreter() {
   args.push_back("-march=native");
 #endif
   // Without clang's builtin headers the interpreter fails at its first
-  // #include. CppInterOp probes only bare `clang`; when just
-  // clang-<major> is installed, resolve and pass it explicitly.
-  std::string resourceDir;
-  if (Cpp::DetectResourceDir("clang").empty())
+  // #include. Prefer the bundled copy: it matches the build clang and
+  // needs no LLVM on the host. DetectResourceDir refuses version
+  // mismatches, and CppInterOp itself probes only bare `clang`.
+  std::string resourceDir = Paths.ClangIncludeDir;
+  if (resourceDir.empty())
     resourceDir = Cpp::DetectResourceDir("clang-" CPPJIT_CLANG_MAJOR);
   if (!resourceDir.empty()) {
     args.push_back("-resource-dir");
@@ -219,7 +231,7 @@ extern "C" int LoadCppInterOp() {
     if (!loadDispatchAPI(Paths))
       return;
 
-    acquireOrCreateInterpreter();
+    acquireOrCreateInterpreter(Paths);
     configureInterpreter(Paths);
     preloadHeaders();
     defineRuntimeHelpers();