Skip to content

Commit 0eca6e2

Browse files
rustyconoverrustyconover
committed
Add native Iroh transports
1 parent c63a3ce commit 0eca6e2

34 files changed

Lines changed: 2141 additions & 211 deletions

‎.github/workflows/MainDistributionPipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
use_prebuilt_images: true
4646
use_prebuilt_wasm_image: true
4747
prebuilt_image_tag: v1.5.5
48-
extra_toolchains: parser_tools
48+
extra_toolchains: parser_tools;rust
4949
# NOTE: this pipeline does NOT run the VGI integration suite.
5050
#
5151
# Every integration .test file carries `require-env VGI_TEST_WORKER`, so

‎.github/workflows/integration.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ jobs:
6666
with:
6767
key: vgi-integration-${{ runner.os }}
6868

69+
- uses: dtolnay/rust-toolchain@1.97
70+
6971
# vcpkg's GitHub-Actions binary cache needs these two, and they are only
7072
# readable from a step's env — hence the explicit export rather than the
7173
# workflow-level `env:` block above.
@@ -100,6 +102,22 @@ jobs:
100102
- name: Install the vgi-python fixture workers
101103
run: ./scripts/ci_install_fixtures.sh
102104

105+
- name: Build pinned Iroh qualification bridge
106+
run: |
107+
git clone --filter=blob:none https://github.com/Query-farm/vgi-rpc-rust.git /tmp/vgi-rpc-rust
108+
git -C /tmp/vgi-rpc-rust checkout c105d7b047e912355a79420ced81cf296ec52c73
109+
cargo build --locked --release --manifest-path /tmp/vgi-rpc-rust/Cargo.toml -p vgi-iroh-bridge
110+
git clone --filter=blob:none https://github.com/Query-farm/vgi-rpc-python.git /tmp/vgi-rpc-python
111+
git -C /tmp/vgi-rpc-python checkout 365395a6144256ec163b17cf9b755b9b5f6a386c
112+
113+
- name: Run native Iroh raw and HTTP integration
114+
env:
115+
BUILD_DIR: release
116+
VGI_PYTHON_DIR: /tmp/vgi-python
117+
VGI_RPC_PYTHON_DIR: /tmp/vgi-rpc-python
118+
VGI_IROH_BRIDGE_BIN: /tmp/vgi-rpc-rust/target/release/vgi-iroh-bridge
119+
run: ./test/run_iroh_integration.sh
120+
103121
# --min-executed is the guard that makes this job worth having. A lane
104122
# that stops running tests still reports green — fewer results, all of
105123
# them passing — which is exactly how this suite went unrun for months.

‎CMakeLists.txt‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,128 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
3838

3939
project(${TARGET_NAME})
4040

41+
# Native Iroh is compiled into the DuckDB extension as a Rust static library.
42+
# Emscripten keeps using the application-owned browser adapter. Release builds
43+
# build from pinned source; local developers may point at a sibling checkout or
44+
# disable the native transport for C++-only iteration.
45+
if(EMSCRIPTEN)
46+
set(VGI_ENABLE_IROH OFF CACHE BOOL "Embed native Iroh transport" FORCE)
47+
else()
48+
option(VGI_ENABLE_IROH "Embed native Iroh transport" ON)
49+
endif()
50+
51+
if(VGI_ENABLE_IROH)
52+
set(VGI_IROH_SOURCE_DIR "" CACHE PATH
53+
"Optional path to a local vgi-rpc-rust source tree")
54+
if(NOT VGI_IROH_SOURCE_DIR)
55+
include(FetchContent)
56+
FetchContent_Declare(vgi_iroh_source
57+
GIT_REPOSITORY https://github.com/Query-farm/vgi-rpc-rust.git
58+
GIT_TAG c105d7b047e912355a79420ced81cf296ec52c73
59+
GIT_SHALLOW FALSE)
60+
FetchContent_GetProperties(vgi_iroh_source)
61+
if(NOT vgi_iroh_source_POPULATED)
62+
FetchContent_Populate(vgi_iroh_source)
63+
endif()
64+
set(VGI_IROH_SOURCE_DIR "${vgi_iroh_source_SOURCE_DIR}")
65+
endif()
66+
if(NOT EXISTS "${VGI_IROH_SOURCE_DIR}/vgi-iroh-cabi/Cargo.toml")
67+
message(FATAL_ERROR
68+
"VGI_ENABLE_IROH requires vgi-iroh-cabi source. Set VGI_IROH_SOURCE_DIR "
69+
"to a pinned vgi-rpc-rust checkout or configure -DVGI_ENABLE_IROH=OFF.")
70+
endif()
71+
72+
set(VGI_IROH_CARGO_ENV)
73+
if(APPLE)
74+
# rustc otherwise defaults native objects to the host OS version, which can
75+
# make a distributable extension unloadable on older supported macOS hosts.
76+
# Apple Silicon starts at macOS 11; use an explicitly configured higher
77+
# deployment target when the enclosing DuckDB build supplies one.
78+
set(VGI_IROH_MACOS_DEPLOYMENT_TARGET "${CMAKE_OSX_DEPLOYMENT_TARGET}")
79+
if(NOT VGI_IROH_MACOS_DEPLOYMENT_TARGET)
80+
set(VGI_IROH_MACOS_DEPLOYMENT_TARGET "11.0")
81+
endif()
82+
list(APPEND VGI_IROH_CARGO_ENV
83+
"MACOSX_DEPLOYMENT_TARGET=${VGI_IROH_MACOS_DEPLOYMENT_TARGET}")
84+
endif()
85+
find_program(VGI_CARGO_EXECUTABLE cargo REQUIRED)
86+
87+
if(CMAKE_CROSSCOMPILING)
88+
message(FATAL_ERROR
89+
"Native VGI Iroh currently requires a native Rust build host; CMake cross-compilation "
90+
"is not yet supported. Configure with -DVGI_ENABLE_IROH=OFF for unsupported builders.")
91+
elseif(APPLE)
92+
if(CMAKE_OSX_ARCHITECTURES MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
93+
set(VGI_IROH_RUST_TARGET x86_64-apple-darwin)
94+
elseif(CMAKE_OSX_ARCHITECTURES MATCHES "arm64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
95+
set(VGI_IROH_RUST_TARGET aarch64-apple-darwin)
96+
else()
97+
message(FATAL_ERROR "Native VGI Iroh supports only x86_64 and arm64 macOS")
98+
endif()
99+
elseif(WIN32)
100+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "ARM|arm|aarch64")
101+
message(FATAL_ERROR "Native VGI Iroh does not yet support Windows ARM")
102+
endif()
103+
if(MINGW)
104+
set(VGI_IROH_RUST_TARGET x86_64-pc-windows-gnu)
105+
else()
106+
set(VGI_IROH_RUST_TARGET x86_64-pc-windows-msvc)
107+
# The enclosing DuckDB/Haybarn build uses /MT. Keep Rust on the same
108+
# static CRT so one extension never crosses allocator/CRT boundaries.
109+
# Preserve any target-specific caller flags rather than replacing them.
110+
set(VGI_IROH_MSVC_RUSTFLAGS "$ENV{CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS}")
111+
string(STRIP "${VGI_IROH_MSVC_RUSTFLAGS} -Ctarget-feature=+crt-static"
112+
VGI_IROH_MSVC_RUSTFLAGS)
113+
list(APPEND VGI_IROH_CARGO_ENV
114+
"CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS=${VGI_IROH_MSVC_RUSTFLAGS}")
115+
endif()
116+
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
117+
if(CMAKE_CXX_COMPILER_TARGET MATCHES "musl")
118+
message(FATAL_ERROR "Native VGI Iroh does not yet support musl targets")
119+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
120+
set(VGI_IROH_RUST_TARGET aarch64-unknown-linux-gnu)
121+
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64")
122+
set(VGI_IROH_RUST_TARGET x86_64-unknown-linux-gnu)
123+
else()
124+
message(FATAL_ERROR "Native VGI Iroh supports only x86_64 and arm64 glibc Linux")
125+
endif()
126+
else()
127+
message(FATAL_ERROR "Native VGI Iroh is currently supported on macOS, Windows x86_64, and glibc Linux")
128+
endif()
129+
130+
set(VGI_IROH_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/vgi-iroh-cargo")
131+
if(WIN32 AND NOT MINGW)
132+
set(VGI_IROH_STATIC_LIB
133+
"${VGI_IROH_CARGO_TARGET_DIR}/${VGI_IROH_RUST_TARGET}/release/vgi_iroh_cabi.lib")
134+
else()
135+
set(VGI_IROH_STATIC_LIB
136+
"${VGI_IROH_CARGO_TARGET_DIR}/${VGI_IROH_RUST_TARGET}/release/libvgi_iroh_cabi.a")
137+
endif()
138+
file(GLOB_RECURSE VGI_IROH_RUST_INPUTS CONFIGURE_DEPENDS
139+
"${VGI_IROH_SOURCE_DIR}/vgi-iroh-cabi/*"
140+
"${VGI_IROH_SOURCE_DIR}/vgi-iroh-transport/*")
141+
list(APPEND VGI_IROH_RUST_INPUTS
142+
"${VGI_IROH_SOURCE_DIR}/Cargo.toml"
143+
"${VGI_IROH_SOURCE_DIR}/Cargo.lock")
144+
add_custom_command(
145+
OUTPUT "${VGI_IROH_STATIC_LIB}"
146+
COMMAND "${CMAKE_COMMAND}" -E env ${VGI_IROH_CARGO_ENV}
147+
"${VGI_CARGO_EXECUTABLE}" build --locked --release
148+
--manifest-path "${VGI_IROH_SOURCE_DIR}/Cargo.toml"
149+
--package vgi-iroh-cabi --target "${VGI_IROH_RUST_TARGET}"
150+
--target-dir "${VGI_IROH_CARGO_TARGET_DIR}"
151+
DEPENDS ${VGI_IROH_RUST_INPUTS}
152+
WORKING_DIRECTORY "${VGI_IROH_SOURCE_DIR}"
153+
COMMENT "Building embedded VGI Iroh transport (${VGI_IROH_RUST_TARGET})"
154+
VERBATIM)
155+
add_custom_target(vgi_iroh_rust DEPENDS "${VGI_IROH_STATIC_LIB}")
156+
add_library(vgi_iroh_cabi STATIC IMPORTED GLOBAL)
157+
set_target_properties(vgi_iroh_cabi PROPERTIES IMPORTED_LOCATION "${VGI_IROH_STATIC_LIB}")
158+
add_dependencies(vgi_iroh_cabi vgi_iroh_rust)
159+
include_directories("${VGI_IROH_SOURCE_DIR}/vgi-iroh-cabi/include")
160+
add_compile_definitions(VGI_ENABLE_IROH=1)
161+
endif()
162+
41163
# Windows: keep <windows.h> from pulling in the legacy <winsock.h>, which
42164
# collides ("redefinition; different linkage") with the <winsock2.h> that
43165
# cpp-httplib (HTTP transport) uses. NOMINMAX stops windows.h's min/max macros
@@ -94,6 +216,8 @@ src/vgi_container_runtime.cpp
94216
src/vgi_http_compression.cpp
95217
src/vgi_http_client.cpp
96218
src/vgi_http_function_connection.cpp
219+
src/vgi_iroh_config.cpp
220+
src/vgi_iroh_native.cpp
97221
src/vgi_cookie_jar.cpp
98222
src/vgi_oauth.cpp
99223
src/vgi_oauth_env.cpp
@@ -173,6 +297,22 @@ build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
173297
target_link_libraries(${EXTENSION_NAME} ${VGI_SSL_LIBS} ${VGI_ARROW_LIB})
174298
target_link_libraries(${LOADABLE_EXTENSION_NAME} ${VGI_SSL_LIBS} ${VGI_ARROW_LIB})
175299

300+
if(VGI_ENABLE_IROH)
301+
add_dependencies(${EXTENSION_NAME} vgi_iroh_rust)
302+
add_dependencies(${LOADABLE_EXTENSION_NAME} vgi_iroh_rust)
303+
target_link_libraries(${EXTENSION_NAME} vgi_iroh_cabi)
304+
target_link_libraries(${LOADABLE_EXTENSION_NAME} vgi_iroh_cabi)
305+
if(APPLE)
306+
target_link_libraries(${EXTENSION_NAME} "-framework Security" "-framework CoreFoundation"
307+
"-framework SystemConfiguration")
308+
target_link_libraries(${LOADABLE_EXTENSION_NAME} "-framework Security" "-framework CoreFoundation"
309+
"-framework SystemConfiguration")
310+
elseif(WIN32)
311+
target_link_libraries(${EXTENSION_NAME} Crypt32)
312+
target_link_libraries(${LOADABLE_EXTENSION_NAME} Crypt32)
313+
endif()
314+
endif()
315+
176316
# SQLite-backed per-value memo disk tier (same-host multi-process + cross-restart).
177317
# sqlite3 comes from vcpkg (see vcpkg.json), like every other native dependency, so it
178318
# builds identically on macOS/Linux/Windows. Native only: WASM has no filesystem tier

‎docs/native-iroh-transport.md‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Native Iroh transport
2+
3+
Native VGI embeds its Iroh client into the DuckDB extension. There is no runtime
4+
download, connector process, localhost proxy, or Rust dependency for extension
5+
users. Rust is a build-time toolchain requirement only.
6+
7+
Raw, connection-stateful VGI uses the canonical `vgi-rpc/arrow-mux/1` ALPN:
8+
9+
```sql
10+
ATTACH 'remote' AS remote (
11+
TYPE vgi,
12+
LOCATION 'iroh://0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
13+
);
14+
```
15+
16+
HTTP-state VGI uses `iroh-http/2` while retaining the existing HTTP continuation,
17+
capability, compression, response-budget, authentication, cookie, and error logic:
18+
19+
```sql
20+
ATTACH 'remote' AS remote (
21+
TYPE vgi,
22+
LOCATION 'httpi://0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef/vgi'
23+
);
24+
```
25+
26+
The client authenticates the remote endpoint ID cryptographically in both modes.
27+
That proves possession of the endpoint key; worker authorization remains an
28+
operator policy.
29+
30+
By default the client creates one process-stable ephemeral identity. A stable
31+
identity can be scoped to a worker without placing key material in SQL history:
32+
33+
```sql
34+
CREATE SECRET worker_iroh_identity (
35+
TYPE iroh,
36+
SECRET_KEY '…',
37+
SCOPE 'iroh://0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
38+
);
39+
```
40+
41+
An `iroh_secret_key` ATTACH option takes precedence over a scoped secret. Invalid
42+
configured keys fail closed; they never downgrade to an ephemeral identity.
43+
Although VGI redacts that option from catalog diagnostics after binding, key
44+
text written directly in SQL may already have been captured by the caller's
45+
SQL history or logging. Use a scoped secret for persistent identities.
46+
`iroh_relay_urls` selects custom relays and `iroh_no_relay` disables relays; the
47+
two options are mutually exclusive. Connect and I/O budgets are controlled by
48+
`vgi_iroh_connect_timeout_seconds` and `vgi_iroh_io_timeout_seconds`.
49+
50+
Private or relay-disabled workers can supply the remote address information
51+
that an EndpointId alone cannot discover: `iroh_remote_relay_url` accepts the
52+
worker's relay URL and `iroh_direct_addresses` accepts a `VARCHAR[]` of Iroh
53+
socket-address strings. These are public routing hints, not identities; the
54+
authenticated EndpointId remains authoritative.
55+
56+
Externalized HTTP response pointers must still be absolute HTTP(S) URLs. An Iroh
57+
worker that cannot expose such object storage should use HTTP continuations
58+
within the advertised response budget instead.

‎docs/wasm-worker-transport.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,11 @@ ATTACH 'iroh://0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef'
2323
```
2424

2525
The EndpointId must be exactly 64 lowercase hexadecimal characters. This target is available
26-
only in DuckDB-WASM and only after the host application supplies one Iroh adapter Worker to the
26+
in DuckDB-WASM only after the host application supplies one Iroh adapter Worker to the
2727
Haybarn bridge. Every remote target receives its own SAB region, while that one adapter keeps
28-
one local Iroh endpoint identity and multiplexes the regions. Native VGI rejects `iroh://`
29-
explicitly; use the native Iroh transport adapter outside the browser instead.
28+
one local Iroh endpoint identity and multiplexes the regions. Native VGI embeds the same
29+
Arrow-free Rust transport core directly in the extension; it does not download or spawn an
30+
adapter executable.
3031

3132
HTTP over Iroh keeps the mature HTTP client state machine and changes only how
3233
the request reaches the server:

‎src/include/vgi_attach_parameters.hpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class CatalogAuth;
3232
class SessionCookieJar;
3333
// Forward declaration — full definition in vgi_http_client.hpp
3434
class VgiHttpClientPool;
35+
struct IrohClientConfig;
3536

3637
// POD constructor argument for ``VgiAttachParameters``. Replaces the
3738
// 8-positional-default-param constructor that previous versions of this
@@ -51,6 +52,9 @@ struct VgiAttachParametersConfig {
5152
std::string data_version_spec;
5253
std::string implementation_version;
5354
std::shared_ptr<SessionCookieJar> cookie_jar;
55+
// Native Iroh endpoint/identity configuration. Null for non-Iroh transports
56+
// and for Emscripten, where the page-owned adapter owns the endpoint.
57+
std::shared_ptr<IrohClientConfig> iroh;
5458
// Per-LOCATION launcher overrides. Both nullopt by default; when set,
5559
// they're applied to the spawn-time ``LaunchConfig`` for `launch:`
5660
// LOCATIONs and rejected at parse time for any other transport.
@@ -80,7 +84,7 @@ struct VgiAttachParameters {
8084
auth_(std::move(cfg.auth)),
8185
data_version_spec_(std::move(cfg.data_version_spec)),
8286
implementation_version_(std::move(cfg.implementation_version)),
83-
cookie_jar_(std::move(cfg.cookie_jar)),
87+
cookie_jar_(std::move(cfg.cookie_jar)), iroh_(std::move(cfg.iroh)),
8488
launcher_idle_timeout_seconds_(cfg.launcher_idle_timeout_seconds),
8589
launcher_state_dir_(std::move(cfg.launcher_state_dir)),
8690
tcp_proxy_(std::move(cfg.tcp_proxy)),
@@ -99,6 +103,7 @@ struct VgiAttachParameters {
99103
/*cache=*/true, std::move(auth),
100104
std::move(data_version_spec),
101105
std::move(implementation_version), std::move(cookie_jar),
106+
/*iroh=*/nullptr,
102107
std::nullopt, std::nullopt,
103108
/*attach_options_canonical=*/std::string()}) {
104109
}
@@ -150,6 +155,10 @@ struct VgiAttachParameters {
150155
return auth_;
151156
}
152157

158+
const std::shared_ptr<IrohClientConfig> &iroh() const {
159+
return iroh_;
160+
}
161+
153162
// Requested semver constraint for the catalog's data version. Empty string
154163
// means the client did not constrain. Pooled worker lookup uses this as
155164
// part of the key so mismatched versions never share a process.
@@ -245,6 +254,7 @@ struct VgiAttachParameters {
245254
std::string data_version_spec_;
246255
std::string implementation_version_;
247256
std::shared_ptr<SessionCookieJar> cookie_jar_;
257+
std::shared_ptr<IrohClientConfig> iroh_;
248258
std::optional<int64_t> launcher_idle_timeout_seconds_;
249259
std::optional<std::string> launcher_state_dir_;
250260
std::string tcp_proxy_;

‎src/include/vgi_catalog_rpc.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ CatalogAttachResult InvokeCatalogAttach(const std::string &worker_path, const st
7474
std::optional<int64_t> launcher_idle_timeout_seconds = std::nullopt,
7575
std::optional<std::string> launcher_state_dir = std::nullopt,
7676
std::shared_ptr<void> worker_artifact_anchor = nullptr,
77-
const std::string &tcp_proxy = "");
77+
const std::string &tcp_proxy = "",
78+
std::shared_ptr<IrohClientConfig> iroh = nullptr);
7879

7980
// List catalogs exposed by a worker. Returns per-catalog discovery records
8081
// carrying implementation_version and data_version_spec metadata alongside the
@@ -92,7 +93,8 @@ std::vector<VgiCatalogInfo> InvokeCatalogs(const std::string &worker_path, Clien
9293
std::optional<int64_t> launcher_idle_timeout_seconds = std::nullopt,
9394
std::optional<std::string> launcher_state_dir = std::nullopt,
9495
std::shared_ptr<void> worker_artifact_anchor = nullptr,
95-
const std::string &tcp_proxy = "");
96+
const std::string &tcp_proxy = "",
97+
std::shared_ptr<IrohClientConfig> iroh = nullptr);
9698

9799
// Invoke catalog_schemas: list schemas in an attached catalog
98100
std::vector<VgiSchemaInfo> InvokeCatalogSchemas(const CatalogRpcContext &ctx, ClientContext &context);

0 commit comments

Comments
 (0)