diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 6592f4b..619ea86 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -22,8 +22,8 @@ jobs: - "" - "vision" - "vlm" - - "ort-download-binaries,ort-api-24" - - "ort-load-dynamic,ort-api-24" + - "ort-download-binaries,ort-api-28" + - "ort-load-dynamic,ort-api-28" - "video" - "viewer" - "annotator" @@ -49,7 +49,7 @@ jobs: - name: Clippy run: | if [ "${{ matrix.feature }}" = "all-features" ]; then - cargo clippy --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-24" --all-targets -- -D warnings + cargo clippy --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-28" --all-targets -- -D warnings elif [ "${{ matrix.feature }}" = "" ]; then cargo clippy --no-default-features --all-targets -- -D warnings else @@ -74,7 +74,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Check - run: cargo check --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-24" --all-targets + run: cargo check --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-28" --all-targets test: name: cargo-test @@ -94,7 +94,7 @@ jobs: uses: dtolnay/rust-toolchain@nightly - name: Test - run: cargo +nightly test --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-24" --all-targets + run: cargo +nightly test --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-28" --all-targets build-linux: needs: test @@ -120,4 +120,4 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Build - run: cargo build --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-24" \ No newline at end of file + run: cargo build --no-default-features --features "all-models,video,viewer,annotator,ort-download-binaries,ort-load-dynamic,ort-api-28" diff --git a/Cargo.toml b/Cargo.toml index eae6725..d328e71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,10 +40,11 @@ fast_image_resize = { version = "5.5.0", default-features = false, features = [" minifb = { version = "0.28.0", optional = true } video-rs = { version = "0.11.0", features = ["ndarray"], optional = true } ndarray-npy = { version = "0.10", optional = true } -ort = { version = "=2.0.0-rc.12", default-features = false, features = [ +ort = { version = "=2.0.0-rc.13", default-features = false, features = [ "tls-rustls", "copy-dylibs", "half", + "ndarray", "std", ] } cudarc = { version = "0.19", optional = true, default-features = false, features = [ @@ -75,7 +76,7 @@ strip = true [features] -default = ["ort-download-binaries", "vision", "annotator", "ort-api-24"] +default = ["ort-download-binaries", "vision", "annotator", "ort-api-28"] # ONNXRuntime loading strategies ort-download-binaries = ["ort/download-binaries"] @@ -90,6 +91,10 @@ ort-api-21 = ["ort/api-21"] ort-api-22 = ["ort/api-22"] ort-api-23 = ["ort/api-23"] ort-api-24 = ["ort/api-24"] +ort-api-25 = ["ort/api-25"] +ort-api-26 = ["ort/api-26"] +ort-api-27 = ["ort/api-27"] +ort-api-28 = ["ort/api-28"] # Cuda features (Internal use) cuda-runtime = ["dep:cudarc"] @@ -324,4 +329,4 @@ required-features = ["vision", "annotator"] [[example]] name = "gaze-estimation" path = "examples/gaze-estimation/main.rs" -required-features = ["vision", "annotator", "video"] \ No newline at end of file +required-features = ["vision", "annotator", "video"] diff --git a/docs/cargo-features/ep.md b/docs/cargo-features/ep.md index 9992cf5..2a9fa34 100644 --- a/docs/cargo-features/ep.md +++ b/docs/cargo-features/ep.md @@ -16,7 +16,7 @@ Hardware acceleration for inference. Enable the one matching your hardware. | `onednn` | Intel | Deep Neural Network Library | | `cann` | Huawei | Ascend NPU | | `rknpu` | Rockchip | NPU acceleration | -| `armnn` | ARM | Neural Network SDK | +| `armnn` | ARM | Neural Network SDK (removed in recent ONNX Runtime; prefer `acl`/`xnnpack`) | | `xnnpack` | Mobile | CPU optimization | | `webgpu` | Web | WebGPU/Chrome | | `nnapi` | Android | Neural Networks API | diff --git a/docs/cargo-features/ort.md b/docs/cargo-features/ort.md index c2a53a0..1a132bf 100644 --- a/docs/cargo-features/ort.md +++ b/docs/cargo-features/ort.md @@ -10,7 +10,7 @@ ONNX Runtime configuration and API version management. ### API Version Selection -This library supports ONNX Runtime versions 1.17 through 1.24 via API version features. +This library supports ONNX Runtime versions 1.17 through 1.28 via API version features. | Feature | ONNX Runtime | Requirements | |---------|--------------|--------------| @@ -21,11 +21,15 @@ This library supports ONNX Runtime versions 1.17 through 1.24 via API version fe | `ort-api-21` | v1.21 | - | | `ort-api-22` | v1.22 | - | | `ort-api-23` | v1.23 | - | -| `ort-api-24` | v1.24 | **Default** - Latest features | +| `ort-api-24` | v1.24 | - | +| `ort-api-25` | v1.25 | - | +| `ort-api-26` | v1.26 | - | +| `ort-api-27` | v1.27 | - | +| `ort-api-28` | v1.28 | **Default** - Latest features | !!! tip "API Version Selection" ```toml - # Default uses api-24 (latest) + # Default uses api-28 (latest) usls = { version = "0.2", features = ["vision"] } # Specify API version explicitly @@ -34,4 +38,4 @@ This library supports ONNX Runtime versions 1.17 through 1.24 via API version fe !!! note "Version Compatibility" - Each API version includes all features from previous versions - - Check [ORT multiversion docs](https://ort.pyke.io/setup/multiversion) for minimum version requirements \ No newline at end of file + - Check [ORT multiversion docs](https://ort.pyke.io/setup/multiversion) for minimum version requirements diff --git a/src/ort/dtype.rs b/src/ort/dtype.rs index 589c773..fd8a7f1 100644 --- a/src/ort/dtype.rs +++ b/src/ort/dtype.rs @@ -25,6 +25,7 @@ impl From for crate::DType { TensorElementType::Complex128 => Self::Complex128, TensorElementType::Bool => Self::Uint8, TensorElementType::String | TensorElementType::Undefined => Self::Auto, + _ => Self::Auto, } } } diff --git a/src/ort/engine.rs b/src/ort/engine.rs index 7263edd..d571e19 100644 --- a/src/ort/engine.rs +++ b/src/ort/engine.rs @@ -3,7 +3,7 @@ use anyhow::Result; use half::{bf16, f16}; use ndarray::{Array, IxDyn}; use ort::{ - execution_providers::ExecutionProvider, + ep::ExecutionProvider, session::{builder::GraphOptimizationLevel, Session, SessionInputValue, SessionInputs}, value::{DynValue, TensorElementType, Value}, }; @@ -133,7 +133,7 @@ impl FromConfig for Engine { params += param; let param = Ops::make_divisible(param, byte_alignment); let n = Self::get_ort_dtype_from_proto_dtype_id(tensor_proto.data_type) - .map(|x| x.byte_size(1)) + .and_then(|x| x.byte_size(1)) .unwrap_or_default(); let wbmem = param * n; wbmems += wbmem; @@ -147,7 +147,7 @@ impl FromConfig for Engine { params += param; let param = Ops::make_divisible(param, byte_alignment); let n = Self::get_ort_dtype_from_proto_dtype_id(tensor.data_type) - .map(|x| x.byte_size(1)) + .and_then(|x| x.byte_size(1)) .unwrap_or_default(); let wbmem = param * n; @@ -595,7 +595,7 @@ impl Engine { #[cfg(feature = "nvrtx")] { let (spec_min, spec_opt, spec_max) = - Self::generate_shape_specs(&inputs.names, &inputs_minoptmax)?; + Self::generate_shape_specs(&inputs.names, inputs_minoptmax)?; let ep = ort::ep::nvrtx::NVRTX::default() .with_device_id(id as _) @@ -637,7 +637,7 @@ impl Engine { let cache_path = crate::Dir::Cache.crate_dir_default_with_subs(&["caches", "tensorrt"])?; - let mut ep = ort::execution_providers::TensorRTExecutionProvider::default() + let mut ep = ort::ep::TensorRT::default() .with_device_id(id as i32) .with_max_workspace_size(config.ep.tensorrt.max_workspace_size) .with_builder_optimization_level( @@ -695,7 +695,7 @@ impl Engine { #[cfg(feature = "cuda")] { - let ep = ort::execution_providers::CUDAExecutionProvider::default() + let ep = ort::ep::CUDA::default() .with_device_id(id as i32) .with_conv_max_workspace(config.ep.cuda.conv_max_workspace) .with_prefer_nhwc(config.ep.cuda.prefer_nhwc) @@ -720,11 +720,9 @@ impl Engine { } #[cfg(feature = "coreml")] { - use ort::execution_providers::coreml::{ - ComputeUnits, ModelFormat, SpecializationStrategy, - }; + use ort::ep::coreml::{ComputeUnits, ModelFormat, SpecializationStrategy}; - let ep = ort::execution_providers::CoreMLExecutionProvider::default() + let ep = ort::ep::CoreML::default() .with_model_cache_dir( crate::Dir::Cache .crate_dir_default_with_subs(&["caches", "coreml"])? @@ -770,8 +768,7 @@ impl Engine { } #[cfg(feature = "directml")] { - let ep = ort::execution_providers::DirectMLExecutionProvider::default() - .with_device_id(id as i32); + let ep = ort::ep::DirectML::default().with_device_id(id as i32); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { @@ -791,7 +788,7 @@ impl Engine { } #[cfg(feature = "openvino")] { - let ep = ort::execution_providers::OpenVINOExecutionProvider::default() + let ep = ort::ep::OpenVINO::default() .with_device_type(dt) .with_num_threads(config.ep.openvino.num_threads) .with_dynamic_shapes(config.ep.openvino.dynamic_shapes) @@ -822,7 +819,7 @@ impl Engine { } #[cfg(feature = "onednn")] { - let ep = ort::execution_providers::OneDNNExecutionProvider::default() + let ep = ort::ep::OneDNN::default() .with_arena_allocator(config.ep.onednn.arena_allocator); match ep.is_available() { Ok(true) => { @@ -843,7 +840,7 @@ impl Engine { } #[cfg(feature = "cann")] { - let ep = ort::execution_providers::CANNExecutionProvider::default() + let ep = ort::ep::CANN::default() .with_device_id(id as i32) .with_cann_graph(config.ep.cann.graph_inference) .with_dump_graphs(config.ep.cann.dump_graphs) @@ -867,8 +864,7 @@ impl Engine { } #[cfg(feature = "qnn")] { - let ep = ort::execution_providers::QNNExecutionProvider::default() - .with_device_id(id as i32); + let ep = ort::ep::QNN::default().with_device_id(id as i32); match ep.is_available() { Ok(true) => { ep.register(&mut builder) @@ -887,7 +883,7 @@ impl Engine { } #[cfg(feature = "migraphx")] { - let ep = ort::execution_providers::MIGraphXExecutionProvider::default() + let ep = ort::ep::MIGraphX::default() .with_device_id(id as i32) .with_fp16(config.ep.migraphx.fp16) .with_exhaustive_tune(config.ep.migraphx.exhaustive_tune); @@ -911,7 +907,7 @@ impl Engine { } #[cfg(feature = "xnnpack")] { - let ep = ort::execution_providers::XNNPACKExecutionProvider::default(); + let ep = ort::ep::XNNPACK::default(); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { @@ -931,7 +927,7 @@ impl Engine { } #[cfg(feature = "rknpu")] { - let ep = ort::execution_providers::RKNPUExecutionProvider::default(); + let ep = ort::ep::RKNPU::default(); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { @@ -951,8 +947,7 @@ impl Engine { } #[cfg(feature = "acl")] { - let ep = ort::execution_providers::ACLExecutionProvider::default() - .with_fast_math(true); + let ep = ort::ep::ACL::default().with_fast_math(true); match ep.is_available() { Ok(true) => { ep.register(&mut builder) @@ -971,7 +966,7 @@ impl Engine { } #[cfg(feature = "nnapi")] { - let ep = ort::execution_providers::NNAPIExecutionProvider::default() + let ep = ort::ep::NNAPI::default() .with_cpu_only(config.ep.nnapi.cpu_only) .with_disable_cpu(config.ep.nnapi.disable_cpu) .with_fp16(config.ep.nnapi.fp16) @@ -995,8 +990,10 @@ impl Engine { .replace("#FEATURE", "armnn")); } #[cfg(feature = "armnn")] + #[allow(deprecated)] + // removed from ONNX Runtime; kept for older runtimes { - let ep = ort::execution_providers::ArmNNExecutionProvider::default() + let ep = ort::ep::ArmNN::default() .with_arena_allocator(config.ep.armnn.arena_allocator); match ep.is_available() { Ok(true) => { @@ -1017,13 +1014,12 @@ impl Engine { } #[cfg(feature = "vitis")] { - let ep = ort::execution_providers::VitisAIExecutionProvider::default() - .with_cache_dir( - crate::Dir::Cache - .crate_dir_default_with_subs(&["caches", "vitis"])? - .display() - .to_string(), - ); + let ep = ort::ep::Vitis::default().with_cache_dir( + crate::Dir::Cache + .crate_dir_default_with_subs(&["caches", "vitis"])? + .display() + .to_string(), + ); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { @@ -1043,7 +1039,7 @@ impl Engine { } #[cfg(feature = "tvm")] { - let ep = ort::execution_providers::TVMExecutionProvider::default(); + let ep = ort::ep::TVM::default(); match ep.is_available() { Ok(true) => { ep.register(&mut builder) @@ -1062,13 +1058,15 @@ impl Engine { } #[cfg(feature = "azure")] { - let ep = ort::execution_providers::AzureExecutionProvider::default(); + let ep = ort::ep::Azure::default(); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { anyhow::anyhow!("Failed to register Azure: {err}") })?; - builder = builder.with_extensions()?; + builder = builder.with_extensions().map_err(|err| { + anyhow::anyhow!("Failed to enable ort extensions: {err}") + })?; } _ => anyhow::bail!(compile_help.replace("#EP", "Azure")), } @@ -1083,7 +1081,7 @@ impl Engine { } #[cfg(feature = "webgpu")] { - let ep = ort::execution_providers::WebGPUExecutionProvider::default(); + let ep = ort::ep::WebGPU::default(); match ep.is_available() { Ok(true) => { ep.register(&mut builder).map_err(|err| { @@ -1103,8 +1101,7 @@ impl Engine { } #[cfg(feature = "rocm")] { - let ep = ort::execution_providers::ROCmExecutionProvider::default() - .with_device_id(id as i32); + let ep = ort::ep::ROCm::default().with_device_id(id as i32); match ep.is_available() { Ok(true) => { ep.register(&mut builder) @@ -1115,8 +1112,8 @@ impl Engine { } } Device::Cpu(_) => { - let ep = ort::execution_providers::CPUExecutionProvider::default() - .with_arena_allocator(config.ep.cpu.arena_allocator); + let ep = + ort::ep::CPU::default().with_arena_allocator(config.ep.cpu.arena_allocator); match ep.is_available() { Ok(true) => { ep.register(&mut builder)