diff --git a/Cargo.lock b/Cargo.lock index ff3a274..4ff52ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,6 +577,7 @@ dependencies = [ "sys_traits", "tokio", "url", + "wasm_dep_analyzer", ] [[package]] diff --git a/mod.ts b/mod.ts index cfb7372..b233e4d 100644 --- a/mod.ts +++ b/mod.ts @@ -515,17 +515,50 @@ export async function build(options: BuildOptions): Promise { })).filter((p) => p.kind === "bin").map((p) => p.path), ); + const writeBinaryFile = (filePath: string, bytes: Uint8Array) => { + const dir = path.dirname(filePath); + if (!createdDirectories.has(dir)) { + Deno.mkdirSync(dir, { recursive: true }); + createdDirectories.add(dir); + } + Deno.writeFileSync( + filePath, + bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes), + ); + }; + for ( const outputFile of [ ...transformOutput.main.files, ...transformOutput.test.files, ] ) { + if (outputFile.bytes) { + const srcPath = path.join(options.outDir, "src", outputFile.filePath); + const esmPath = path.join(options.outDir, "esm", outputFile.filePath); + const scriptPath = path.join( + options.outDir, + "script", + outputFile.filePath, + ); + if (!options.skipSourceOutput) { + writeBinaryFile(srcPath, outputFile.bytes); + } + if (options.esModule) { + writeBinaryFile(esmPath, outputFile.bytes); + } + if (options.scriptModule) { + writeBinaryFile(scriptPath, outputFile.bytes); + } + continue; + } + const outputFilePath = path.join( options.outDir, "src", outputFile.filePath, ); + const outputFileText = binaryEntryPointPaths.has(outputFile.filePath) ? `#!/usr/bin/env node\n${ outputFile.fileText.replace(/^#![^\n\r]*\r?\n?/, "") diff --git a/rs-lib/Cargo.toml b/rs-lib/Cargo.toml index bbd10fd..2d8adf6 100644 --- a/rs-lib/Cargo.toml +++ b/rs-lib/Cargo.toml @@ -35,6 +35,7 @@ serde = { version = "1.0.159", features = ["derive"], optional = true } serde_json.workspace = true sys_traits.workspace = true url.workspace = true +wasm_dep_analyzer = "0.4.0" [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/rs-lib/src/lib.rs b/rs-lib/src/lib.rs index 52596b0..394a8e3 100644 --- a/rs-lib/src/lib.rs +++ b/rs-lib/src/lib.rs @@ -65,6 +65,7 @@ pub use deno_graph::source::LoaderChecksum; use crate::declaration_file_resolution::TypesDependency; use crate::utils::strip_bom; +use crate::utils::with_extension; mod analyze; mod declaration_file_resolution; @@ -83,6 +84,8 @@ mod visitors; pub struct OutputFile { pub file_path: PathBuf, pub file_text: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes: Option>, } #[cfg_attr(feature = "serialization", derive(serde::Serialize))] @@ -462,7 +465,7 @@ pub async fn transform( let (specifier_mappings, specifier_mapping_keys) = resolve_specifier_mappings( options.specifier_mappings, - &deno_resolver, + deno_resolver, &cjs_tracker, options .entry_points @@ -683,10 +686,46 @@ pub async fn transform( Module::Json(module) => { format!("export default {};", strip_bom(&module.source.text).trim(),) } - Module::Node(_) - | Module::Npm(_) - | Module::External(_) - | Module::Wasm(_) => { + Module::Wasm(wasm_module) => { + let file_path = mappings.get_file_path(specifier).to_owned(); + env_context.environment.files.push(OutputFile { + file_path: file_path.clone(), + file_text: String::new(), + bytes: Some(wasm_module.source.to_vec()), + }); + + let export_names = wasm_dep_analyzer::WasmDeps::parse( + &wasm_module.source, + wasm_dep_analyzer::ParseOptions::default(), + ) + .ok() + .map(|deps| { + deps + .exports + .into_iter() + .map(|e| e.name.to_string()) + .collect::>() + }) + .unwrap_or_default(); + + let wasm_filename = file_path + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + + let wrapper_file_text = + generate_wasm_js_wrapper(&wasm_filename, &export_names); + let wrapper_file_path = with_extension(&file_path, "wasm.js"); + + env_context.environment.files.push(OutputFile { + file_path: wrapper_file_path, + file_text: wrapper_file_text, + bytes: None, + }); + + continue; + } + Module::Node(_) | Module::Npm(_) | Module::External(_) => { bail!("Not implemented module kind for {}", module.specifier()) } }; @@ -695,6 +734,7 @@ pub async fn transform( env_context.environment.files.push(OutputFile { file_path, file_text, + bytes: None, }); } @@ -850,6 +890,7 @@ fn check_add_polyfill_file_to_environment( env_context.environment.files.push(OutputFile { file_path: polyfill_file_path.to_path_buf(), file_text: polyfill_file_text, + bytes: None, }); for entry_point in env_context.environment.entry_points.iter() { @@ -895,6 +936,7 @@ fn check_add_shim_file_to_environment( env_context.environment.files.push(OutputFile { file_path: shim_file_path.to_path_buf(), file_text: shim_file_text, + bytes: None, }); for shim in env_context.shims.iter() { @@ -969,7 +1011,7 @@ fn check_add_shim_file_to_environment( .map(get_specifer_text) .collect::>() .join(", "), - &module_specifier_text, + module_specifier_text, )); } @@ -981,7 +1023,7 @@ fn check_add_shim_file_to_environment( .map(get_specifer_text) .collect::>() .join(", "), - &module_specifier_text, + module_specifier_text, )); } @@ -1230,6 +1272,87 @@ fn get_declaration_warnings(specifiers: &Specifiers) -> Vec { } } +fn generate_wasm_js_wrapper( + wasm_filename: &str, + export_names: &[String], +) -> String { + let mut text = String::new(); + text.push_str("// Generated by dnt for WASM module wrapper\n"); + text.push_str("function getNodeModule(name) {\n"); + text.push_str(" if (typeof require !== \"undefined\") {\n"); + text.push_str(" return require(name);\n"); + text.push_str(" }\n"); + text.push_str(" if (typeof process !== \"undefined\" && typeof process.getBuiltinModule === \"function\") {\n"); + text.push_str(" return process.getBuiltinModule(name);\n"); + text.push_str(" }\n"); + text.push_str(" if (typeof module !== \"undefined\" && typeof module.createRequire === \"function\") {\n"); + text.push_str(" return module.createRequire(import.meta.url)(name);\n"); + text.push_str(" }\n"); + text.push_str(" return null;\n"); + text.push_str("}\n\n"); + + text.push_str("function getWasmBytes() {\n"); + text.push_str( + " if (typeof process !== \"undefined\" && process?.versions?.node) {\n", + ); + text.push_str(" const fs = getNodeModule(\"node:fs\");\n"); + text.push_str(" const path = getNodeModule(\"node:path\");\n"); + text.push_str(" const url = getNodeModule(\"node:url\");\n"); + text.push_str(" if (fs && path && url) {\n"); + text.push_str(" const dir = typeof __dirname !== \"undefined\" ? __dirname : path.dirname(url.fileURLToPath(import.meta.url));\n"); + text.push_str(&format!( + " return fs.readFileSync(path.resolve(dir, \"./{wasm_filename}\"));\n" + )); + text.push_str(" }\n"); + text.push_str(" }\n"); + text.push_str(" if (typeof XMLHttpRequest !== \"undefined\") {\n"); + text.push_str(" const req = new XMLHttpRequest();\n"); + text.push_str(&format!( + " req.open(\"GET\", new URL(\"./{wasm_filename}\", import.meta.url), false);\n" + )); + text.push_str(" req.responseType = \"arraybuffer\";\n"); + text.push_str(" req.send(null);\n"); + text.push_str(" if (req.status === 200 || req.status === 0) {\n"); + text.push_str(" return new Uint8Array(req.response);\n"); + text.push_str(" }\n"); + text.push_str(" }\n"); + text.push_str(&format!( + " throw new Error(\"Unable to load WASM file '{wasm_filename}' in current environment\");\n" + )); + text.push_str("}\n\n"); + + text.push_str("const wasmBytes = getWasmBytes();\n"); + text.push_str("const wasmModule = new WebAssembly.Module(wasmBytes);\n"); + text.push_str( + "const wasmInstance = new WebAssembly.Instance(wasmModule, {});\n\n", + ); + + text.push_str("export default wasmInstance.exports;\n"); + + for name in export_names { + if is_valid_js_identifier(name) { + text.push_str(&format!( + "export const {name} = wasmInstance.exports.{name};\n" + )); + } + } + + text +} + +fn is_valid_js_identifier(name: &str) -> bool { + if name.is_empty() || name == "default" { + return false; + } + let mut chars = name.chars(); + if let Some(first) = chars.next() { + if !first.is_alphabetic() && first != '_' && first != '$' { + return false; + } + } + chars.all(|c| c.is_alphanumeric() || c == '_' || c == '$') +} + #[cfg(test)] mod test { diff --git a/rs-lib/src/loader/mod.rs b/rs-lib/src/loader/mod.rs index d83e986..76dcfcb 100644 --- a/rs-lib/src/loader/mod.rs +++ b/rs-lib/src/loader/mod.rs @@ -58,7 +58,7 @@ impl<'a> SourceLoader<'a> { fn mapping( &self, specifier: &ModuleSpecifier, - ) -> Option> { + ) -> Option> { match self.specifier_mappings.get(specifier) { Some(mapping) => Some(Cow::Borrowed(mapping)), None => self.jsr_specifier_mappings.get(specifier).map(Cow::Owned), diff --git a/rs-lib/src/mappings.rs b/rs-lib/src/mappings.rs index b2dd309..03bd6a1 100644 --- a/rs-lib/src/mappings.rs +++ b/rs-lib/src/mappings.rs @@ -494,10 +494,12 @@ fn get_mapped_file_path( let filepath_no_ext = get_unique_path(without_ext(path.as_ref()), mapped_filepaths_no_ext); let extension = match media_type { + MediaType::Wasm => "wasm", MediaType::Json => "js", MediaType::Mjs | MediaType::Mts => "js", _ => &media_type.as_ts_extension()[1..], }; + with_extension( &filepath_no_ext, &if let Some(sub_ext) = filepath_no_ext.extension() { diff --git a/rs-lib/src/specifiers.rs b/rs-lib/src/specifiers.rs index a76bb03..4a1fbd5 100644 --- a/rs-lib/src/specifiers.rs +++ b/rs-lib/src/specifiers.rs @@ -123,7 +123,7 @@ pub fn get_specifiers<'a>( for module in all_modules.iter() { match module { - Module::Js(_) | Module::Json(_) => { + Module::Js(_) | Module::Json(_) | Module::Wasm(_) => { match module.specifier().scheme().to_lowercase().as_str() { "file" => local_specifiers.push(module.specifier().clone()), "http" | "https" => { @@ -137,12 +137,7 @@ pub fn get_specifiers<'a>( Module::Npm(_) | Module::Node(_) => { // ignore } - Module::Wasm(_) => { - anyhow::bail!( - "Not implemented support for Wasm modules: {}", - module.specifier() - ); - } + Module::External(module) => { let specifier = &module.specifier; if let Ok(npm_specifier) = @@ -191,11 +186,11 @@ pub fn get_specifiers<'a>( Ok(Specifiers { local: local_specifiers .into_iter() - .filter(|l| !declaration_specifiers.contains(&l)) + .filter(|l| !declaration_specifiers.contains(l)) .collect(), remote: remote_specifiers .into_iter() - .filter(|l| !declaration_specifiers.contains(&l)) + .filter(|l| !declaration_specifiers.contains(l)) .collect(), types, types_packages: declaration_files.types_packages, @@ -249,7 +244,8 @@ fn get_reachable( roots: &[ModuleSpecifier], ) -> HashSet { let mut found = HashSet::new(); - let mut pending = roots.iter().cloned().collect::>(); + let mut pending = roots.to_vec(); + while let Some(specifier) = pending.pop() { let specifier = module_graph.resolve(&specifier).clone(); if !found.insert(specifier.clone()) { diff --git a/rs-lib/src/utils.rs b/rs-lib/src/utils.rs index 164c63a..a833ef9 100644 --- a/rs-lib/src/utils.rs +++ b/rs-lib/src/utils.rs @@ -23,19 +23,15 @@ pub fn get_relative_specifier( from: impl AsRef, to: impl AsRef, ) -> String { - let to = with_extension( - to.as_ref(), - if to - .as_ref() - .to_string_lossy() - .to_lowercase() - .ends_with(".d.ts") - { - "" - } else { - "js" - }, - ); + let to_str = to.as_ref().to_string_lossy().to_lowercase(); + let to = if to_str.ends_with(".wasm") { + with_extension(to.as_ref(), "wasm.js") + } else if to_str.ends_with(".d.ts") { + with_extension(to.as_ref(), "") + } else { + with_extension(to.as_ref(), "js") + }; + let relative_path = get_relative_path(from, to); let relative_path_str = relative_path .to_string_lossy() diff --git a/rs-lib/tests/integration/in_memory_loader.rs b/rs-lib/tests/integration/in_memory_loader.rs index 894a0eb..bf9814f 100644 --- a/rs-lib/tests/integration/in_memory_loader.rs +++ b/rs-lib/tests/integration/in_memory_loader.rs @@ -60,6 +60,23 @@ impl InMemoryLoader { self } + pub fn add_local_bytes( + &mut self, + path: impl AsRef, + bytes: &[u8], + ) -> &mut Self { + let path = path.as_ref(); + let path = if cfg!(windows) && path.starts_with("/") { + PathBuf::from(format!("C:{}", path)) + } else { + PathBuf::from(path) + }; + let parent_dir = path.parent().unwrap(); + self.sys.fs_create_dir_all(parent_dir).unwrap(); + self.sys.fs_write(path, bytes).unwrap(); + self + } + pub fn add_remote_file( &mut self, specifier: impl AsRef, @@ -138,16 +155,14 @@ impl deno_cache_dir::file_fetcher::HttpClient for InMemoryLoader { location.clone(), )])))); } - let result = self - .remote_files - .get(&specifier) - .map(|result| match result { - Ok(result) => Ok(SendResponse::Success( - to_headers(result.1.clone().unwrap_or_default()), - result.0.clone().into_bytes().into(), - )), - Err(err) => Err(SendError::Failed(err.clone().into())), - }); + let result = self.remote_files.get(specifier).map(|result| match result { + Ok(result) => Ok(SendResponse::Success( + to_headers(result.1.clone().unwrap_or_default()), + result.0.clone().into_bytes(), + )), + Err(err) => Err(SendError::Failed(err.clone().into())), + }); + match result { Some(result) => result, None => Err(SendError::NotFound), diff --git a/rs-lib/tests/integration/mod.rs b/rs-lib/tests/integration/mod.rs index c5f76f0..c456728 100644 --- a/rs-lib/tests/integration/mod.rs +++ b/rs-lib/tests/integration/mod.rs @@ -27,6 +27,7 @@ macro_rules! assert_files { .map(|(file_path, file_text)| deno_node_transform::OutputFile { file_path: std::path::PathBuf::from(file_path), file_text: file_text.to_string(), + bytes: None, }) .collect::>(); expected.sort_by(|a, b| a.file_path.cmp(&b.file_path)); @@ -57,7 +58,7 @@ pub async fn assert_transforms(files: Vec<(&str, &str)>) { test_builder .with_loader(|loader| { for (file_name, file) in files.iter() { - loader.add_local_file(&format!("/{}", file_name), file.0); + loader.add_local_file(format!("/{}", file_name), file.0); } loader.add_local_file("/example.js", ""); }) diff --git a/rs-lib/tests/integration_test.rs b/rs-lib/tests/integration_test.rs index 76883e1..cd1ae08 100644 --- a/rs-lib/tests/integration_test.rs +++ b/rs-lib/tests/integration_test.rs @@ -1288,7 +1288,7 @@ async fn transform_deno_types_and_type_ref_for_different_remote_file() { .main .files .iter() - .find(|f| f.file_path == PathBuf::from("deps/localhost/file.d.ts")) + .find(|f| f.file_path == std::path::Path::new("deps/localhost/file.d.ts")) .unwrap() .file_text, "declare function test2(): number;" @@ -2653,7 +2653,7 @@ async fn polyfills_all() { assert_files!( result.test.files, - &[("mod.test.ts", concat!("import * as mod from './mod.js';",),)] + &[("mod.test.ts", "import * as mod from './mod.js';")] ); assert_eq!(result.test.entry_points, &[PathBuf::from("mod.test.ts")]); } @@ -3562,3 +3562,42 @@ fn add_config_discovery_files(loader: &mut InMemoryLoader) { ) .add_local_file("/other.ts", "export function test() {}"); } + +#[tokio::test] +async fn transform_wasm_imports() { + let wasm_bytes = &[0, 97, 115, 109, 1, 0, 0, 0]; + let result = TestBuilder::new() + .with_loader(|loader| { + loader.add_local_file("/mod.ts", r#"import wasm from "./math.wasm";"#); + loader.add_local_bytes("/math.wasm", wasm_bytes); + }) + .transform() + .await + .unwrap(); + + let mod_file = result + .main + .files + .iter() + .find(|f| f.file_path.to_string_lossy() == "mod.ts") + .unwrap(); + assert_eq!(mod_file.file_text, r#"import wasm from "./math.wasm.js";"#); + + let wasm_file = result + .main + .files + .iter() + .find(|f| f.file_path.to_string_lossy() == "math.wasm") + .unwrap(); + assert_eq!(wasm_file.bytes.as_deref(), Some(wasm_bytes.as_slice())); + + let wrapper_file = result + .main + .files + .iter() + .find(|f| f.file_path.to_string_lossy() == "math.wasm.js") + .unwrap(); + assert!(wrapper_file + .file_text + .contains("const wasmModule = new WebAssembly.Module(wasmBytes);")); +} diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 24416a9..537b56a 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -1360,6 +1360,39 @@ Deno.test("should handle json modules", async () => { }); }); +Deno.test("should handle wasm modules", async () => { + await runTest("wasm_module_project", { + entryPoints: ["mod.ts"], + outDir: "./npm", + shims: { + ...getAllShimOptions(false), + deno: "dev", + }, + package: { + name: "wasm-module-package", + version: "1.0.0", + }, + typeCheck: false, + declaration: false, + test: false, + }, (output) => { + output.assertExists("esm/math.wasm"); + output.assertExists("script/math.wasm"); + output.assertExists("esm/math.wasm.js"); + output.assertExists("script/math.wasm.js"); + const esmWrapperText = output.getFileText("esm/math.wasm.js"); + + assertStringIncludes( + esmWrapperText, + "export const add = wasmInstance.exports.add;", + ); + assertStringIncludes( + esmWrapperText, + "export default wasmInstance.exports;", + ); + }); +}); + Deno.test("should build project with another package manager", async () => { await runTest("test_project", { entryPoints: ["mod.ts"], @@ -1878,6 +1911,7 @@ async function runTest( | "types_package_project" | "web_socket_project" | "using_decl_project" + | "wasm_module_project" | "workspace_project", options: BuildOptions, checkOutput?: (output: Output) => Promise | void, diff --git a/tests/wasm_module_project/math.wasm b/tests/wasm_module_project/math.wasm new file mode 100644 index 0000000..09508a1 Binary files /dev/null and b/tests/wasm_module_project/math.wasm differ diff --git a/tests/wasm_module_project/mod.test.ts b/tests/wasm_module_project/mod.test.ts new file mode 100644 index 0000000..bd69f37 --- /dev/null +++ b/tests/wasm_module_project/mod.test.ts @@ -0,0 +1,8 @@ +import { callAdd } from "./mod.ts"; + +Deno.test("wasm module test", () => { + const add = callAdd(); + if (typeof add !== "function" && typeof add !== "undefined") { + throw new Error("Unexpected export type"); + } +}); diff --git a/tests/wasm_module_project/mod.ts b/tests/wasm_module_project/mod.ts new file mode 100644 index 0000000..94cb823 --- /dev/null +++ b/tests/wasm_module_project/mod.ts @@ -0,0 +1,6 @@ +// @ts-ignore +import { add } from "./math.wasm"; + +export function callAdd() { + return add; +} diff --git a/transform.ts b/transform.ts index 263521d..d441de7 100644 --- a/transform.ts +++ b/transform.ts @@ -152,6 +152,7 @@ export interface TransformOutputEnvironment { export interface OutputFile { filePath: string; fileText: string; + bytes?: Uint8Array; } /** Analyzes the provided entry point to get all the dependended on modules and