Add an embed_module!() macro to boa_interop (#3784)

* Add an embed_module!() macro to boa_interop

The macro creates a ModuleLoader that includes all JS files from a directory.

* Add description of function

* Run prettier

* Remove reference to a unaccessible crate

* Remove one more reference to a unaccessible crate

* Disable test that plays with paths on Windows

* Block the whole test module instead of just the fn

* This is a bit insane

* Replace path separators into JavaScript specifier separators

* cargo fmt

* cargo fmt part deux

* fix some issues with relative path and pathing on windows

* fix module resolver when there are no base path

* use the platform's path separator

* cargo fmt

* prettier

* Remove caching of the error

* Pedantic clippy gonna pedant

Cargo.lock

@@ -511,6 +511,7 @@ version = "0.18.0"
 dependencies = [
  "boa_engine",
  "boa_gc",
+ "boa_macros",
  "rustc-hash",
 ]
 

core/engine/src/module/loader.rs

@@ -48,7 +48,7 @@ pub fn resolve_module_specifier(
     referrer: Option<&Path>,
     _context: &mut Context,
 ) -> JsResult<PathBuf> {
-    let base = base.map_or_else(|| PathBuf::from(""), PathBuf::from);
+    let base_path = base.map_or_else(|| PathBuf::from(""), PathBuf::from);
     let referrer_dir = referrer.and_then(|p| p.parent());
 
     let specifier = specifier.to_std_string_escaped();
@@ -65,17 +65,17 @@ pub fn resolve_module_specifier(
 
     let long_path = if is_relative {
         if let Some(r_path) = referrer_dir {
-            base.join(r_path).join(short_path)
+            base_path.join(r_path).join(short_path)
         } else {
             return Err(JsError::from_opaque(
                 js_string!("relative path without referrer").into(),
             ));
         }
     } else {
-        base.join(&specifier)
+        base_path.join(&specifier)
     };
 
-    if long_path.is_relative() {
+    if long_path.is_relative() && base.is_some() {
         return Err(JsError::from_opaque(
             js_string!("resolved path is relative").into(),
         ));
@@ -100,7 +100,7 @@ pub fn resolve_module_specifier(
             Ok(acc)
         })?;
 
-    if path.starts_with(&base) {
+    if path.starts_with(&base_path) {
         Ok(path)
     } else {
         Err(JsError::from_opaque(
@@ -371,6 +371,39 @@ mod tests {
         assert_eq!(actual.map_err(|_| ()), expected.map(PathBuf::from));
     }
 
+    // This tests the same cases as the previous test, but without a base path.
+    #[rustfmt::skip]
+    #[cfg(target_family = "unix")]
+    #[test_case(Some("hello/ref.js"),       "a.js",             Ok("a.js"))]
+    #[test_case(Some("base/ref.js"),        "./b.js",           Ok("base/b.js"))]
+    #[test_case(Some("base/other/ref.js"),  "./c.js",           Ok("base/other/c.js"))]
+    #[test_case(Some("base/other/ref.js"),  "../d.js",          Ok("base/d.js"))]
+    #[test_case(Some("base/ref.js"),        "e.js",             Ok("e.js"))]
+    #[test_case(Some("base/ref.js"),        "./f.js",           Ok("base/f.js"))]
+    #[test_case(Some("./ref.js"),           "./g.js",           Ok("g.js"))]
+    #[test_case(Some("./other/ref.js"),     "./other/h.js",     Ok("other/other/h.js"))]
+    #[test_case(Some("./other/ref.js"),     "./other/../h1.js", Ok("other/h1.js"))]
+    #[test_case(Some("./other/ref.js"),     "./../h2.js",       Ok("h2.js"))]
+    #[test_case(None,                       "./i.js",           Err(()))]
+    #[test_case(None,                       "j.js",             Ok("j.js"))]
+    #[test_case(None,                       "other/k.js",       Ok("other/k.js"))]
+    #[test_case(None,                       "other/../../l.js", Err(()))]
+    #[test_case(Some("/base/ref.js"),       "other/../../m.js", Err(()))]
+    #[test_case(None,                       "../n.js",          Err(()))]
+    fn resolve_test_no_base(ref_path: Option<&str>, spec: &str, expected: Result<&str, ()>) {
+        let mut context = Context::default();
+        let spec = js_string!(spec);
+        let ref_path = ref_path.map(PathBuf::from);
+
+        let actual = resolve_module_specifier(
+            None,
+            &spec,
+            ref_path.as_deref(),
+            &mut context,
+        );
+        assert_eq!(actual.map_err(|_| ()), expected.map(PathBuf::from));
+    }
+
     #[rustfmt::skip]
     #[cfg(target_family = "windows")]
     #[test_case(Some("a:\\hello\\ref.js"),       "a.js",                Ok("a:\\base\\a.js"))]

core/interop/Cargo.toml

@@ -13,6 +13,7 @@ rust-version.workspace = true
 [dependencies]
 boa_engine.workspace = true
 boa_gc.workspace = true
+boa_macros.workspace = true
 rustc-hash = { workspace = true, features = ["std"] }
 
 [lints]

core/interop/src/lib.rs

@@ -3,6 +3,7 @@
 use boa_engine::module::SyntheticModuleInitializer;
 use boa_engine::value::TryFromJs;
 use boa_engine::{Context, JsResult, JsString, JsValue, Module, NativeFunction};
+pub use boa_macros;
 
 pub mod loaders;
 

core/interop/src/loaders.rs

@@ -3,4 +3,5 @@
 
 pub use hashmap::HashMapModuleLoader;
 
+pub mod embedded;
 pub mod hashmap;

core/interop/src/loaders/embedded.rs

@@ -0,0 +1,139 @@
+//! Embedded module loader. Creates a `ModuleLoader` instance that contains
+//! files embedded in the binary at build time.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::path::Path;
+
+use boa_engine::module::{ModuleLoader, Referrer};
+use boa_engine::{Context, JsNativeError, JsResult, JsString, Module, Source};
+
+/// Create a module loader that embeds files from the filesystem at build
+/// time. This is useful for bundling assets with the binary.
+///
+/// By default, will error if the total file size exceeds 1MB. This can be
+/// changed by specifying the `max_size` parameter.
+///
+/// The embedded module will only contain files that have the `.js`, `.mjs`,
+/// or `.cjs` extension.
+#[macro_export]
+macro_rules! embed_module {
+    ($path: literal, max_size = $max_size: literal) => {
+        $crate::loaders::embedded::EmbeddedModuleLoader::from_iter(
+            $crate::boa_macros::embed_module_inner!($path, $max_size),
+        )
+    };
+    ($path: literal) => {
+        embed_module!($path, max_size = 1_048_576)
+    };
+}
+
+#[derive(Debug, Clone)]
+enum EmbeddedModuleEntry {
+    Source(JsString, &'static [u8]),
+    Module(Module),
+}
+
+impl EmbeddedModuleEntry {
+    fn from_source(path: JsString, source: &'static [u8]) -> Self {
+        Self::Source(path, source)
+    }
+
+    fn cache(&mut self, context: &mut Context) -> JsResult<&Module> {
+        if let Self::Source(path, source) = self {
+            let mut bytes: &[u8] = source;
+            let path = path.to_std_string_escaped();
+            let source = Source::from_reader(&mut bytes, Some(Path::new(&path)));
+            match Module::parse(source, None, context) {
+                Ok(module) => {
+                    *self = Self::Module(module);
+                }
+                Err(err) => {
+                    return Err(err);
+                }
+            }
+        };
+
+        match self {
+            Self::Module(module) => Ok(module),
+            EmbeddedModuleEntry::Source(_, _) => unreachable!(),
+        }
+    }
+
+    fn as_module(&self) -> Option<&Module> {
+        match self {
+            Self::Module(module) => Some(module),
+            Self::Source(_, _) => None,
+        }
+    }
+}
+
+/// The resulting type of creating an embedded module loader.
+#[derive(Debug, Clone)]
+#[allow(clippy::module_name_repetitions)]
+pub struct EmbeddedModuleLoader {
+    map: HashMap<JsString, RefCell<EmbeddedModuleEntry>>,
+}
+
+impl FromIterator<(&'static str, &'static [u8])> for EmbeddedModuleLoader {
+    fn from_iter<T: IntoIterator<Item = (&'static str, &'static [u8])>>(iter: T) -> Self {
+        Self {
+            map: iter
+                .into_iter()
+                .map(|(path, source)| {
+                    let p = JsString::from(path);
+                    (
+                        p.clone(),
+                        RefCell::new(EmbeddedModuleEntry::from_source(p, source)),
+                    )
+                })
+                .collect(),
+        }
+    }
+}
+
+impl ModuleLoader for EmbeddedModuleLoader {
+    fn load_imported_module(
+        &self,
+        referrer: Referrer,
+        specifier: JsString,
+        finish_load: Box<dyn FnOnce(JsResult<Module>, &mut Context)>,
+        context: &mut Context,
+    ) {
+        let Ok(specifier_path) = boa_engine::module::resolve_module_specifier(
+            None,
+            &specifier,
+            referrer.path(),
+            context,
+        ) else {
+            let err = JsNativeError::typ().with_message(format!(
+                "could not resolve module specifier `{}`",
+                specifier.to_std_string_escaped()
+            ));
+            finish_load(Err(err.into()), context);
+            return;
+        };
+
+        if let Some(module) = self
+            .map
+            .get(&JsString::from(specifier_path.to_string_lossy().as_ref()))
+        {
+            let mut embedded = module.borrow_mut();
+            let module = embedded.cache(context);
+
+            finish_load(module.cloned(), context);
+        } else {
+            let err = JsNativeError::typ().with_message(format!(
+                "could not find module `{}`",
+                specifier.to_std_string_escaped()
+            ));
+            finish_load(Err(err.into()), context);
+        }
+    }
+
+    fn get_module(&self, specifier: JsString) -> Option<Module> {
+        self.map
+            .get(&specifier)
+            .and_then(|module| module.borrow().as_module().cloned())
+    }
+}

core/interop/tests/embedded.rs

@@ -0,0 +1,67 @@
+#![allow(unused_crate_dependencies)]
+
+use std::rc::Rc;
+
+use boa_engine::builtins::promise::PromiseState;
+use boa_engine::module::ModuleLoader;
+use boa_engine::{js_string, Context, JsString, JsValue, Module, Source};
+use boa_interop::embed_module;
+
+#[test]
+fn simple() {
+    #[cfg(target_family = "unix")]
+    let module_loader = Rc::new(embed_module!("tests/embedded/"));
+    #[cfg(target_family = "windows")]
+    let module_loader = Rc::new(embed_module!("tests\\embedded\\"));
+
+    let mut context = Context::builder()
+        .module_loader(module_loader.clone())
+        .build()
+        .unwrap();
+
+    // Resolving modules that exist but haven't been cached yet should return None.
+    assert_eq!(module_loader.get_module(JsString::from("/file1.js")), None);
+    assert_eq!(
+        module_loader.get_module(JsString::from("/non-existent.js")),
+        None
+    );
+
+    let module = Module::parse(
+        Source::from_bytes(b"export { bar } from '/file1.js';"),
+        None,
+        &mut context,
+    )
+    .expect("failed to parse module");
+    let promise = module.load_link_evaluate(&mut context);
+    context.run_jobs();
+
+    match promise.state() {
+        PromiseState::Fulfilled(value) => {
+            assert!(
+                value.is_undefined(),
+                "Expected undefined, got {}",
+                value.display()
+            );
+
+            let bar = module
+                .namespace(&mut context)
+                .get(js_string!("bar"), &mut context)
+                .unwrap()
+                .as_callable()
+                .cloned()
+                .unwrap();
+            let value = bar.call(&JsValue::undefined(), &[], &mut context).unwrap();
+            assert_eq!(
+                value.as_number(),
+                Some(6.),
+                "Expected 6, got {}",
+                value.display()
+            );
+        }
+        PromiseState::Rejected(err) => panic!(
+            "promise was not fulfilled: {:?}",
+            err.to_string(&mut context)
+        ),
+        PromiseState::Pending => panic!("Promise was not settled"),
+    }
+}

core/interop/tests/embedded/dir1/file3.js

@@ -0,0 +1,2 @@
+// Enable this when https://github.com/boa-dev/boa/pull/3781 is fixed and merged.
+export { foo } from "./file4.js";

core/interop/tests/embedded/dir1/file4.js

@@ -0,0 +1,3 @@
+export function foo() {
+  return 3;
+}

core/interop/tests/embedded/file1.js

@@ -0,0 +1,6 @@
+import { foo } from "./file2.js";
+import { foo as foo2 } from "./dir1/file3.js";
+
+export function bar() {
+  return foo() + foo2() + 1;
+}

core/interop/tests/embedded/file2.js

@@ -0,0 +1,3 @@
+export function foo() {
+  return 2;
+}

core/macros/src/embedded_module_loader.rs

@@ -0,0 +1,118 @@
+//! Embedded module loader. Creates a `ModuleLoader` instance that contains
+//! files embedded in the binary at build time.
+
+use proc_macro::TokenStream;
+use std::path::PathBuf;
+
+use quote::quote;
+use syn::{parse::Parse, LitInt, LitStr, Token};
+
+struct EmbedModuleMacroInput {
+    path: LitStr,
+    max_size: u64,
+}
+
+impl Parse for EmbedModuleMacroInput {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let path = input.parse()?;
+        let _comma: Token![,] = input.parse()?;
+        let max_size = input.parse::<LitInt>()?.base10_parse()?;
+
+        Ok(Self { path, max_size })
+    }
+}
+
+/// List all the files readable from the given directory, recursively.
+fn find_all_files(dir: &mut std::fs::ReadDir, root: &PathBuf) -> Vec<PathBuf> {
+    let mut files = Vec::new();
+    for entry in dir {
+        let Ok(entry) = entry else {
+            continue;
+        };
+
+        let path = entry.path();
+        if path.is_dir() {
+            let Ok(mut sub_dir) = std::fs::read_dir(&path) else {
+                continue;
+            };
+            files.append(&mut find_all_files(&mut sub_dir, root));
+        } else if let Ok(path) = path.strip_prefix(root) {
+            files.push(path.to_path_buf());
+        }
+    }
+    files
+}
+
+/// Implementation of the `embed_module_inner!` macro.
+/// This should not be used directly. Use the `embed_module!` macro from the `boa_interop`
+/// crate instead.
+pub(crate) fn embed_module_impl(input: TokenStream) -> TokenStream {
+    let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap_or_default());
+
+    let input = syn::parse_macro_input!(input as EmbedModuleMacroInput);
+
+    let root = manifest_dir.join(input.path.value());
+    let max_size = input.max_size;
+
+    let mut dir = match std::fs::read_dir(root.clone()) {
+        Ok(dir) => dir,
+        Err(e) => {
+            return syn::Error::new_spanned(
+                input.path.clone(),
+                format!("Error reading directory: {e}"),
+            )
+            .to_compile_error()
+            .into();
+        }
+    };
+
+    let mut total = 0;
+    let files = find_all_files(&mut dir, &root);
+
+    let inner = match files.into_iter().try_fold(quote! {}, |acc, relative_path| {
+        let path = root.join(&relative_path);
+        let absolute_path = manifest_dir.join(&path).to_string_lossy().to_string();
+        let Some(relative_path) = relative_path.to_str() else {
+            return Err(syn::Error::new_spanned(
+                input.path.clone(),
+                "Path has non-Unicode characters",
+            ));
+        };
+        let relative_path = format!("{}{}", std::path::MAIN_SEPARATOR, relative_path);
+
+        // Check the size.
+        let size = std::fs::metadata(&path)
+            .map_err(|e| {
+                syn::Error::new_spanned(input.path.clone(), format!("Error reading file size: {e}"))
+            })?
+            .len();
+
+        total += size;
+        if total > max_size {
+            return Err(syn::Error::new_spanned(
+                input.path.clone(),
+                "Total embedded file size exceeds the maximum size",
+            ));
+        }
+
+        Ok(quote! {
+            #acc
+
+            (
+                #relative_path,
+                include_bytes!(#absolute_path).as_ref(),
+            ),
+        })
+    }) {
+        Ok(inner) => inner,
+        Err(e) => return e.to_compile_error().into(),
+    };
+
+    let stream = quote! {
+        [
+            #inner
+        ]
+    };
+
+    stream.into()
+}

core/macros/src/lib.rs

@@ -16,6 +16,19 @@ use syn::{
 };
 use synstructure::{decl_derive, AddBounds, Structure};
 
+mod embedded_module_loader;
+
+/// Implementation of the inner iterator of the `embed_module!` macro. All
+/// arguments are required.
+///
+/// # Warning
+/// This should not be used directly as is, and instead should be used through
+/// the `embed_module!` macro in `boa_interop` for convenience.
+#[proc_macro]
+pub fn embed_module_inner(input: TokenStream) -> TokenStream {
+    embedded_module_loader::embed_module_impl(input)
+}
+
 struct Static {
     literal: LitStr,
     ident: Ident,