Skip to content

Commit 7aee407

Browse files
authored
feat: Support for external libraries in JS sandbox (#47)
1 parent d2ebcd0 commit 7aee407

File tree

25 files changed

+2013
-151
lines changed

25 files changed

+2013
-151
lines changed

Cargo.lock

Lines changed: 1505 additions & 119 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
[workspace]
22
members = [
33
"rust/usuba",
4-
"rust/usuba-compat"
4+
"rust/usuba-compat",
5+
"rust/usuba-bundle"
56
]
67

78
# See: https://github.com/rust-lang/rust/issues/90148#issuecomment-949194352
@@ -13,11 +14,16 @@ async-trait = { version = "0.1" }
1314
axum = { version = "0.7" }
1415
blake3 = { version = "1.5" }
1516
bytes = { version = "1" }
17+
deno_emit = { version = "0.42" }
18+
deno_graph = { version = "0.78" }
19+
http = { version = "1.1" }
20+
http-body-util = { version = "0.1" }
1621
hyper-util = { version = "0.1", features = ["client", "client-legacy"] }
1722
js-component-bindgen = { version = "1", features = ["transpile-bindgen"] }
1823
js-sys = { version = "0.3" }
1924
mime_guess = { version = "2" }
2025
redb = { version = "2" }
26+
reqwest = { version = "0.12", default-features = false }
2127
rust-embed = { version = "8.4" }
2228
serde = { version = "1" }
2329
serde_json = { version = "1" }
@@ -28,6 +34,8 @@ tower-http = { version = "0.5" }
2834
tracing = { version = "0.1" }
2935
tracing-subscriber = { version = "0.3", features = ["env-filter", "tracing-log", "json"] }
3036
tracing-web = { version = "0.1" }
37+
url = { version = "2" }
38+
usuba-bundle = { path = "./rust/usuba-bundle" }
3139
utoipa = { version = "4" }
3240
utoipa-swagger-ui = { version = "7" }
3341
wasm-bindgen = { version = "0.2" }

rust/usuba-bundle/Cargo.toml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[package]
2+
name = "usuba-bundle"
3+
description = "Code preparation steps for Common modules"
4+
version = "0.1.0"
5+
edition = "2021"
6+
7+
[dependencies]
8+
anyhow = { workspace = true }
9+
bytes = { workspace = true }
10+
deno_emit = { workspace = true }
11+
deno_graph = { workspace = true }
12+
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset", "http2", "macos-system-configuration"] }
13+
tokio = { workspace = true, features = ["rt-multi-thread", "io-util", "process", "fs"] }
14+
tracing = { workspace = true }
15+
url = { workspace = true }
16+
17+
[dev-dependencies]
18+
tracing-subscriber = { workspace = true }

rust/usuba-bundle/src/lib.rs

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#[macro_use]
2+
extern crate tracing;
3+
4+
use anyhow::{anyhow, Result};
5+
use bytes::Bytes;
6+
use deno_emit::{
7+
bundle, BundleOptions, BundleType, EmitOptions, LoadFuture, LoadOptions, Loader,
8+
ModuleSpecifier, SourceMapOption, TranspileOptions,
9+
};
10+
use deno_graph::source::LoadResponse;
11+
use url::Url;
12+
13+
pub struct JavaScriptLoader {
14+
root: Option<Bytes>,
15+
}
16+
17+
impl JavaScriptLoader {
18+
pub fn new(root: Option<Bytes>) -> Self {
19+
Self { root }
20+
}
21+
}
22+
23+
impl Loader for JavaScriptLoader {
24+
fn load(&self, specifier: &ModuleSpecifier, _options: LoadOptions) -> LoadFuture {
25+
let root = self.root.clone();
26+
let specifier = specifier.clone();
27+
28+
debug!("Attempting to load '{}'", specifier);
29+
30+
Box::pin(async move {
31+
match specifier.scheme() {
32+
"usuba" => {
33+
debug!("Usuba!");
34+
Ok(Some(LoadResponse::Module {
35+
content: root
36+
.ok_or_else(|| {
37+
anyhow!("Attempted to load root module, but no root was specified!")
38+
})?
39+
.to_vec()
40+
.into(),
41+
specifier,
42+
maybe_headers: None,
43+
}))
44+
}
45+
"common" => {
46+
debug!("Common!");
47+
Ok(Some(LoadResponse::External {
48+
specifier: specifier.clone(),
49+
}))
50+
}
51+
"https" => {
52+
debug!("Https!");
53+
let response = reqwest::get(specifier.clone()).await?;
54+
let headers = response.headers().to_owned();
55+
let bytes = response.bytes().await?;
56+
let content = bytes.to_vec().into();
57+
58+
trace!("Loaded remote module: {}", String::from_utf8_lossy(&bytes));
59+
Ok(Some(LoadResponse::Module {
60+
content,
61+
specifier,
62+
maybe_headers: Some(
63+
headers
64+
.into_iter()
65+
.filter_map(|(h, v)| {
66+
h.map(|header| {
67+
(
68+
header.to_string(),
69+
v.to_str().unwrap_or_default().to_string(),
70+
)
71+
})
72+
})
73+
.collect(),
74+
),
75+
}))
76+
}
77+
"node" | "npm" => Err(anyhow!(
78+
"Could not import '{specifier}'. Node.js and NPM modules are not supported."
79+
)),
80+
_ => Err(anyhow!(
81+
"Could not import '{specifier}'. Unrecognize specifier format.'"
82+
)),
83+
}
84+
})
85+
}
86+
}
87+
88+
pub struct JavaScriptBundler {}
89+
90+
impl JavaScriptBundler {
91+
fn bundle_options() -> BundleOptions {
92+
BundleOptions {
93+
bundle_type: BundleType::Module,
94+
transpile_options: TranspileOptions::default(),
95+
emit_options: EmitOptions {
96+
source_map: SourceMapOption::None,
97+
source_map_file: None,
98+
inline_sources: false,
99+
remove_comments: true,
100+
},
101+
emit_ignore_directives: false,
102+
minify: false,
103+
}
104+
}
105+
106+
pub async fn bundle_url(url: Url) -> Result<String> {
107+
let mut loader = JavaScriptLoader::new(None);
108+
let emit = bundle(url, &mut loader, None, Self::bundle_options()).await?;
109+
Ok(emit.code)
110+
}
111+
112+
pub async fn bundle_module(module: Bytes) -> Result<String> {
113+
let mut loader = JavaScriptLoader::new(Some(module));
114+
let emit = bundle(
115+
Url::parse("usuba:root")?,
116+
&mut loader,
117+
None,
118+
Self::bundle_options(),
119+
)
120+
.await?;
121+
Ok(emit.code)
122+
}
123+
}
124+
125+
#[cfg(test)]
126+
pub mod tests {
127+
use anyhow::Result;
128+
use url::Url;
129+
130+
use crate::JavaScriptBundler;
131+
132+
#[tokio::test]
133+
async fn it_loads_a_module_from_esm_sh() -> Result<()> {
134+
let candidate = Url::parse("https://esm.sh/canvas-confetti@1.6.0")?;
135+
let bundle = JavaScriptBundler::bundle_url(candidate).await?;
136+
137+
assert!(bundle.len() > 0);
138+
139+
Ok(())
140+
}
141+
142+
#[tokio::test]
143+
async fn it_loads_a_module_from_deno_land() -> Result<()> {
144+
let candidate = Url::parse("https://deno.land/x/zod@v3.16.1/mod.ts")?;
145+
let bundle = JavaScriptBundler::bundle_url(candidate).await?;
146+
147+
assert!(bundle.len() > 0);
148+
149+
Ok(())
150+
}
151+
152+
#[tokio::test]
153+
async fn it_can_bundle_a_module_file() -> Result<()> {
154+
let candidate = format!(
155+
r#"export * from "https://esm.sh/canvas-confetti@1.6.0";
156+
"#
157+
);
158+
let bundle = JavaScriptBundler::bundle_module(candidate.into()).await?;
159+
160+
assert!(bundle.len() > 0);
161+
162+
Ok(())
163+
}
164+
165+
#[tokio::test]
166+
async fn it_skips_common_modules_when_bundling() -> Result<()> {
167+
let candidate = format!(
168+
r#"
169+
import {{ read, write }} from "common:io/state@0.0.1";
170+
171+
// Note: must use imports else they are tree-shaken
172+
// Caveat: cannot re-export built-ins as it provokes bundling
173+
console.log(read, write);
174+
"#
175+
);
176+
177+
let bundle = JavaScriptBundler::bundle_module(candidate.into())
178+
.await
179+
.map_err(|error| {
180+
error!("{}", error);
181+
error
182+
})
183+
.unwrap();
184+
185+
debug!("{bundle}");
186+
187+
assert!(bundle.contains("import { read, write } from \"common:io/state@0.0.1\""));
188+
189+
Ok(())
190+
}
191+
}

rust/usuba/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "usuba"
3-
description = "An anything-to-Common-Wasm build server"
3+
description = "A Common server"
44
version = "0.1.0"
55
edition = "2021"
66

@@ -24,6 +24,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "io-util", "process",
2424
tower-http = { workspace = true, features = ["cors"] }
2525
tracing = { workspace = true }
2626
tracing-subscriber = { workspace = true }
27+
usuba-bundle = { workspace = true }
2728
utoipa = { workspace = true, features = ["axum_extras"] }
2829
utoipa-swagger-ui = { workspace = true, features = ["axum"] }
2930
wit-parser = { workspace = true }

rust/usuba/src/bake/javascript.rs

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use tempfile::TempDir;
77

88
use tokio::process::Command;
99
use tokio::task::JoinSet;
10+
use usuba_bundle::JavaScriptBundler;
1011

1112
use crate::write_file;
1213

@@ -18,7 +19,7 @@ impl Bake for JavaScriptBaker {
1819
#[instrument]
1920
async fn bake(
2021
&self,
21-
world: &str,
22+
_world: &str,
2223
wit: Vec<Bytes>,
2324
source_code: Bytes,
2425
library: Vec<Bytes>,
@@ -29,6 +30,12 @@ impl Bake for JavaScriptBaker {
2930
workspace.path().display()
3031
);
3132

33+
let bundled_source_code = tokio::task::spawn_blocking(move || {
34+
tokio::runtime::Handle::current()
35+
.block_on(JavaScriptBundler::bundle_module(source_code))
36+
})
37+
.await??;
38+
3239
let wasm_path = workspace.path().join("module.wasm");
3340
let js_path = workspace.path().join("module.js");
3441

@@ -44,7 +51,10 @@ impl Bake for JavaScriptBaker {
4451
wit.into_iter()
4552
.enumerate()
4653
.map(|(i, wit)| write_file(wit_path.join(format!("module{}.wit", i)), wit))
47-
.chain([write_file(js_path.clone(), source_code)])
54+
.chain([write_file(
55+
js_path.clone(),
56+
Bytes::from(bundled_source_code),
57+
)])
4858
.chain(
4959
library.into_iter().enumerate().map(|(i, wit)| {
5060
write_file(wit_deps_path.join(format!("library{}.wit", i)), wit)

rust/usuba/src/bin/usuba.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,13 @@ extern crate tracing;
33

44
use std::net::SocketAddr;
55

6-
use tracing::Level;
7-
use tracing_subscriber::{fmt::Layer, layer::SubscriberExt, FmtSubscriber};
6+
use tracing_subscriber::{fmt::Layer, layer::SubscriberExt, EnvFilter, FmtSubscriber};
87
use usuba::{serve, UsubaError};
98

109
#[tokio::main]
1110
pub async fn main() -> Result<(), UsubaError> {
1211
let subscriber = FmtSubscriber::builder()
13-
.with_max_level(Level::TRACE)
12+
.with_env_filter(EnvFilter::from_default_env())
1413
.finish();
1514
tracing::subscriber::set_global_default(subscriber.with(Layer::default().pretty()))?;
1615

rust/usuba/src/openapi.rs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
use utoipa::OpenApi;
22

33
use crate::{
4-
routes::{BuildModuleRequest, BuildModuleResponse},
4+
routes::{BuildModuleRequest, BuildModuleResponse, BundleRequest},
55
ErrorResponse,
66
};
77

88
#[derive(OpenApi)]
99
#[openapi(
10-
paths(crate::routes::build_module, crate::routes::retrieve_module),
10+
paths(
11+
crate::routes::build_module,
12+
crate::routes::retrieve_module,
13+
crate::routes::bundle_javascript
14+
),
1115
components(
1216
schemas(BuildModuleResponse),
1317
schemas(ErrorResponse),
14-
schemas(BuildModuleRequest)
18+
schemas(BuildModuleRequest),
19+
schemas(BundleRequest)
1520
)
1621
)]
1722
pub struct OpenApiDocs;
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
use axum::extract::Multipart;
2+
use usuba_bundle::JavaScriptBundler;
3+
use utoipa::ToSchema;
4+
5+
use crate::UsubaError;
6+
7+
#[derive(ToSchema)]
8+
pub struct BundleRequest {
9+
pub source: Vec<Vec<u8>>,
10+
}
11+
12+
#[utoipa::path(
13+
post,
14+
path = "/api/v0/bundle",
15+
request_body(content = BundleRequest, content_type = "multipart/form-data"),
16+
responses(
17+
(status = 200, description = "Successfully built the module", body = String, content_type = "text/javascript"),
18+
(status = 400, description = "Bad request body", body = ErrorResponse),
19+
(status = 500, description = "Internal error", body = ErrorResponse)
20+
)
21+
)]
22+
pub async fn bundle_javascript(mut form_data: Multipart) -> Result<String, UsubaError> {
23+
let first_field = if let Some(field) = form_data.next_field().await? {
24+
field
25+
} else {
26+
return Err(UsubaError::BadRequest);
27+
};
28+
29+
match first_field.name() {
30+
Some("source") => match first_field.file_name() {
31+
Some(name) if name.ends_with(".js") => {
32+
let source_code = first_field.bytes().await?;
33+
return Ok(tokio::task::spawn_blocking(move || {
34+
tokio::runtime::Handle::current()
35+
.block_on(JavaScriptBundler::bundle_module(source_code))
36+
})
37+
.await??);
38+
}
39+
_ => warn!("Skipping unexpected content type"),
40+
},
41+
_ => warn!("Skipping unexpected multipart content"),
42+
}
43+
44+
Err(UsubaError::BadRequest)
45+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
mod javascript;
2+
3+
pub use javascript::*;

0 commit comments

Comments
 (0)