Skip to content

Commit ffefbf9

Browse files
cdatacdata
committed
feat: Python baker and frontend support
1 parent 68c6e20 commit ffefbf9

File tree

11 files changed

+205
-30
lines changed

11 files changed

+205
-30
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ target
22
.wireit
33
node_modules
44
dist
5-
*.tsbuildinfo
5+
lib
6+
*.tsbuildinfo

‎rust/usuba/src/bake/fs.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
use std::{io::Cursor, path::PathBuf};
2+
3+
use bytes::Bytes;
4+
5+
use crate::UsubaError;
6+
7+
pub async fn write_file(path: PathBuf, bytes: Bytes) -> Result<(), UsubaError> {
8+
let mut file = tokio::fs::File::create(&path).await?;
9+
let mut cursor = Cursor::new(bytes.as_ref());
10+
tokio::io::copy(&mut cursor, &mut file).await?;
11+
Ok(())
12+
}

‎rust/usuba/src/bake/javascript.rs‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
1-
use std::io::Cursor;
2-
use std::path::{Path, PathBuf};
31
use tracing::instrument;
42

5-
use crate::UsubaError;
6-
73
use super::Bake;
84
use async_trait::async_trait;
95
use bytes::Bytes;
@@ -12,14 +8,7 @@ use tempfile::TempDir;
128
use tokio::process::Command;
139
use tokio::task::JoinSet;
1410

15-
use wit_parser::UnresolvedPackage;
16-
17-
async fn write_file(path: PathBuf, bytes: Bytes) -> Result<(), UsubaError> {
18-
let mut file = tokio::fs::File::create(&path).await?;
19-
let mut cursor = Cursor::new(bytes.as_ref());
20-
tokio::io::copy(&mut cursor, &mut file).await?;
21-
Ok(())
22-
}
11+
use crate::write_file;
2312

2413
#[derive(Debug)]
2514
pub struct JavaScriptBaker {}

‎rust/usuba/src/bake/mod.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
mod bake;
2+
mod fs;
23
mod javascript;
4+
mod python;
35

46
pub use bake::*;
7+
pub use fs::*;
58
pub use javascript::*;
9+
pub use python::*;
610

711
use async_trait::async_trait;
812
use bytes::Bytes;
913

1014
pub enum Baker {
1115
JavaScript,
16+
Python,
1217
}
1318

1419
#[async_trait]
@@ -26,6 +31,11 @@ impl Bake for Baker {
2631
.bake(world, wit, source_code, library)
2732
.await
2833
}
34+
Baker::Python => {
35+
(PythonBaker {})
36+
.bake(world, wit, source_code, library)
37+
.await
38+
}
2939
}
3040
}
3141
}

‎rust/usuba/src/bake/python.rs‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
use async_trait::async_trait;
2+
use bytes::Bytes;
3+
use tempfile::TempDir;
4+
use tokio::{process::Command, task::JoinSet};
5+
6+
use crate::{write_file, Bake};
7+
8+
#[derive(Debug)]
9+
pub struct PythonBaker {}
10+
11+
#[async_trait]
12+
impl Bake for PythonBaker {
13+
#[instrument]
14+
async fn bake(
15+
&self,
16+
world: &str,
17+
wit: Vec<Bytes>,
18+
source_code: Bytes,
19+
library: Vec<Bytes>,
20+
) -> Result<Bytes, crate::UsubaError> {
21+
let workspace = TempDir::new()?;
22+
debug!(
23+
"Created temporary workspace in {}",
24+
workspace.path().display()
25+
);
26+
27+
let wasm_path = workspace.path().join("module.wasm");
28+
let python_path = workspace.path().join("module.py");
29+
30+
debug!(?workspace, "Created temporary workspace");
31+
32+
let wit_path = workspace.path().join("wit");
33+
let wit_deps_path = wit_path.join("deps");
34+
35+
tokio::fs::create_dir_all(&wit_deps_path).await?;
36+
37+
let mut writes = JoinSet::new();
38+
39+
wit.into_iter()
40+
.enumerate()
41+
.map(|(i, wit)| write_file(wit_path.join(format!("module{}.wit", i)), wit))
42+
.chain([write_file(python_path.clone(), source_code)])
43+
.chain(
44+
library.into_iter().enumerate().map(|(i, wit)| {
45+
write_file(wit_deps_path.join(format!("library{}.wit", i)), wit)
46+
}),
47+
)
48+
.for_each(|fut| {
49+
writes.spawn(fut);
50+
});
51+
52+
while let Some(result) = writes.try_join_next() {
53+
result??;
54+
continue;
55+
}
56+
57+
debug!(?workspace, "Populated temporary input files");
58+
59+
let mut command = Command::new("componentize-py");
60+
61+
command
62+
.current_dir(workspace.path())
63+
.arg("-d")
64+
.arg(wit_path)
65+
.arg("-w")
66+
.arg(world)
67+
.arg("componentize")
68+
.arg("-p")
69+
.arg(workspace.path().display().to_string())
70+
.arg("-o")
71+
.arg("module.wasm")
72+
.arg("module");
73+
74+
let child = command.spawn()?;
75+
let output = child.wait_with_output().await?;
76+
77+
if output.stderr.len() > 0 {
78+
warn!("{}", String::from_utf8_lossy(&output.stderr));
79+
}
80+
81+
debug!("Finished building with componentize-py");
82+
83+
let wasm_bytes = tokio::fs::read(&wasm_path).await?;
84+
85+
info!("Finished baking");
86+
87+
Ok(wasm_bytes.into())
88+
}
89+
}

‎rust/usuba/src/routes/module/build.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ pub async fn build_module(
8585
source_code = Some(field.bytes().await?);
8686
baker = Some(Baker::JavaScript);
8787
}
88+
Some("py") => {
89+
source_code = Some(field.bytes().await?);
90+
baker = Some(Baker::Python);
91+
}
8892
_ => (),
8993
};
9094
}
@@ -106,6 +110,7 @@ pub async fn build_module(
106110
id: hash.to_string(),
107111
})
108112
} else {
113+
warn!("Insufficient payload inputs to build the module");
109114
Err(UsubaError::BadRequest)
110115
}
111116
}

‎typescript/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,4 @@
4242
]
4343
}
4444
}
45-
}
45+
}

‎typescript/packages/runtime-demo/src/index.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,5 +146,65 @@ export const demoTwo = async () => {
146146
console.log('fin');
147147
};
148148

149+
/**
150+
* Demo Three
151+
*/
152+
const EXAMPLE_HELLO_PY = `
153+
import random
154+
import hello
155+
from hello.imports import lookup
156+
157+
class Hello(hello.Hello):
158+
def hello(self) -> str:
159+
return "Hello, Agent %s!" % lookup.entry(random.randint(0, 9))
160+
`;
161+
162+
export const demoThree = async () => {
163+
console.log('Initializing first Runtime');
164+
165+
const rtOne = new Runtime([]);
166+
167+
console.log('Defining first Module');
168+
169+
type ExpectedExportsOne = {
170+
lookup: {
171+
entry(index: number): string;
172+
};
173+
};
174+
175+
const moduleOne = await rtOne.defineModule<ExpectedExportsOne>({
176+
contentType: 'text/javascript',
177+
wit: COMMON_DIRECTORY_WIT,
178+
sourceCode: COMMON_DIRECTORY_JS,
179+
});
180+
181+
const rtTwo = new Runtime([COMMON_DIRECTORY_WIT]);
182+
183+
console.log('Defining second Module');
184+
185+
type ExpectedExportsTwo = {
186+
hello: () => string;
187+
};
188+
189+
const moduleTwo = await rtTwo.defineModule<ExpectedExportsTwo>({
190+
contentType: 'text/x-python',
191+
wit: EXAMPLE_HELLO_WIT,
192+
sourceCode: EXAMPLE_HELLO_PY,
193+
});
194+
195+
console.log('Instantiating both Modules');
196+
197+
const { hello } = await moduleTwo.instantiate({
198+
'common:directory/lookup': (await moduleOne.instantiate({})).lookup,
199+
});
200+
201+
console.log('Invoking final Module API:');
202+
203+
console.log(`%c${hello()}`, 'font-size: 1.5em; font-weight: bold;');
204+
205+
console.log('fin');
206+
};
207+
149208
(self as any).demoOne = demoOne;
150209
(self as any).demoTwo = demoTwo;
210+
(self as any).demoThree = demoThree;

‎typescript/packages/usuba-rt/src/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import * as apiClient from '@commontools/usuba-api';
33
export type SourceCode = string | Uint8Array;
44
export type PendingSourceCode = SourceCode | Promise<SourceCode>;
55

6-
export type ContentType = 'text/javascript';
6+
export type ContentType = 'text/javascript' | 'text/x-python';
77

88
export type ContentTypeFileExtensions = {
99
[C in ContentType]: string;
@@ -31,6 +31,7 @@ export type ImportableMap = {
3131

3232
const FILE_EXTENSIONS: ContentTypeFileExtensions = {
3333
'text/javascript': 'js',
34+
'text/x-python': 'py',
3435
};
3536

3637
/**

‎typescript/packages/usuba-sw/src/index.ts‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import * as apiClient from '@commontools/usuba-api';
22
import { polyfill, hash } from './usuba_compat/usuba_compat.component.js';
33

4-
const SERVICE_WORKER_VERSION = '0.0.1';
4+
const SERVICE_WORKER_VERSION = '0.0.1-alpha.6';
55

66
self.addEventListener('install', (_event) => {
77
console.log(
8-
`Usuba Service Worker installed (version ${SERVICE_WORKER_VERSION}`
8+
`Usuba Service Worker installed (version ${SERVICE_WORKER_VERSION})`
99
);
1010
});
1111

@@ -216,6 +216,7 @@ const buildOnDemandModule = (event: FetchEvent, url: URL) => {
216216
const buildRuntimeModule = (event: FetchEvent, url: URL) => {
217217
event.respondWith(
218218
(async () => {
219+
console.log('Preparing to build Runtime Module...');
219220
const formData = await event.request.formData();
220221
const moduleFiles = formData.getAll('module') as File[];
221222
const libraryFiles = formData.getAll('library') as File[];
@@ -250,19 +251,8 @@ const buildRuntimeModule = (event: FetchEvent, url: URL) => {
250251
exports: _exports,
251252
} = await buildModule(moduleSlug, moduleFiles, libraryFiles, 'manual');
252253

253-
const wasiShimImports = [];
254-
255-
for (const specifier of Object.values(WASI_SHIM_MAP)) {
256-
const trimmedSpecifier = specifier.split('#').shift();
257-
wasiShimImports.push(`'${trimmedSpecifier}': import('${specifier}')`);
258-
}
259-
260-
// const wrapperModule = `export * from '${ON_DEMAND_TRANSPILED_MODULE_DIRNAME}/${moduleSlug}.js'`;
261254
const wrapperModule = `import {instantiate as innerInstantiate} from '${RUNTIME_TRANSPILED_MODULE_DIRNAME}/${moduleSlug}.js';
262-
263-
const wasiShimImportPromises = {
264-
${wasiShimImports.join(',\n ')}
265-
};
255+
import {shim as wasiShimImportPromises} from '/wasi.js';
266256
267257
const wasiShimImports = Promise.all(
268258
Object.entries(wasiShimImportPromises)
@@ -281,7 +271,8 @@ export const instantiate = async (imports) => {
281271
}
282272
283273
const getCoreModule = async (name) => fetch('${RUNTIME_TRANSPILED_MODULE_DIRNAME}/' + name).then(WebAssembly.compileStreaming);
284-
console.log('Instantiating with:', imports);
274+
console.log('Instantiating module with these resolved imports:', imports);
275+
285276
return innerInstantiate(getCoreModule, imports);
286277
};`;
287278

0 commit comments

Comments
 (0)