feat: Python baker and frontend support

.gitignore

@@ -2,4 +2,5 @@ target
 .wireit
 node_modules
 dist
-*.tsbuildinfo
+lib
+*.tsbuildinfo

rust/usuba/src/bake/fs.rs

@@ -0,0 +1,12 @@
+use std::{io::Cursor, path::PathBuf};
+
+use bytes::Bytes;
+
+use crate::UsubaError;
+
+pub async fn write_file(path: PathBuf, bytes: Bytes) -> Result<(), UsubaError> {
+    let mut file = tokio::fs::File::create(&path).await?;
+    let mut cursor = Cursor::new(bytes.as_ref());
+    tokio::io::copy(&mut cursor, &mut file).await?;
+    Ok(())
+}

rust/usuba/src/bake/javascript.rs

@@ -1,9 +1,5 @@
-use std::io::Cursor;
-use std::path::{Path, PathBuf};
 use tracing::instrument;
 
-use crate::UsubaError;
-
 use super::Bake;
 use async_trait::async_trait;
 use bytes::Bytes;
@@ -12,14 +8,7 @@ use tempfile::TempDir;
 use tokio::process::Command;
 use tokio::task::JoinSet;
 
-use wit_parser::UnresolvedPackage;
-
-async fn write_file(path: PathBuf, bytes: Bytes) -> Result<(), UsubaError> {
-    let mut file = tokio::fs::File::create(&path).await?;
-    let mut cursor = Cursor::new(bytes.as_ref());
-    tokio::io::copy(&mut cursor, &mut file).await?;
-    Ok(())
-}
+use crate::write_file;
 
 #[derive(Debug)]
 pub struct JavaScriptBaker {}

rust/usuba/src/bake/mod.rs

@@ -1,14 +1,19 @@
 mod bake;
+mod fs;
 mod javascript;
+mod python;
 
 pub use bake::*;
+pub use fs::*;
 pub use javascript::*;
+pub use python::*;
 
 use async_trait::async_trait;
 use bytes::Bytes;
 
 pub enum Baker {
     JavaScript,
+    Python,
 }
 
 #[async_trait]
@@ -26,6 +31,11 @@ impl Bake for Baker {
                     .bake(world, wit, source_code, library)
                     .await
             }
+            Baker::Python => {
+                (PythonBaker {})
+                    .bake(world, wit, source_code, library)
+                    .await
+            }
         }
     }
 }

rust/usuba/src/bake/python.rs

@@ -0,0 +1,89 @@
+use async_trait::async_trait;
+use bytes::Bytes;
+use tempfile::TempDir;
+use tokio::{process::Command, task::JoinSet};
+
+use crate::{write_file, Bake};
+
+#[derive(Debug)]
+pub struct PythonBaker {}
+
+#[async_trait]
+impl Bake for PythonBaker {
+    #[instrument]
+    async fn bake(
+        &self,
+        world: &str,
+        wit: Vec<Bytes>,
+        source_code: Bytes,
+        library: Vec<Bytes>,
+    ) -> Result<Bytes, crate::UsubaError> {
+        let workspace = TempDir::new()?;
+        debug!(
+            "Created temporary workspace in {}",
+            workspace.path().display()
+        );
+
+        let wasm_path = workspace.path().join("module.wasm");
+        let python_path = workspace.path().join("module.py");
+
+        debug!(?workspace, "Created temporary workspace");
+
+        let wit_path = workspace.path().join("wit");
+        let wit_deps_path = wit_path.join("deps");
+
+        tokio::fs::create_dir_all(&wit_deps_path).await?;
+
+        let mut writes = JoinSet::new();
+
+        wit.into_iter()
+            .enumerate()
+            .map(|(i, wit)| write_file(wit_path.join(format!("module{}.wit", i)), wit))
+            .chain([write_file(python_path.clone(), source_code)])
+            .chain(
+                library.into_iter().enumerate().map(|(i, wit)| {
+                    write_file(wit_deps_path.join(format!("library{}.wit", i)), wit)
+                }),
+            )
+            .for_each(|fut| {
+                writes.spawn(fut);
+            });
+
+        while let Some(result) = writes.try_join_next() {
+            result??;
+            continue;
+        }
+
+        debug!(?workspace, "Populated temporary input files");
+
+        let mut command = Command::new("componentize-py");
+
+        command
+            .current_dir(workspace.path())
+            .arg("-d")
+            .arg(wit_path)
+            .arg("-w")
+            .arg(world)
+            .arg("componentize")
+            .arg("-p")
+            .arg(workspace.path().display().to_string())
+            .arg("-o")
+            .arg("module.wasm")
+            .arg("module");
+
+        let child = command.spawn()?;
+        let output = child.wait_with_output().await?;
+
+        if output.stderr.len() > 0 {
+            warn!("{}", String::from_utf8_lossy(&output.stderr));
+        }
+
+        debug!("Finished building with componentize-py");
+
+        let wasm_bytes = tokio::fs::read(&wasm_path).await?;
+
+        info!("Finished baking");
+
+        Ok(wasm_bytes.into())
+    }
+}

rust/usuba/src/routes/module/build.rs

@@ -85,6 +85,10 @@ pub async fn build_module(
                                 source_code = Some(field.bytes().await?);
                                 baker = Some(Baker::JavaScript);
                             }
+                            Some("py") => {
+                                source_code = Some(field.bytes().await?);
+                                baker = Some(Baker::Python);
+                            }
                             _ => (),
                         };
                     }
@@ -106,6 +110,7 @@ pub async fn build_module(
             id: hash.to_string(),
         })
     } else {
+        warn!("Insufficient payload inputs to build the module");
         Err(UsubaError::BadRequest)
     }
 }

typescript/package.json

@@ -42,4 +42,4 @@
       ]
     }
   }
-}
+}

typescript/packages/runtime-demo/src/index.ts

@@ -146,5 +146,65 @@ export const demoTwo = async () => {
   console.log('fin');
 };
 
+/**
+ * Demo Three
+ */
+const EXAMPLE_HELLO_PY = `
+import random
+import hello
+from hello.imports import lookup
+
+class Hello(hello.Hello):
+    def hello(self) -> str:
+        return "Hello, Agent %s!" % lookup.entry(random.randint(0, 9))
+`;
+
+export const demoThree = async () => {
+  console.log('Initializing first Runtime');
+
+  const rtOne = new Runtime([]);
+
+  console.log('Defining first Module');
+
+  type ExpectedExportsOne = {
+    lookup: {
+      entry(index: number): string;
+    };
+  };
+
+  const moduleOne = await rtOne.defineModule<ExpectedExportsOne>({
+    contentType: 'text/javascript',
+    wit: COMMON_DIRECTORY_WIT,
+    sourceCode: COMMON_DIRECTORY_JS,
+  });
+
+  const rtTwo = new Runtime([COMMON_DIRECTORY_WIT]);
+
+  console.log('Defining second Module');
+
+  type ExpectedExportsTwo = {
+    hello: () => string;
+  };
+
+  const moduleTwo = await rtTwo.defineModule<ExpectedExportsTwo>({
+    contentType: 'text/x-python',
+    wit: EXAMPLE_HELLO_WIT,
+    sourceCode: EXAMPLE_HELLO_PY,
+  });
+
+  console.log('Instantiating both Modules');
+
+  const { hello } = await moduleTwo.instantiate({
+    'common:directory/lookup': (await moduleOne.instantiate({})).lookup,
+  });
+
+  console.log('Invoking final Module API:');
+
+  console.log(`%c${hello()}`, 'font-size: 1.5em; font-weight: bold;');
+
+  console.log('fin');
+};
+
 (self as any).demoOne = demoOne;
 (self as any).demoTwo = demoTwo;
+(self as any).demoThree = demoThree;

typescript/packages/usuba-rt/src/index.ts

@@ -3,7 +3,7 @@ import * as apiClient from '@commontools/usuba-api';
 export type SourceCode = string | Uint8Array;
 export type PendingSourceCode = SourceCode | Promise<SourceCode>;
 
-export type ContentType = 'text/javascript';
+export type ContentType = 'text/javascript' | 'text/x-python';
 
 export type ContentTypeFileExtensions = {
   [C in ContentType]: string;
@@ -31,6 +31,7 @@ export type ImportableMap = {
 
 const FILE_EXTENSIONS: ContentTypeFileExtensions = {
   'text/javascript': 'js',
+  'text/x-python': 'py',
 };
 
 /**

typescript/packages/usuba-sw/src/index.ts

@@ -1,11 +1,11 @@
 import * as apiClient from '@commontools/usuba-api';
 import { polyfill, hash } from './usuba_compat/usuba_compat.component.js';
 
-const SERVICE_WORKER_VERSION = '0.0.1';
+const SERVICE_WORKER_VERSION = '0.0.1-alpha.6';
 
 self.addEventListener('install', (_event) => {
   console.log(
-    `Usuba Service Worker installed (version ${SERVICE_WORKER_VERSION}`
+    `Usuba Service Worker installed (version ${SERVICE_WORKER_VERSION})`
   );
 });
 
@@ -216,6 +216,7 @@ const buildOnDemandModule = (event: FetchEvent, url: URL) => {
 const buildRuntimeModule = (event: FetchEvent, url: URL) => {
   event.respondWith(
     (async () => {
+      console.log('Preparing to build Runtime Module...');
       const formData = await event.request.formData();
       const moduleFiles = formData.getAll('module') as File[];
       const libraryFiles = formData.getAll('library') as File[];
@@ -250,19 +251,8 @@ const buildRuntimeModule = (event: FetchEvent, url: URL) => {
         exports: _exports,
       } = await buildModule(moduleSlug, moduleFiles, libraryFiles, 'manual');
 
-      const wasiShimImports = [];
-
-      for (const specifier of Object.values(WASI_SHIM_MAP)) {
-        const trimmedSpecifier = specifier.split('#').shift();
-        wasiShimImports.push(`'${trimmedSpecifier}': import('${specifier}')`);
-      }
-
-      // const wrapperModule = `export * from '${ON_DEMAND_TRANSPILED_MODULE_DIRNAME}/${moduleSlug}.js'`;
       const wrapperModule = `import {instantiate as innerInstantiate} from '${RUNTIME_TRANSPILED_MODULE_DIRNAME}/${moduleSlug}.js';
-
-const wasiShimImportPromises = {
-  ${wasiShimImports.join(',\n  ')}
-};
+import {shim as wasiShimImportPromises} from '/wasi.js';
 
 const wasiShimImports = Promise.all(
   Object.entries(wasiShimImportPromises)
@@ -281,7 +271,8 @@ export const instantiate = async (imports) => {
   }
 
   const getCoreModule = async (name) => fetch('${RUNTIME_TRANSPILED_MODULE_DIRNAME}/' + name).then(WebAssembly.compileStreaming);
-  console.log('Instantiating with:', imports);
+  console.log('Instantiating module with these resolved imports:', imports);
+
   return innerInstantiate(getCoreModule, imports);
 };`;
 

typescript/packages/usuba-ui/public/wasi.js

@@ -0,0 +1,17 @@
+import * as cli from '/wasi-shim/cli.js';
+import * as clocks from '/wasi-shim/clocks.js';
+import * as filesystem from '/wasi-shim/filesystem.js';
+import * as http from '/wasi-shim/http.js';
+import * as io from '/wasi-shim/io.js';
+import * as random from '/wasi-shim/random.js';
+import * as sockets from '/wasi-shim/sockets.js';
+
+export const shim = {
+  '/wasi-shim/cli.js': Promise.resolve(cli),
+  '/wasi-shim/clocks.js': Promise.resolve(clocks),
+  '/wasi-shim/filesystem.js': Promise.resolve(filesystem),
+  '/wasi-shim/http.js': Promise.resolve(http),
+  '/wasi-shim/io.js': Promise.resolve(io),
+  '/wasi-shim/random.js': Promise.resolve(random),
+  '/wasi-shim/sockets.js': Promise.resolve(sockets),
+};