feat: Implement basic Module Build Server

Author: cdata

Cargo.lock

@@ -1,19 +1,32 @@
 [workspace]
 members = [
   "rust/example-crate",
+  "rust/usuba"
 ]
 
 # See: https://github.com/rust-lang/rust/issues/90148#issuecomment-949194352
 resolver = "2"
 
 [workspace.dependencies]
+async-trait = { version = "0.1" }
+axum = { version = "0.7" }
+blake3 = { version = "1.5" }
+bytes = { version = "1" }
 js-sys = { version = "0.3" }
+redb = { version = "2" }
+serde = { version = "1" }
+serde_json = { version = "1" }
+tempfile = { version = "3" }
+thiserror = { version = "1" }
+tokio = { version = "1" }
 tracing = { version = "0.1" }
 tracing-subscriber = { version = "0.3", features = ["env-filter", "tracing-log", "json"] }
 tracing-web = { version = "0.1" }
+utoipa = { version = "4" }
+utoipa-swagger-ui = { version = "7" }
 wasm-bindgen = { version = "0.2" }
 web-sys = { version = "0.3" }
-wit-bindgen = { version = "0.24" }
+wit-bindgen = { version = "0.25" }
 
 [profile.release]
 opt-level = 'z'

Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "usuba"
+description = "An anything-to-Common-Wasm build server"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = { workspace = true }
+axum = { workspace = true, features = ["multipart"] }
+blake3 = { workspace = true }
+bytes = { workspace = true }
+redb = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tempfile = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread", "io-util", "process", "fs"] }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+utoipa = { workspace = true, features = ["axum_extras"] }
+utoipa-swagger-ui = { workspace = true, features = ["axum"] }
+# gcloud-sdk = { version = "0.24.6", features = ["google-cloud-aiplatform-v1"] }
+

rust/usuba/Cargo.toml

@@ -0,0 +1,22 @@
+FROM rust:1.75 as builder
+
+WORKDIR /build-root
+
+COPY "./Cargo.toml" "./Cargo.lock" ./
+COPY ./rust ./rust
+RUN cargo build --release --bin usuba
+
+FROM node:latest
+
+WORKDIR /usuba
+
+EXPOSE 8888
+
+RUN apt-get update && apt-get install -y libssl-dev ca-certificates
+RUN npm install -g @bytecodealliance/jco
+
+COPY --from=builder /build-root/target/release/usuba /usr/bin/usuba
+
+ENV RUST_LOG="debug"
+
+ENTRYPOINT ["/usr/bin/usuba"]

rust/usuba/Dockerfile

@@ -0,0 +1,4 @@
+# Usuba
+
+An anything-to-Common-Wasm build server.
+

rust/usuba/README.md

@@ -0,0 +1,8 @@
+use crate::UsubaError;
+use async_trait::async_trait;
+use bytes::Bytes;
+
+#[async_trait]
+pub trait Bake {
+    async fn bake(&self, wit: Bytes, source_code: Bytes) -> Result<Bytes, UsubaError>;
+}

rust/usuba/src/bake/bake.rs

@@ -0,0 +1,68 @@
+use std::io::Cursor;
+use tracing::instrument;
+
+use super::Bake;
+use async_trait::async_trait;
+use bytes::Bytes;
+use tempfile::TempDir;
+
+use tokio::process::Command;
+
+#[derive(Debug)]
+pub struct JavaScriptBaker {}
+
+#[async_trait]
+impl Bake for JavaScriptBaker {
+    #[instrument]
+    async fn bake(&self, wit: Bytes, source_code: Bytes) -> Result<Bytes, crate::UsubaError> {
+        let workspace = TempDir::new()?;
+        debug!(
+            "Created temporary workspace in {}",
+            workspace.path().display()
+        );
+
+        let wasm_path = workspace.path().join("module.wasm");
+        let js_path = workspace.path().join("module.js");
+        let wit_path = workspace.path().join("module.wit");
+
+        let (mut wit_file, mut js_file) = tokio::try_join!(
+            tokio::fs::File::create(&wit_path),
+            tokio::fs::File::create(&js_path),
+        )?;
+
+        debug!(?wit_path, ?js_path, "Created temporary input files");
+
+        let mut wit_cursor = Cursor::new(wit);
+        let mut js_cursor = Cursor::new(source_code);
+
+        tokio::try_join!(
+            tokio::io::copy(&mut wit_cursor, &mut wit_file),
+            tokio::io::copy(&mut js_cursor, &mut js_file),
+        )?;
+
+        debug!(?wit_path, ?js_path, "Populated temporary input files");
+
+        let mut command = Command::new("jco");
+
+        command
+            .arg("componentize")
+            .arg("-w")
+            .arg(wit_path.display().to_string())
+            .arg("-o")
+            .arg(wasm_path.display().to_string())
+            .arg(js_path.display().to_string());
+
+        let child = command.spawn()?;
+        let output = child.wait_with_output().await?;
+
+        if output.stderr.len() > 0 {
+            warn!("{}", String::from_utf8_lossy(&output.stderr));
+        }
+
+        debug!("Finished building with jco");
+
+        let wasm_bytes = tokio::fs::read(&wasm_path).await?;
+
+        Ok(wasm_bytes.into())
+    }
+}

rust/usuba/src/bake/javascript.rs

@@ -0,0 +1,21 @@
+mod bake;
+mod javascript;
+
+pub use bake::*;
+pub use javascript::*;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+
+pub enum Baker {
+    JavaScript,
+}
+
+#[async_trait]
+impl Bake for Baker {
+    async fn bake(&self, wit: Bytes, source_code: Bytes) -> Result<Bytes, crate::UsubaError> {
+        match self {
+            Baker::JavaScript => (JavaScriptBaker {}).bake(wit, source_code).await,
+        }
+    }
+}

rust/usuba/src/bake/mod.rs

@@ -0,0 +1,25 @@
+#[macro_use]
+extern crate tracing;
+
+use std::net::SocketAddr;
+
+use tracing::Level;
+use tracing_subscriber::{fmt::Layer, layer::SubscriberExt, FmtSubscriber};
+use usuba::{serve, UsubaError};
+
+#[tokio::main]
+pub async fn main() -> Result<(), UsubaError> {
+    let subscriber = FmtSubscriber::builder()
+        .with_max_level(Level::TRACE)
+        .finish();
+    tracing::subscriber::set_global_default(subscriber.with(Layer::default().pretty()))?;
+
+    let socket_address: SocketAddr = "127.0.0.1:8080".parse()?;
+    let listener = tokio::net::TcpListener::bind(socket_address).await?;
+
+    info!("Server listening on {}", socket_address);
+
+    serve(listener).await?;
+
+    Ok(())
+}

rust/usuba/src/bin/usuba.rs

@@ -0,0 +1,115 @@
+// use std::fmt::Display;
+
+use axum::{extract::multipart::MultipartError, http::StatusCode, response::IntoResponse, Json};
+use blake3::HexError;
+use redb::{CommitError, DatabaseError, StorageError, TableError, TransactionError};
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use tracing::subscriber::SetGlobalDefaultError;
+use utoipa::ToSchema;
+
+#[derive(Debug, Error)]
+pub enum UsubaError {
+    #[error("Bad request body")]
+    BadRequest,
+    #[error("Failed to bake the module: {0}")]
+    BakeFailure(String),
+    #[error("Invalid configuration: {0}")]
+    InvalidConfiguration(String),
+    #[error("Module not found")]
+    ModuleNotFound,
+    #[error("An internal error occurred")]
+    Internal(String),
+}
+
+impl From<std::net::AddrParseError> for UsubaError {
+    fn from(value: std::net::AddrParseError) -> Self {
+        UsubaError::InvalidConfiguration(format!("{}", value))
+    }
+}
+
+impl From<std::io::Error> for UsubaError {
+    fn from(value: std::io::Error) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<MultipartError> for UsubaError {
+    fn from(_value: MultipartError) -> Self {
+        UsubaError::BadRequest
+    }
+}
+
+impl From<SetGlobalDefaultError> for UsubaError {
+    fn from(value: SetGlobalDefaultError) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<StorageError> for UsubaError {
+    fn from(value: StorageError) -> Self {
+        error!("{}", value);
+        UsubaError::ModuleNotFound
+    }
+}
+
+impl From<TransactionError> for UsubaError {
+    fn from(value: TransactionError) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<TableError> for UsubaError {
+    fn from(value: TableError) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<CommitError> for UsubaError {
+    fn from(value: CommitError) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<DatabaseError> for UsubaError {
+    fn from(value: DatabaseError) -> Self {
+        error!("{}", value);
+        UsubaError::Internal(format!("{}", value))
+    }
+}
+
+impl From<HexError> for UsubaError {
+    fn from(_value: HexError) -> Self {
+        UsubaError::BadRequest
+    }
+}
+
+impl IntoResponse for UsubaError {
+    fn into_response(self) -> axum::response::Response {
+        let status = match self {
+            UsubaError::BadRequest => StatusCode::BAD_REQUEST,
+            UsubaError::BakeFailure(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            UsubaError::InvalidConfiguration(_) => StatusCode::BAD_REQUEST,
+            UsubaError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR,
+            UsubaError::ModuleNotFound => StatusCode::NOT_FOUND,
+        };
+
+        (
+            status,
+            Json(ErrorResponse {
+                error: self.to_string(),
+            }),
+        )
+            .into_response()
+    }
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+pub struct ErrorResponse {
+    error: String,
+}