diff --git a/docs/usuba.md b/docs/usuba.md new file mode 100644 index 000000000..dc9a0ca63 --- /dev/null +++ b/docs/usuba.md @@ -0,0 +1,41 @@ +## Local Usuba Setup + +Install some environment dependencies: + +```sh +cargo install wit-deps-cli wasm-tools +rustup target add wasm32-wasi wasm32-unknown-unknown + +npm install -g @bytecodealliance/jco +npm install -g @bytecodealliance/componentize-js + +pip install componentize-py +``` + +On a Mac, also do this also: + +```sh +brew install wget gsed +``` + +Set up an NPM project (using whatever stack) in `/typescript/packages` and add `@commontools/runtime` to your dependencies: + +```sh +mkdir -p ./typescript/packages/my-project +cd ./typescript/packages/my-project + +# Actually, better to copy a package.json from another package +echo '{"type": "module"}' > ./package.json +npm install --save @commontools/runtime +``` + +Get a web server going and note the `PORT` it is running on. + +In another terminal, get `usuba` running: + +```sh +cd ./rust/usuba +UPSTREAM=localhost:$PORT cargo run --bin usuba +``` + +Open your browser to http://localhost:8080. You should see whatever you would expect to see from the first web server you started. diff --git a/rust/usuba/src/bake/javascript.rs b/rust/usuba/src/bake/javascript.rs index 70decac21..01d70c70c 100644 --- a/rust/usuba/src/bake/javascript.rs +++ b/rust/usuba/src/bake/javascript.rs @@ -61,6 +61,14 @@ impl Bake for JavaScriptBaker { debug!(?workspace, "Populated temporary input files"); + Command::new("cp") + .arg("-r") + .arg(format!("{}", workspace.path().display())) + .arg("/tmp/failed") + .spawn()? + .wait() + .await?; + let mut command = Command::new("jco"); command diff --git a/typescript/common/.gitignore b/typescript/common/.gitignore new file mode 100644 index 000000000..1aa628036 --- /dev/null +++ b/typescript/common/.gitignore @@ -0,0 +1 @@ +deps \ No newline at end of file diff --git a/typescript/common/data/.gitignore b/typescript/common/data/.gitignore new file mode 100644 index 000000000..e8310385c --- /dev/null +++ b/typescript/common/data/.gitignore @@ -0,0 +1 @@ +src \ No newline at end of file diff --git a/typescript/common/data/package.json b/typescript/common/data/package.json new file mode 100644 index 000000000..2964cf027 --- /dev/null +++ b/typescript/common/data/package.json @@ -0,0 +1,119 @@ +{ + "name": "@commontools/data", + "author": "The Common Authors", + "version": "0.0.1", + "description": "", + "license": "UNLICENSED", + "private": true, + "type": "module", + "files": [ + "./lib/**/*" + ], + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./interfaces/common-data-types.js": { + "types": "./lib/interfaces/common-data-types.js" + } + }, + "scripts": { + "build": "wireit", + "clean": "wireit", + "update:wit": "wireit", + "lint:wit": "wireit", + "build:wit": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "dependencies": {}, + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "install:wit": { + "command": "wit-deps", + "files": [ + "./wit/deps.*" + ], + "output": [ + "./wit/deps/**/*" + ] + }, + "lint:wit": { + "dependencies": [ + "install:wit" + ], + "command": "wasm-tools component wit ./wit", + "files": [ + "./wit/**/*" + ] + }, + "update:wit": { + "command": "wit-deps update", + "files": [ + "./wit/deps.toml" + ], + "output": [ + "./wit/deps.lock", + "./wit/deps/**/*" + ] + }, + "build:jco": { + "dependencies": [ + "install:wit", + "lint:wit" + ], + "command": "jco types -o ./lib --name index ./wit", + "files": [ + "./wit/**/*" + ], + "output": [ + "./lib/**/*" + ] + }, + "build:wit:index": { + "dependencies": [ + "build:jco" + ], + "command": "cat ./wit/*.wit | ./scripts/generate-wit-module.sh > ./lib/index.js", + "files": [ + "./scripts/generate-wit-module.sh", + "./wit/*.wit" + ] + }, + "build:wit:types": { + "dependencies": [ + "build:jco" + ], + "command": "echo 'export const wit: string;' >> ./lib/index.d.ts", + "files": [ + "./lib/index.d.ts" + ] + }, + "build:wit": { + "dependencies": [ + "build:wit:index", + "build:wit:types" + ] + }, + "build": { + "dependencies": [ + "build:jco", + "build:wit" + ] + }, + "clean": { + "command": "rm -rf ./src ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/common/data/scripts b/typescript/common/data/scripts new file mode 120000 index 000000000..e62fc5c53 --- /dev/null +++ b/typescript/common/data/scripts @@ -0,0 +1 @@ +../module/scripts \ No newline at end of file diff --git a/typescript/common/data/tsconfig.json b/typescript/common/data/tsconfig.json new file mode 100644 index 000000000..d40c067b9 --- /dev/null +++ b/typescript/common/data/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["es2022", "DOM", "DOM.Iterable"], + "outDir": "./lib", + "rootDir": "./src", + }, + "include": [ + "src/**/*", + ] +} diff --git a/typescript/common/data/wit/data.wit b/typescript/common/data/wit/data.wit new file mode 100644 index 000000000..9ea0cd565 --- /dev/null +++ b/typescript/common/data/wit/data.wit @@ -0,0 +1,33 @@ +package common:data@0.0.1; + +interface types { + resource reference { + // Dereference a reference to a value + // This call is fallible (for example, if the dereference is not allowed) + // The value may be none (for example, if it is strictly opaque) + deref: func() -> result, %string>; + } + + resource dictionary { + get: func(key: %string) -> option; + } + + type array = list; + type %string = string; + type boolean = bool; + type number = f64; + type buffer = list; + + variant value { + %string(%string), + number(number), + boolean(boolean), + buffer(buffer), + array(array), + dictionary(dictionary) + } +} + +world common { + export types; +} \ No newline at end of file diff --git a/typescript/common/data/wit/deps.lock b/typescript/common/data/wit/deps.lock new file mode 100644 index 000000000..e69de29bb diff --git a/typescript/common/data/wit/deps.toml b/typescript/common/data/wit/deps.toml new file mode 100644 index 000000000..e69de29bb diff --git a/typescript/common/io/package.json b/typescript/common/io/package.json new file mode 100644 index 000000000..34e7db861 --- /dev/null +++ b/typescript/common/io/package.json @@ -0,0 +1,112 @@ +{ + "name": "@commontools/io", + "author": "The Common Authors", + "version": "0.0.1", + "description": "", + "license": "UNLICENSED", + "private": true, + "type": "module", + "types": "./lib/module.d.ts", + "files": [ + "./lib/**/*" + ], + "exports": "./lib/index.js", + "scripts": { + "build": "wireit", + "clean": "wireit", + "update:wit": "wireit", + "lint:wit": "wireit", + "build:wit:inline": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "dependencies": {}, + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "install:wit": { + "command": "wit-deps", + "files": [ + "./wit/deps.*" + ], + "output": [ + "./wit/deps/**/*" + ] + }, + "lint:wit": { + "dependencies": [ + "install:wit" + ], + "command": "wasm-tools component wit ./wit", + "files": [ + "./wit/**/*" + ] + }, + "update:wit": { + "command": "wit-deps update", + "files": [ + "./wit/deps.toml" + ], + "output": [ + "./wit/deps.lock", + "./wit/deps/**/*" + ] + }, + "build:jco": { + "dependencies": [ + "install:wit", + "lint:wit" + ], + "command": "jco types -o ./lib --name index ./wit", + "files": [ + "./wit/**/*" + ], + "output": [ + "./lib/**/*" + ] + }, + "build:wit:index": { + "dependencies": [ + "build:jco" + ], + "command": "cat ./wit/*.wit | ./scripts/generate-wit-module.sh > ./lib/index.js", + "files": [ + "./scripts/generate-wit-module.sh", + "./wit/*.wit" + ] + }, + "build:wit:types": { + "dependencies": [ + "build:jco" + ], + "command": "echo 'export const wit: string;' >> ./lib/index.d.ts", + "files": [ + "./lib/index.d.ts" + ] + }, + "build:wit": { + "dependencies": [ + "build:wit:index", + "build:wit:types" + ] + }, + "build": { + "dependencies": [ + "build:jco", + "build:wit" + ] + }, + "clean": { + "command": "rm -rf ./src ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/common/io/scripts b/typescript/common/io/scripts new file mode 120000 index 000000000..e62fc5c53 --- /dev/null +++ b/typescript/common/io/scripts @@ -0,0 +1 @@ +../module/scripts \ No newline at end of file diff --git a/typescript/common/io/wit.patch.d.ts b/typescript/common/io/wit.patch.d.ts new file mode 100644 index 000000000..4a0d81380 --- /dev/null +++ b/typescript/common/io/wit.patch.d.ts @@ -0,0 +1,6 @@ +declare module 'common:io/state@0.0.1' { + export function read(name: String): Reference | undefined; + export function write(name: String, value: Value): void; + export { Stream }; + export function subscribe(name: String): Stream | undefined; +} diff --git a/typescript/common/io/wit/deps.lock b/typescript/common/io/wit/deps.lock new file mode 100644 index 000000000..a7387021f --- /dev/null +++ b/typescript/common/io/wit/deps.lock @@ -0,0 +1,4 @@ +[data] +path = "../../data/wit" +sha256 = "e092c9d7a74aaab5a9a99dcf43ae00f3bcf70d78cb5841124dae526698888815" +sha512 = "e55af0160d09492d61567710cf0fa0c0ae032931cc9f0cc4b44a7e1f4706ca879034790d6c9f393be31711530a4751222767708c16701cf076d2053eec27d212" diff --git a/typescript/common/io/wit/deps.toml b/typescript/common/io/wit/deps.toml new file mode 100644 index 000000000..509840b22 --- /dev/null +++ b/typescript/common/io/wit/deps.toml @@ -0,0 +1 @@ +data = "../../data/wit" \ No newline at end of file diff --git a/typescript/common/io/wit/io.wit b/typescript/common/io/wit/io.wit new file mode 100644 index 000000000..d1d3b70f3 --- /dev/null +++ b/typescript/common/io/wit/io.wit @@ -0,0 +1,12 @@ +package common:io@0.0.1; + +interface state { + use common:data/types@0.0.1.{reference,value,%string}; + + read: func(name: %string) -> option; + write: func(name: %string, value: value); +} + +world common { + export state; +} \ No newline at end of file diff --git a/typescript/common/module/package.json b/typescript/common/module/package.json new file mode 100644 index 000000000..2b7dc4d83 --- /dev/null +++ b/typescript/common/module/package.json @@ -0,0 +1,112 @@ +{ + "name": "@commontools/module", + "author": "The Common Authors", + "version": "0.0.1", + "description": "", + "license": "UNLICENSED", + "private": true, + "type": "module", + "types": "./lib/module.d.ts", + "files": [ + "./lib/**/*" + ], + "exports": "./lib/index.js", + "scripts": { + "build": "wireit", + "clean": "wireit", + "update:wit": "wireit", + "lint:wit": "wireit", + "build:wit:inline": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "dependencies": {}, + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "install:wit": { + "command": "wit-deps", + "files": [ + "./wit/deps.*" + ], + "output": [ + "./wit/deps/**/*" + ] + }, + "lint:wit": { + "dependencies": [ + "install:wit" + ], + "command": "wasm-tools component wit ./wit", + "files": [ + "./wit/**/*" + ] + }, + "update:wit": { + "command": "wit-deps update", + "files": [ + "./wit/deps.toml" + ], + "output": [ + "./wit/deps.lock", + "./wit/deps/**/*" + ] + }, + "build:jco": { + "dependencies": [ + "install:wit", + "lint:wit" + ], + "command": "jco types -o ./lib --name index ./wit", + "files": [ + "./wit/**/*" + ], + "output": [ + "./lib/**/*" + ] + }, + "build:wit:index": { + "dependencies": [ + "build:jco" + ], + "command": "cat ./wit/*.wit | ./scripts/generate-wit-module.sh > ./lib/index.js", + "files": [ + "./scripts/generate-wit-module.sh", + "./wit/*.wit" + ] + }, + "build:wit:types": { + "dependencies": [ + "build:jco" + ], + "command": "echo 'export const wit: string;' >> ./lib/index.d.ts", + "files": [ + "./lib/index.d.ts" + ] + }, + "build:wit": { + "dependencies": [ + "build:wit:index", + "build:wit:types" + ] + }, + "build": { + "dependencies": [ + "build:jco", + "build:wit" + ] + }, + "clean": { + "command": "rm -rf ./src ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/common/module/scripts/generate-wit-module.sh b/typescript/common/module/scripts/generate-wit-module.sh new file mode 100755 index 000000000..6a62e4391 --- /dev/null +++ b/typescript/common/module/scripts/generate-wit-module.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +cd $SCRIPT_DIR + +BODY="$( body; +} + +world common { + import common:data/types@0.0.1; + import common:io/state@0.0.1; + + export module; +} \ No newline at end of file diff --git a/typescript/common/runtime/package.json b/typescript/common/runtime/package.json new file mode 100644 index 000000000..d50735032 --- /dev/null +++ b/typescript/common/runtime/package.json @@ -0,0 +1,68 @@ +{ + "name": "@commontools/runtime", + "author": "The Common Authors", + "version": "0.0.1", + "description": "A Module Runtime with Common Tools APIs baked in", + "license": "UNLICENSED", + "private": true, + "type": "module", + "scripts": { + "build": "wireit", + "clean": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + } + }, + "files": [ + "./lib/*.js" + ], + "dependencies": { + "@commontools/data": "^0.0.1", + "@commontools/io": "^0.0.1", + "@commontools/module": "^0.0.1", + "@commontools/usuba-rt": "^0.0.1", + "ses": "^1.5.0" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "build": { + "dependencies": [ + "../../packages/usuba-rt:build", + "../data:build", + "../io:build", + "../module:build" + ], + "files": [ + "./src/**/*" + ], + "output": [ + "./lib/**/*" + ], + "command": "tsc --build -f" + }, + "clean": { + "dependencies": [ + "../../packages/usuba-rt:clean", + "../data:clean", + "../io:clean", + "../module:clean" + ], + "command": "rm -rf ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/common/runtime/src/common/data/dictionary.ts b/typescript/common/runtime/src/common/data/dictionary.ts new file mode 100644 index 000000000..85ade70a7 --- /dev/null +++ b/typescript/common/runtime/src/common/data/dictionary.ts @@ -0,0 +1,28 @@ +import type { + Dictionary as CommonDictionary, + Value, +} from '@commontools/data/interfaces/common-data-types.js'; +import { IO as CommonIO } from '../../state/io/index.js'; +import { infer } from './infer.js'; +import { Reference } from './reference.js'; + +export class Dictionary implements CommonDictionary, CommonIO { + #inner: any; + + constructor(inner: any) { + this.#inner = inner; + } + + read(key: string): Value | undefined { + console.log(`Reading '${key}' from`, this.#inner); + return infer(this.#inner[key]); + } + + write(_key: string, _value: Value): void { + throw new Error('Write into dictionary is not supported'); + } + + get(key: string): Reference | undefined { + return new Reference(this, key); + } +} diff --git a/typescript/common/runtime/src/common/data/infer.ts b/typescript/common/runtime/src/common/data/infer.ts new file mode 100644 index 000000000..bdd9a036f --- /dev/null +++ b/typescript/common/runtime/src/common/data/infer.ts @@ -0,0 +1,50 @@ +import type { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { Dictionary } from './dictionary.js'; + +/** + * Attempts to infer a `Value` from any given JS value. See data.wit for a + * reference of all possible variants of `Value`. Inference has marginal cost; + * it is more efficient to avoid it if you can. + * + * NOTE: Currently "array" `Value` types aren't really going to work. + * Stick to "string", "number", "boolean", "buffer" and "dictionary". + */ +export const infer = (value: any): Value | undefined => { + const val = value; + let tag: string = typeof value; + + switch (tag) { + case 'string': + case 'boolean': + case 'number': + return { + tag, + val, + }; + case 'object': { + if (Array.isArray(val)) { + return { + tag: 'array', + val, + }; + } + + if (val instanceof Uint8Array) { + return { + tag: 'buffer', + val, + }; + } + + // NOTE: `typeof null == 'object'` + if (val) { + return { + tag: 'dictionary', + val: new Dictionary(val), + }; + } + } + } + + return undefined; +}; diff --git a/typescript/common/runtime/src/common/data/reference.ts b/typescript/common/runtime/src/common/data/reference.ts new file mode 100644 index 000000000..c7ca375fe --- /dev/null +++ b/typescript/common/runtime/src/common/data/reference.ts @@ -0,0 +1,20 @@ +import type { + Value, + Reference as CommonReference, +} from '@commontools/data/interfaces/common-data-types.js'; +import { IO } from '../../state/io/index.js'; + +export class Reference implements CommonReference { + #io; + #key; + + constructor(io: IO, key: string) { + // TODO: Validate attempt to read (aka attempt to create a Reference) here + this.#io = io; + this.#key = key; + } + + deref(): Value | undefined { + return this.#io.read(this.#key); + } +} diff --git a/typescript/common/runtime/src/helpers.ts b/typescript/common/runtime/src/helpers.ts new file mode 100644 index 000000000..26e3368de --- /dev/null +++ b/typescript/common/runtime/src/helpers.ts @@ -0,0 +1,38 @@ +export type ElementUnion = T[number]; + +export type EventMap = { + [E in keyof Events]: never; +}; + +export const assertNever = (value: never): never => { + throw new Error(`Unhandled value: ${value}`); +}; + +export const downcast = (value: unknown): T => { + return value as T; +}; + +export const isEvent = ( + event: AllEvents, + check: Event +): event is Event => { + return event === check; +}; + +export const isError = (candidate: unknown): candidate is { error: string } => { + return ( + candidate != null && + typeof candidate == 'object' && + 'error' in candidate && + typeof candidate.error != 'undefined' + ); +}; + +export const throwIfError = ( + candidate: unknown +): candidate is { error: string } => { + if (isError(candidate)) { + throw new Error(candidate.error); + } + return false; +}; diff --git a/typescript/common/runtime/src/index.ts b/typescript/common/runtime/src/index.ts new file mode 100644 index 000000000..c0679c5fd --- /dev/null +++ b/typescript/common/runtime/src/index.ts @@ -0,0 +1,95 @@ +import { WorkerPool } from './worker/pool.js'; +import { Input } from './state/input.js'; +import { HostToModuleRPC } from './rpc/index.js'; +import { Output } from './state/output.js'; +import { assertNever, throwIfError } from './helpers.js'; +import { HostModuleEventHandler } from './rpc/host.js'; + +export type { Value } from '@commontools/data/interfaces/common-data-types.js'; + +export * from './state/io/index.js'; +export * from './state/input.js'; +export * from './state/output.js'; +export * from './state/storage/index.js'; +export * from './state/storage/localstorage.js'; +export * from './common/data/dictionary.js'; +export * from './common/data/reference.js'; +export * from './common/data/infer.js'; + +export type ContentType = 'text/javascript'; + +export const SES_SANDBOX = 'ses'; +export const WASM_SANDBOX = 'wasm'; +export const CONFIDENTIAL_COMPUTE_SANDBOX = 'confidential-compute'; + +export type Sandbox = + | typeof SES_SANDBOX + | typeof WASM_SANDBOX + | typeof CONFIDENTIAL_COMPUTE_SANDBOX; + +export class Module { + #rpc; + #input; + + constructor(port: MessagePort, input: Input) { + this.#rpc = new HostToModuleRPC(port, this.#handleModuleRPCEvent); + this.#input = input; + } + + async run(): Promise { + throwIfError(await this.#rpc.send('module:run', undefined)); + } + + output(keys: string[]) { + return new Output(this.#rpc, keys); + } + + #handleModuleRPCEvent = (async (event, detail) => { + switch (event) { + case 'host:storage:read': + try { + return { + value: await this.#input.read(detail.key), + }; + } catch (error) { + return { + error: `${error}`, + }; + } + } + + return assertNever(event as never); + }) as HostModuleEventHandler; +} + +export class Runtime { + #workerPool = new WorkerPool(); + + async eval( + id: string, + sandbox: Sandbox, + contentType: 'text/javascript', + sourceCode: string, + input: Input + ): Promise { + const { rpc: runtimeRpc } = await this.#workerPool.get(sandbox); + const { port1: moduleTx, port2: moduleRx } = new MessageChannel(); + const module = new Module(moduleTx, input); + + throwIfError( + await runtimeRpc.send( + 'runtime:eval', + { + id, + contentType, + sourceCode, + inputKeys: input.keys, + port: moduleRx, + }, + [moduleRx] + ) + ); + + return module; + } +} diff --git a/typescript/common/runtime/src/rpc/host.ts b/typescript/common/runtime/src/rpc/host.ts new file mode 100644 index 000000000..b61f15b25 --- /dev/null +++ b/typescript/common/runtime/src/rpc/host.ts @@ -0,0 +1,57 @@ +// import { Value } from '@commontools/data/interfaces/common-data-types.js'; + +import { ElementUnion, EventMap } from '../helpers.js'; +import { Value } from '../index.js'; +import { RPCEventHandler } from './index.js'; + +/** + * Messages sent to the host runtime from a worker + */ + +export const HOST_WORKER_EVENTS = ['rpc:handshake:confirmed'] as const; + +export type HostWorkerEvents = ElementUnion; + +export type HostWorkerRequests = EventMap & { + 'rpc:handshake:confirmed': void; +}; + +export type HostWorkerResponses = EventMap & { + 'rpc:handshake:confirmed': void; +}; + +export type HostWorkerEventHandler = RPCEventHandler< + HostWorkerEvents, + HostWorkerRequests, + HostWorkerResponses +>; + +/** + * Messages sent to the host runtime from a module-within-a-worker + */ + +export const HOST_MODULE_EVENTS = ['host:storage:read'] as const; + +export type HostModuleEvents = ElementUnion; + +export type HostModuleRequests = EventMap & { + 'host:storage:read': { + key: string; + }; +}; + +export type HostModuleResponses = EventMap & { + 'host:storage:read': + | { + error: string; + } + | { + value: Value; + }; +}; + +export type HostModuleEventHandler = RPCEventHandler< + HostModuleEvents, + HostModuleRequests, + HostModuleResponses +>; diff --git a/typescript/common/runtime/src/rpc/index.ts b/typescript/common/runtime/src/rpc/index.ts new file mode 100644 index 000000000..06eedb6f8 --- /dev/null +++ b/typescript/common/runtime/src/rpc/index.ts @@ -0,0 +1,167 @@ +import { RuntimeEvents, RuntimeRequests, RuntimeResponses } from './runtime.js'; +import { + HostModuleEvents, + HostModuleRequests, + HostModuleResponses, + HostWorkerEvents, + HostWorkerRequests, + HostWorkerResponses, +} from './host.js'; +import { ModuleEvents, ModuleRequests, ModuleResponses } from './module.js'; +import { EventMap } from '../helpers.js'; + +export const HANDSHAKE_EVENT = 'rpc:handshake'; + +export type RPCEvent = { + event: string; + detail: any; +}; + +// NOTE: This asserts the top-level shape but not the substructure +export const isRPCEvent = (candidate: unknown): candidate is RPCEvent => + candidate != null && + typeof candidate == 'object' && + 'event' in candidate && + typeof candidate.event == 'string'; + +export type RPCEventHandler< + Events extends string, + Requests extends EventMap, + Responses extends EventMap +> = < + E extends Events & keyof Requests & keyof Responses, + Req extends Requests[E] = Requests[E], + Res extends Responses[E] = Responses[E] +>( + event: E, + detail: Req +) => Promise; + +export abstract class RPCClient< + RxEvents extends string, + RxRequests extends EventMap, + RxResponses extends EventMap, + TxEvents extends string, + TxRequests extends EventMap, + TxResponses extends EventMap +> { + #port; + + #handler; + + constructor( + port: MessagePort, + handler: RPCEventHandler + ) { + this.#port = port; + this.#handler = handler; + this.#port.addEventListener('message', this.#onPortMessage); + this.#port.start(); + } + + async send< + E extends TxEvents & + keyof EventMap & + keyof EventMap, + D = TxRequests[E], + R = TxResponses[E] + >(event: E, detail: D, transfer: Transferable[] = []): Promise { + console.log( + `[RPC] [sender] Sending ${event}:`, + detail || '(empty request)' + ); + const { port1: tx, port2: rx } = new MessageChannel(); + const response = new Promise((resolve) => { + const handler = (responseEvent: MessageEvent) => { + rx.close(); + rx.removeEventListener('message', handler); + + console.log( + `[RPC] [sender] Received ${event} response:`, + responseEvent.data || '(empty response)' + ); + + resolve(responseEvent.data as R); + }; + rx.addEventListener('message', handler); + rx.start(); + }); + this.#port.postMessage( + { + event, + detail, + tx, + }, + ([tx] as Transferable[]).concat(transfer) + ); + return response; + } + + #onPortMessage = async (event: MessageEvent) => { + const { + data, + ports: [port, ..._], + } = event; + if (isRPCEvent(data)) { + console.log( + `[RPC] [receiver] Received ${data.event}:`, + data.detail || '(empty request)' + ); + const response = await this.#handler(data.event as any, data.detail); + if (port) { + console.log( + `[RPC] [receiver] Sending ${data.event} response:`, + response || '(empty response)' + ); + port.postMessage(response); + } + } + }; +} + +export class HostToRuntimeRPC extends RPCClient< + HostWorkerEvents, + HostWorkerRequests, + HostWorkerResponses, + RuntimeEvents, + RuntimeRequests, + RuntimeResponses +> {} + +export class RuntimeToHostRPC extends RPCClient< + RuntimeEvents, + RuntimeRequests, + RuntimeResponses, + HostWorkerEvents, + HostWorkerRequests, + HostWorkerResponses +> {} + +export class HostToModuleRPC extends RPCClient< + HostModuleEvents, + HostModuleRequests, + HostModuleResponses, + ModuleEvents, + ModuleRequests, + ModuleResponses +> {} + +export class ModuleToHostRPC extends RPCClient< + ModuleEvents, + ModuleRequests, + ModuleResponses, + HostModuleEvents, + HostModuleRequests, + HostModuleResponses +> {} + +export type NoEvents = ''; +export type NoRequests = EventMap; +export type NoResponses = EventMap; + +// @ts-ignore +export const NO_RPC_EVENT_HANDLER = (async () => {}) as RPCEventHandler< + '', + '', + '' +>; diff --git a/typescript/common/runtime/src/rpc/module.ts b/typescript/common/runtime/src/rpc/module.ts new file mode 100644 index 000000000..306ded1ff --- /dev/null +++ b/typescript/common/runtime/src/rpc/module.ts @@ -0,0 +1,34 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { ElementUnion, EventMap } from '../helpers.js'; +import { RPCEventHandler } from './index.js'; + +/** + * Messages sent to a module-within-a-worker from the host runtime + */ + +export const MODULE_EVENTS = ['module:run', 'module:output:read'] as const; +export type ModuleEvents = ElementUnion; + +export type ModuleRequests = EventMap & { + 'module:run': void; + 'module:output:read': { + key: string; + }; +}; + +export type ModuleResponses = EventMap & { + 'module:run': + | {} + | { + error: string; + }; + 'module:output:read': { + value: Value | undefined; + }; +}; + +export type ModuleEventHandler = RPCEventHandler< + ModuleEvents, + ModuleRequests, + ModuleResponses +>; diff --git a/typescript/common/runtime/src/rpc/runtime.ts b/typescript/common/runtime/src/rpc/runtime.ts new file mode 100644 index 000000000..313f97c19 --- /dev/null +++ b/typescript/common/runtime/src/rpc/runtime.ts @@ -0,0 +1,35 @@ +import { ContentType } from '../index.js'; +import { ElementUnion, EventMap } from '../helpers.js'; +import { RPCEventHandler } from './index.js'; + +/** + * Events sent to a runtime-within-a-worker from the host runtime + */ + +export const RUNTIME_EVENTS = ['runtime:eval'] as const; + +export type RuntimeEvents = ElementUnion; + +export type RuntimeRequests = EventMap & { + 'runtime:eval': { + id: string; + contentType: ContentType; + sourceCode: string; + inputKeys: string[]; + port: MessagePort; + }; +}; + +export type RuntimeResponses = EventMap & { + 'runtime:eval': + | {} + | { + error: string; + }; +}; + +export type RuntimeEventHandler = RPCEventHandler< + RuntimeEvents, + RuntimeRequests, + RuntimeResponses +>; diff --git a/typescript/common/runtime/src/state/input.ts b/typescript/common/runtime/src/state/input.ts new file mode 100644 index 000000000..10ad4cca8 --- /dev/null +++ b/typescript/common/runtime/src/state/input.ts @@ -0,0 +1,33 @@ +import { Value } from '../index.js'; +import { StateSlice } from './io/slice.js'; +import { Storage } from './storage/index.js'; + +export class Input implements Storage { + #storage; + #keys; + + constructor(storage: Storage, keys: string[]) { + this.#storage = storage; + this.#keys = keys; + } + + read(key: string): Promise { + if (this.#keys.includes(key)) { + return this.#storage.read(key); + } else { + throw new Error(`Read of key '${key}' not allowed`); + } + } + + write(_key: string, _value: Value): Promise { + throw new Error('Method not implemented.'); + } + + get keys() { + return this.#keys; + } + + async toStateSlice(): Promise { + return StateSlice.fromStorage(this.#storage, this.#keys); + } +} diff --git a/typescript/common/runtime/src/state/io/duplex.ts b/typescript/common/runtime/src/state/io/duplex.ts new file mode 100644 index 000000000..fec924914 --- /dev/null +++ b/typescript/common/runtime/src/state/io/duplex.ts @@ -0,0 +1,28 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { IO } from './index.js'; + +export class DuplexState implements IO { + #i; + #o; + + get input() { + return this.#i; + } + + get output() { + return this.#o; + } + + constructor(i: IO, o: IO) { + this.#i = i; + this.#o = o; + } + + read(key: string): Value | undefined { + return this.#i.read(key); + } + + write(key: string, value: Value): void { + return this.#o.write(key, value); + } +} diff --git a/typescript/common/runtime/src/state/io/index.ts b/typescript/common/runtime/src/state/io/index.ts new file mode 100644 index 000000000..3cf179e6d --- /dev/null +++ b/typescript/common/runtime/src/state/io/index.ts @@ -0,0 +1,6 @@ +import type { Value } from '@commontools/data/interfaces/common-data-types.js'; + +export interface IO { + read(key: string): Value | undefined; + write(key: string, value: Value): void; +} diff --git a/typescript/common/runtime/src/state/io/slice.ts b/typescript/common/runtime/src/state/io/slice.ts new file mode 100644 index 000000000..ee5cefcea --- /dev/null +++ b/typescript/common/runtime/src/state/io/slice.ts @@ -0,0 +1,58 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { IO } from './index.js'; +import { Storage } from '../storage/index.js'; + +export class StateSlice implements IO { + #table; + + constructor(table: Map) { + this.#table = table; + } + + table(): Map { + return this.#table; + } + + clone(): StateSlice { + const table = new Map(); + for (const [key, value] of this.#table.entries()) { + table.set(key, value); + } + return new StateSlice(table); + } + + freeze() { + Object.freeze(this.#table); + } + + read(key: string): Value | undefined { + return this.#table.get(key); + } + + write(key: string, value: Value): void { + this.#table.set(key, value); + } + + async populateFrom(storage: Storage, keys: string[]) { + const values: Promise<[string, Value | void]>[] = []; + + for (const key of keys) { + values.push( + storage.read(key).then((value: Value | void) => [key, value]) + ); + } + + (await Promise.all(values)).reduce((map, [key, value]) => { + if (typeof value !== 'undefined') { + map.set(key, value); + } + return map; + }, this.#table); + } + + static async fromStorage(storage: Storage, keys: string[]) { + let state = new StateSlice(new Map()); + await state.populateFrom(storage, keys); + return state; + } +} diff --git a/typescript/common/runtime/src/state/output.ts b/typescript/common/runtime/src/state/output.ts new file mode 100644 index 000000000..f480de6ca --- /dev/null +++ b/typescript/common/runtime/src/state/output.ts @@ -0,0 +1,23 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { HostToModuleRPC } from '../rpc/index.js'; +import { Storage } from './storage/index.js'; + +export class Output implements Storage { + #rpc; + #keys; + + constructor(rpc: HostToModuleRPC, keys: string[]) { + this.#rpc = rpc; + this.#keys = keys; + } + + async read(key: string): Promise { + if (this.#keys.includes(key)) { + return await this.#rpc.send('module:output:read', { key }); + } + } + + async write(_key: string, _value: Value): Promise { + throw new Error('Writing to outputs is not allowed.'); + } +} diff --git a/typescript/common/runtime/src/state/storage/hoststorage.ts b/typescript/common/runtime/src/state/storage/hoststorage.ts new file mode 100644 index 000000000..b8dbaaed3 --- /dev/null +++ b/typescript/common/runtime/src/state/storage/hoststorage.ts @@ -0,0 +1,40 @@ +import { throwIfError } from '../../helpers.js'; +import { Value } from '../../index.js'; +import { ModuleToHostRPC } from '../../rpc/index.js'; +import { DuplexState } from '../io/duplex.js'; +import { StateSlice } from '../io/slice.js'; +import { Storage } from './index.js'; + +export class HostStorage implements Storage { + #rpc; + #inputKeys; + + constructor(rpc: ModuleToHostRPC, inputKeys: string[]) { + this.#rpc = rpc; + this.#inputKeys = inputKeys; + } + + async read(key: string): Promise { + if (this.#inputKeys.includes(key)) { + const result = await this.#rpc.send('host:storage:read', { + key, + }); + + if (!throwIfError(result)) { + return result.value; + } + } + throw new Error(`Read of key '${key}' not allowed`); + } + + write(_key: string, _value: Value): Promise { + throw new Error('Method not implemented.'); + } + + async toDuplexState(): Promise { + const input = await StateSlice.fromStorage(this, this.#inputKeys); + const output = new StateSlice(new Map()); + + return new DuplexState(input, output); + } +} diff --git a/typescript/common/runtime/src/state/storage/index.ts b/typescript/common/runtime/src/state/storage/index.ts new file mode 100644 index 000000000..afa59c25e --- /dev/null +++ b/typescript/common/runtime/src/state/storage/index.ts @@ -0,0 +1,6 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; + +export interface Storage { + read(key: string): Promise; + write(key: string, value: Value): Promise; +} diff --git a/typescript/common/runtime/src/state/storage/localstorage.ts b/typescript/common/runtime/src/state/storage/localstorage.ts new file mode 100644 index 000000000..a661d146b --- /dev/null +++ b/typescript/common/runtime/src/state/storage/localstorage.ts @@ -0,0 +1,14 @@ +import { Value } from '@commontools/data/interfaces/common-data-types.js'; +import { Storage } from './index.js'; + +export class LocalStorage implements Storage { + async read(key: string): Promise { + const serialized = localStorage.getItem(key); + return serialized ? JSON.parse(serialized) : undefined; + } + + async write(key: string, value: Value): Promise { + const serialized = JSON.stringify(value); + localStorage.setItem(key, serialized); + } +} diff --git a/typescript/common/runtime/src/worker/index.ts b/typescript/common/runtime/src/worker/index.ts new file mode 100644 index 000000000..a50ca34ca --- /dev/null +++ b/typescript/common/runtime/src/worker/index.ts @@ -0,0 +1,145 @@ +import { ContentType } from '../index.js'; +import { RuntimeEventHandler, RuntimeRequests } from '../rpc/runtime.js'; +import { + HANDSHAKE_EVENT, + ModuleToHostRPC, + RuntimeToHostRPC, +} from '../rpc/index.js'; +import { DuplexState } from '../state/io/duplex.js'; +import { IO } from '../state/io/index.js'; +import { StateSlice } from '../state/io/slice.js'; +import { + ModuleEventHandler, + ModuleRequests, + ModuleResponses, +} from '../rpc/module.js'; +import { assertNever } from '../helpers.js'; +import { HostStorage } from '../state/storage/hoststorage.js'; + +export interface ThreadLocalModule { + id: string; + run: () => void; +} + +export class ModuleInstance { + // @ts-ignore + #rpc; + #storage; + #input = new StateSlice(new Map()); + #output = new StateSlice(new Map()); + #inputKeys; + + #moduleInitializes; + + constructor( + port: MessagePort, + runtime: LocalRuntime, + contentType: ContentType, + sourceCode: string, + inputKeys: string[] + ) { + const rpc = new ModuleToHostRPC(port, this.#handleRPCEvent); + const storage = new HostStorage(rpc, inputKeys); + + this.#moduleInitializes = (async () => { + await this.#input.populateFrom(storage, inputKeys); + const io = new DuplexState(this.#input, this.#output); + const module = await runtime.eval(contentType, sourceCode, io); + + return module; + })(); + + this.#inputKeys = inputKeys; + this.#rpc = rpc; + this.#storage = storage; + } + + #handleRPCEvent = (async (event, detail) => { + switch (event) { + case 'module:run': { + try { + const module = await this.#moduleInitializes; + await this.#input.populateFrom(this.#storage, this.#inputKeys); + module.run(); + } catch (error) { + return { + error: `${error}`, + }; + } + return {} as ModuleResponses['module:run']; + } + case 'module:output:read': { + const { key } = detail as ModuleRequests['module:output:read']; + + return { + value: this.#output.read(key), + } as ModuleResponses['module:output:read']; + } + } + + return assertNever(event); + }) as ModuleEventHandler; +} + +export interface LocalRuntime { + eval( + contentType: ContentType, + sourceCode: string, + io: IO + ): Promise; +} + +export class RuntimeContext { + // @ts-ignore + #rpc: RuntimeToHostRPC | null = null; + #instances = new Map(); + #runtime; + + constructor(runtime: LocalRuntime) { + this.#runtime = runtime; + self.addEventListener('message', this.#handleGlobalMessage); + console.log('Worker runtime context initialized'); + } + + #handleGlobalMessage = (event: MessageEvent) => { + if (event.data != HANDSHAKE_EVENT) { + console.warn('Ignoring unexpected message:', event.data); + return; + } + + console.log('Got handshake event', event); + + const [port] = event.ports; + + this.#rpc = new RuntimeToHostRPC(port, this.#handleRPCEvent); + this.#rpc.send('rpc:handshake:confirmed', undefined); + }; + + #handleRPCEvent = (async (event, detail) => { + switch (event) { + case 'runtime:eval': + const { id, contentType, sourceCode, inputKeys, port } = + detail as RuntimeRequests['runtime:eval']; + + try { + const moduleInstance = await new ModuleInstance( + port, + this.#runtime, + contentType, + sourceCode, + inputKeys + ); + + this.#instances.set(id, moduleInstance); + } catch (error) { + return { + error: `${error}`, + }; + } + + return {}; + } + + return assertNever(event as never); + }) as RuntimeEventHandler; +} diff --git a/typescript/common/runtime/src/worker/pool.ts b/typescript/common/runtime/src/worker/pool.ts new file mode 100644 index 000000000..5304acb84 --- /dev/null +++ b/typescript/common/runtime/src/worker/pool.ts @@ -0,0 +1,72 @@ +import { assertNever } from '../helpers.js'; +import { Sandbox, WASM_SANDBOX } from '../index.js'; +import { HostWorkerEventHandler } from '../rpc/host.js'; +import { HostToRuntimeRPC, HANDSHAKE_EVENT } from '../rpc/index.js'; + +export type WorkerContext = { + worker: Worker; + rpc: HostToRuntimeRPC; +}; + +export class WorkerPool { + // TODO: Support multiple workers per sandbox type and some kind + // of policy-based compartmentalization for them + #categories = new Map>(); + + async get(sandbox: Sandbox): Promise { + // TODO: Handle worker failure cases + if (!this.#categories.has(sandbox)) { + console.log(`Creating new worker for '${sandbox}' modules`); + this.#categories.set( + sandbox, + new Promise((resolve) => { + const worker = this.#makeWorkerForSandbox(sandbox); + const { port1: port, port2: workerPort } = new MessageChannel(); + + const rpc: HostToRuntimeRPC = new HostToRuntimeRPC( + port, + this.#makeRpcEventHandler(() => + resolve({ + worker, + rpc, + }) + ) + ); + + setTimeout(() => { + console.log('Sending handshake'); + worker.postMessage(HANDSHAKE_EVENT, [workerPort]); + }, 1000); + }) + ); + } + + return this.#categories.get(sandbox)!; + } + + #makeWorkerForSandbox = (sandbox: Sandbox) => { + // NOTE: We use string literals for worker scripts so that bundlers (e.g., + // Vite) will correctly identify the file paths and bundle them. + // + // https://vitejs.dev/guide/features.html#web-workers:~:text=The%20worker%20detection%20will%20only%20work%20if%20the%20new%20URL()%20constructor%20is%20used%20directly%20inside%20the%20new%20Worker()%20declaration.%20Additionally%2C%20all%20options%20parameters%20must%20be%20static%20values%20(i.e.%20string%20literals). + + switch (sandbox) { + case WASM_SANDBOX: + return new Worker(new URL('./wasm/index.js', import.meta.url), { + type: 'module', + }); + default: + throw new Error(`Sandbox type '${sandbox}' not yet supported`); + } + }; + + #makeRpcEventHandler = (resolve: () => void) => + (async (event, _detail) => { + switch (event) { + case 'rpc:handshake:confirmed': { + return resolve(); + } + } + return assertNever(event as never); + }) as HostWorkerEventHandler; +} diff --git a/typescript/common/runtime/src/worker/wasm/index.ts b/typescript/common/runtime/src/worker/wasm/index.ts new file mode 100644 index 000000000..d88041a52 --- /dev/null +++ b/typescript/common/runtime/src/worker/wasm/index.ts @@ -0,0 +1,4 @@ +import { RuntimeContext } from '../index.js'; +import { RuntimeWasmWorker } from './runtime.js'; + +(self as any).runtimeContext = new RuntimeContext(new RuntimeWasmWorker()); diff --git a/typescript/common/runtime/src/worker/wasm/runtime.ts b/typescript/common/runtime/src/worker/wasm/runtime.ts new file mode 100644 index 000000000..e3ac99805 --- /dev/null +++ b/typescript/common/runtime/src/worker/wasm/runtime.ts @@ -0,0 +1,50 @@ +import { wit as commonDataWit } from '@commontools/data'; +import { wit as commonIoWit } from '@commontools/io'; +import { wit as commonModuleWit } from '@commontools/module'; +import { Runtime as UsubaRuntime } from '@commontools/usuba-rt'; + +import { Dictionary, Reference, Value } from '../../index.js'; +import { LocalRuntime, ThreadLocalModule } from '../index.js'; +import { IO } from '../../state/io/index.js'; + +export class RuntimeWasmWorker implements LocalRuntime { + #inner = new UsubaRuntime([commonDataWit, commonIoWit]); + + async eval( + contentType: 'text/javascript', + sourceCode: string, + io: IO + ): Promise { + const blueprint = await this.#inner.defineModule<{ + module: { create(): { run: () => void } }; + }>({ + contentType, + sourceCode, + wit: commonModuleWit, + }); + + const { module } = await blueprint.instantiate({ + 'common:data/types': { + Reference, + Dictionary, + }, + 'common:io/state': { + read(name: string) { + return new Reference(io, name); + }, + write(name: string, value: Value) { + io.write(name, value); + }, + }, + }); + + const innerModule = module.create(); + + return { + id: blueprint.id(), + run() { + return innerModule.run(); + }, + }; + } +} diff --git a/typescript/common/runtime/tsconfig.json b/typescript/common/runtime/tsconfig.json new file mode 100644 index 000000000..d40c067b9 --- /dev/null +++ b/typescript/common/runtime/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["es2022", "DOM", "DOM.Iterable"], + "outDir": "./lib", + "rootDir": "./src", + }, + "include": [ + "src/**/*", + ] +} diff --git a/typescript/package-lock.json b/typescript/package-lock.json index 3798405cd..66efc658e 100644 --- a/typescript/package-lock.json +++ b/typescript/package-lock.json @@ -9,12 +9,60 @@ "version": "0.0.1", "license": "UNLICENSED", "workspaces": [ - "./packages/*" + "./packages/*", + "./common/*" ], "devDependencies": { "wireit": "^0.14.4" } }, + "common/data": { + "name": "@commontools/data", + "version": "0.0.1", + "license": "UNLICENSED", + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, + "common/io": { + "name": "@commontools/io", + "version": "0.0.1", + "license": "UNLICENSED", + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, + "common/module": { + "name": "@commontools/module", + "version": "0.0.1", + "license": "UNLICENSED", + "devDependencies": { + "@bytecodealliance/jco": "^1.2.4", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, + "common/runtime": { + "name": "@commontools/runtime", + "version": "0.0.1", + "license": "UNLICENSED", + "dependencies": { + "@commontools/data": "^0.0.1", + "@commontools/io": "^0.0.1", + "@commontools/module": "^0.0.1", + "@commontools/usuba-rt": "^0.0.1", + "ses": "^1.5.0" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.6.2", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.2.tgz", @@ -74,12 +122,32 @@ "integrity": "sha512-36MwesmbLSf3Y5/OHcS85iBaF0N92CQ4gpjtDVKSbrjxmrBKCWlWVfoQ03F/cqDg8k5K7pzVaVBH0XBIbTCfTQ==", "dev": true }, - "node_modules/@commontools/runtime-demo": { - "resolved": "packages/runtime-demo", + "node_modules/@commontools/data": { + "resolved": "common/data", "link": true }, - "node_modules/@commontools/std": { - "resolved": "packages/std", + "node_modules/@commontools/example-module": { + "resolved": "packages/example-module", + "link": true + }, + "node_modules/@commontools/framework": { + "resolved": "packages/framework", + "link": true + }, + "node_modules/@commontools/io": { + "resolved": "common/io", + "link": true + }, + "node_modules/@commontools/module": { + "resolved": "common/module", + "link": true + }, + "node_modules/@commontools/runtime": { + "resolved": "common/runtime", + "link": true + }, + "node_modules/@commontools/runtime-demo": { + "resolved": "packages/runtime-demo", "link": true }, "node_modules/@commontools/usuba-api": { @@ -106,6 +174,11 @@ "node": ">=14" } }, + "node_modules/@endo/env-options": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@endo/env-options/-/env-options-1.1.4.tgz", + "integrity": "sha512-hBwS+uijkN+KDFcEM4FRMvkCeusqm/drC2WuUXZA2aqjzaWgNjneIks3m5VcFmGvEZSpjDmGtGydAS31vdk7Mw==" + }, "node_modules/@esbuild/linux-x64": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz", @@ -1558,6 +1631,14 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/ses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/ses/-/ses-1.5.0.tgz", + "integrity": "sha512-mtbONkuHpOZbJn0ueOMpBzIdNual1Ou+Tg1LbStRNsWnRljzg9ZLnoF0PyuzPPQG7kiwv2V2tN3GSJMXHPrQWg==", + "dependencies": { + "@endo/env-options": "^1.1.4" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -1865,6 +1946,20 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "packages/example-module": { + "name": "@commontools/example-module", + "version": "0.0.1", + "license": "UNLICENSED", + "dependencies": { + "@commontools/io": "^0.0.1", + "@commontools/module": "^0.0.1" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, "packages/example-package": { "version": "0.0.1", "extraneous": true, @@ -1876,11 +1971,29 @@ "wireit": "^0.14.4" } }, + "packages/framework": { + "name": "@commontools/framework", + "version": "0.0.1", + "license": "UNLICENSED", + "dependencies": { + "@commontools/io": "^0.0.1", + "@commontools/module": "^0.0.1" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + } + }, "packages/runtime-demo": { "name": "@commontools/runtime-demo", "version": "0.0.1", "license": "UNLICENSED", "devDependencies": { + "@commontools/data": "^0.0.1", + "@commontools/io": "^0.0.1", + "@commontools/module": "^0.0.1", + "@commontools/runtime": "^0.0.1", "@commontools/usuba-rt": "^0.0.1", "@shoelace-style/shoelace": "^2.15.1", "typescript": "^5.2.2", @@ -1891,6 +2004,7 @@ "packages/std": { "name": "@commontools/std", "version": "0.0.1", + "extraneous": true, "license": "UNLICENSED", "devDependencies": { "@bytecodealliance/jco": "^1.2.4", @@ -1947,6 +2061,7 @@ "devDependencies": { "@commontools/usuba-api": "^0.0.1", "@commontools/usuba-sw": "^0.0.1", + "tslib": "^2.6.2", "typescript": "^5.2.2", "vite": "^5.2.0", "wireit": "^0.14.4" diff --git a/typescript/package.json b/typescript/package.json index d5aab8573..5c353eec5 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -7,7 +7,8 @@ "private": true, "type": "module", "workspaces": [ - "./packages/*" + "./packages/*", + "./common/*" ], "scripts": { "build": "wireit", @@ -30,7 +31,13 @@ "./packages/usuba-rt:build", "./packages/usuba-api:build", "./packages/usuba-sw:build", - "./packages/usuba-ui:build" + "./packages/usuba-ui:build", + "./packages/runtime-demo:build", + "./packages/example-module:build", + "./common/data:build", + "./common/io:build", + "./common/module:build", + "./common/runtime:build" ] }, "clean": { @@ -38,8 +45,14 @@ "./packages/usuba-rt:clean", "./packages/usuba-api:clean", "./packages/usuba-sw:clean", - "./packages/usuba-ui:clean" + "./packages/usuba-ui:clean", + "./packages/runtime-demo:clean", + "./packages/example-module:clean", + "./common/data:clean", + "./common/io:clean", + "./common/module:clean", + "./common/runtime:clean" ] } } -} +} \ No newline at end of file diff --git a/typescript/packages/.gitignore b/typescript/packages/.gitignore new file mode 100644 index 000000000..3e88696bf --- /dev/null +++ b/typescript/packages/.gitignore @@ -0,0 +1 @@ +example-js-component \ No newline at end of file diff --git a/typescript/packages/example-module/package.json b/typescript/packages/example-module/package.json new file mode 100644 index 000000000..f0ae1c9ed --- /dev/null +++ b/typescript/packages/example-module/package.json @@ -0,0 +1,52 @@ +{ + "name": "@commontools/example-module", + "author": "The Common Authors", + "version": "0.0.1", + "description": "An example of a Module that is run in an isolated context", + "license": "UNLICENSED", + "private": true, + "type": "module", + "scripts": { + "build": "wireit", + "clean": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "exports": "./lib/index.js", + "files": [ + "./lib/index.js" + ], + "dependencies": { + "@commontools/module": "^0.0.1", + "@commontools/io": "^0.0.1" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "build": { + "dependencies": [ + "../../common/module:build", + "../../common/io:build" + ], + "files": [ + "./src/**/*" + ], + "output": [ + "./lib/**/*" + ], + "command": "tsc --build -f" + }, + "clean": { + "command": "rm -rf ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/packages/example-module/src/index.ts b/typescript/packages/example-module/src/index.ts new file mode 100644 index 000000000..0ceb8e081 --- /dev/null +++ b/typescript/packages/example-module/src/index.ts @@ -0,0 +1,21 @@ +// @ts-ignore +import { read } from 'common:io/state@0.0.1'; + +export class Body { + run() { + console.log('Running!'); + + const foo = read('foo'); + + console.log('Reference:', foo); + + const value = foo?.deref(); + + console.log('Value:', value); + } +} + +export const create = () => { + console.log('Creating!'); + // return new Body(); +}; diff --git a/typescript/packages/example-module/tsconfig.json b/typescript/packages/example-module/tsconfig.json new file mode 100644 index 000000000..5e1a61b10 --- /dev/null +++ b/typescript/packages/example-module/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["es2022", "DOM"], + "outDir": "./lib", + "rootDir": "./src", + "strict": false, + "paths": { + "common:module/module@0.0.1": [ + "../../node_modules/@commontools/module/lib/index.d.ts" + ], + "common:io/state@0.0.1": [ + "../../node_modules/@commontools/io/lib/index.d.ts" + ] + } + }, + "include": [ + "src/**/*", + ] +} diff --git a/typescript/packages/framework/package.json b/typescript/packages/framework/package.json new file mode 100644 index 000000000..c20c7a987 --- /dev/null +++ b/typescript/packages/framework/package.json @@ -0,0 +1,57 @@ +{ + "name": "@commontools/framework", + "author": "The Common Authors", + "version": "0.0.1", + "description": "A framework for building Common JavaScript Modules", + "license": "UNLICENSED", + "private": true, + "type": "module", + "scripts": { + "build": "wireit", + "clean": "wireit" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/commontoolsinc/labs.git" + }, + "bugs": { + "url": "https://github.com/commontoolsinc/labs/issues" + }, + "homepage": "https://github.com/commontoolsinc/labs#readme", + "exports": "./lib/index.js", + "files": [ + "./lib/*.js" + ], + "dependencies": { + "@commontools/module": "^0.0.1", + "@commontools/io": "^0.0.1" + }, + "devDependencies": { + "tslib": "^2.6.2", + "typescript": "^5.2.2", + "wireit": "^0.14.4" + }, + "wireit": { + "build:tsc": { + "dependencies": [ + "../../common/module:build", + "../../common/io:build" + ], + "files": [ + "./src/**/*" + ], + "output": [ + "./lib/**/*" + ], + "command": "tsc --build -f" + }, + "build": { + "dependencies": [ + "build:tsc" + ] + }, + "clean": { + "command": "rm -rf ./lib ./.wireit" + } + } +} \ No newline at end of file diff --git a/typescript/packages/framework/src/index.ts b/typescript/packages/framework/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/typescript/packages/framework/tsconfig.json b/typescript/packages/framework/tsconfig.json new file mode 100644 index 000000000..d40c067b9 --- /dev/null +++ b/typescript/packages/framework/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["es2022", "DOM", "DOM.Iterable"], + "outDir": "./lib", + "rootDir": "./src", + }, + "include": [ + "src/**/*", + ] +} diff --git a/typescript/packages/runtime-demo/index.html b/typescript/packages/runtime-demo/index.html index cc8e35529..df9a2c810 100644 --- a/typescript/packages/runtime-demo/index.html +++ b/typescript/packages/runtime-demo/index.html @@ -23,6 +23,8 @@ Demo 2 Demo 3 Demo 4 + Demo 5 + Demo 6 diff --git a/typescript/packages/runtime-demo/package.json b/typescript/packages/runtime-demo/package.json index 0dbfcf435..a4cd87c0e 100644 --- a/typescript/packages/runtime-demo/package.json +++ b/typescript/packages/runtime-demo/package.json @@ -19,7 +19,11 @@ }, "homepage": "https://github.com/commontoolsinc/labs#readme", "devDependencies": { + "@commontools/module": "^0.0.1", + "@commontools/data": "^0.0.1", + "@commontools/io": "^0.0.1", "@commontools/usuba-rt": "^0.0.1", + "@commontools/runtime": "^0.0.1", "@shoelace-style/shoelace": "^2.15.1", "typescript": "^5.2.2", "vite": "^5.2.0", @@ -28,12 +32,23 @@ "wireit": { "build": { "dependencies": [ + "../../common/module:build", + "../../common/data:build", + "../../common/io:build", + "../../common/runtime:build", "../usuba-rt:build" ], "command": "vite build" }, "clean": { - "command": "rm -rf ./lib ./dist ./.wireit" + "dependencies": [ + "../../common/module:clean", + "../../common/data:clean", + "../../common/io:clean", + "../../common/runtime:clean", + "../usuba-rt:clean" + ], + "command": "rm -rf ./lib ./dist ./.wireit ./node_modules" } } } \ No newline at end of file diff --git a/typescript/packages/runtime-demo/src/demo5.ts b/typescript/packages/runtime-demo/src/demo5.ts new file mode 100644 index 000000000..39c2208bb --- /dev/null +++ b/typescript/packages/runtime-demo/src/demo5.ts @@ -0,0 +1,91 @@ +import { Runtime } from '@commontools/usuba-rt'; +import { wit as commonModuleWit } from '@commontools/module'; +import { wit as commonDataWit } from '@commontools/data'; +import { wit as commonIoWit } from '@commontools/io'; + +const EXAMPLE_MODULE_JS = ` +import { read } from 'common:io/state@0.0.1'; + +export class Body { + run() { + console.log('Running!'); + const foo = read('foo'); + console.log('Reference:', foo); + const value = foo?.deref(); + console.log('Value:', value); + } +} + +export const module = { + Body, + + create() { + console.log('Creating!'); + return new Body(); + } +};`; + +export const demo = async () => { + const runtime = new Runtime([commonDataWit, commonIoWit]); + + console.log(Runtime); + + const module = await runtime.defineModule({ + contentType: 'text/javascript', + wit: commonModuleWit, + sourceCode: EXAMPLE_MODULE_JS, + }); + + class Reference { + #value: any; + constructor(value: any) { + this.#value = value; + } + deref() { + return this.#value; + } + } + + const theOnlyReference = new Reference({ + tag: 'string', + val: 'WOOT', + }); + + const api = await module.instantiate({ + 'common:data/types': { + Map: Map, + Reference, + }, + 'common:io/state': { + Stream: class Stream { + next() { + console.log('Not implemented!'); + } + }, + read(name: string) { + return theOnlyReference; + }, + write(_name: string, _value: any) { + console.log('Write ignored!'); + }, + subscribe(_name: string) { + console.log('Subscribe ignored!'); + }, + }, + }); + const { + module: { create }, + }: any = api; + + console.log(api, create); + + const body = create(); + + body.run(); + body.run(); + body.run(); + body.run(); + body.run(); + + console.log('FIN'); +}; diff --git a/typescript/packages/runtime-demo/src/demo6.ts b/typescript/packages/runtime-demo/src/demo6.ts new file mode 100644 index 000000000..3c7358985 --- /dev/null +++ b/typescript/packages/runtime-demo/src/demo6.ts @@ -0,0 +1,73 @@ +import { + IO, + Runtime, + Dictionary, + Value, + infer, + Input, + LocalStorage, + WASM_SANDBOX, +} from '@commontools/runtime'; + +const EXAMPLE_MODULE_JS = ` +import { read, write } from 'common:io/state@0.0.1'; + +export class Body { + run() { + console.log('Running!'); + + const foo = read('foo'); + const value = foo?.deref(); + + console.log('Value:', value); + + console.log('Setting some output...'); + + write('baz', { + tag: 'string', + val: 'quux' + }); + } +} + +export const module = { + Body, + + create() { + console.log('Creating!'); + return new Body(); + } +};`; + +const bar = { + baz: 'quux', +}; + +export const demo = async () => { + localStorage.clear(); + + const rt = new Runtime(); + const storage = new LocalStorage(); + + console.log('Instantiating the module'); + + const module = await rt.eval( + 'example', + WASM_SANDBOX, + 'text/javascript', + EXAMPLE_MODULE_JS, + new Input(storage, ['foo', 'baz']) + ); + + console.log(`Setting 'foo => bar' at the host level`); + + await storage.write('foo', infer('bar')); + + console.log('Running the module:'); + + await module.run(); + + const output = module.output(['baz']); + + console.log('Reading output in host thread:', await output.read('baz')); +}; diff --git a/typescript/packages/runtime-demo/src/index.ts b/typescript/packages/runtime-demo/src/index.ts index 886cb67fc..66ac82ab7 100644 --- a/typescript/packages/runtime-demo/src/index.ts +++ b/typescript/packages/runtime-demo/src/index.ts @@ -2,8 +2,15 @@ import { demo as demoOne } from './demo1.js'; import { demo as demoTwo } from './demo2.js'; import { demo as demoThree } from './demo3.js'; import { demo as demoFour } from './demo4.js'; +import { demo as demoFive } from './demo5.js'; +import { demo as demoSix } from './demo6.js'; +import { activateServiceWorker } from '@commontools/usuba-rt'; + +await activateServiceWorker(); (self as any).demoOne = demoOne; (self as any).demoTwo = demoTwo; (self as any).demoThree = demoThree; (self as any).demoFour = demoFour; +(self as any).demoFive = demoFive; +(self as any).demoSix = demoSix; diff --git a/typescript/packages/runtime-demo/vite.config.js b/typescript/packages/runtime-demo/vite.config.js index edb6fa879..30270c6a8 100644 --- a/typescript/packages/runtime-demo/vite.config.js +++ b/typescript/packages/runtime-demo/vite.config.js @@ -8,5 +8,11 @@ export default defineConfig({ }, resolve: { preserveSymlinks: true + }, + worker: { + format: 'es' + }, + optimizeDeps: { + noDiscovery: true } }) \ No newline at end of file diff --git a/typescript/packages/usuba-api/scripts/fix-paths.sh b/typescript/packages/usuba-api/scripts/fix-paths.sh index 348a002eb..511782741 100755 --- a/typescript/packages/usuba-api/scripts/fix-paths.sh +++ b/typescript/packages/usuba-api/scripts/fix-paths.sh @@ -11,7 +11,14 @@ set -euo pipefail pushd $SCRIPT_DIR/../src/openapi-client -find ./ -type f -exec sed -i "/import\|export/ s/\(.* '\.\/[^']*\)/\0.js/g" {} + +SED=sed + +if command -v gsed &> /dev/null; +then + SED=gsed +fi + +find ./ -type f -exec $SED -i "/import\|export/ s/\(.* '\.\/[^']*\)/\0.js/g" {} + popd diff --git a/typescript/packages/usuba-rt/src/index.ts b/typescript/packages/usuba-rt/src/index.ts index 537a635fd..b30c14fe2 100644 --- a/typescript/packages/usuba-rt/src/index.ts +++ b/typescript/packages/usuba-rt/src/index.ts @@ -34,7 +34,7 @@ const FILE_EXTENSIONS: ContentTypeFileExtensions = { 'text/x-python': 'py', }; -const serviceWorkerActivates = (async () => { +export const activateServiceWorker = async () => { if (typeof navigator.serviceWorker == 'undefined') { throw new Error( 'Service Worker is not supported in this browser; Usuba will not work here.' @@ -78,7 +78,7 @@ const serviceWorkerActivates = (async () => { } catch (error) { console.error(`Registration failed with ${error}`); } -})(); +}; /** * A Runtime embodies: @@ -93,13 +93,12 @@ const serviceWorkerActivates = (async () => { * definitions or promises that resolve to WIT definitions. */ export class Runtime { - #serviceWorkerActivates: Promise = serviceWorkerActivates; #library: Promise; #usubaHost: URL; constructor( library: PendingSourceCode[], - usubaHost: URL = new URL(window.location.origin) + usubaHost: URL = new URL(new URL(import.meta.url).origin) ) { this.#library = Promise.all(library).then((library) => library.map( @@ -126,11 +125,10 @@ export class Runtime { async defineModule( definition: ModuleDefinition ): Promise> { - const [library, wit, sourceCode, _] = await Promise.all([ + const [library, wit, sourceCode] = await Promise.all([ this.#library, definition.wit, definition.sourceCode, - this.#serviceWorkerActivates, ]); apiClient.OpenAPI.BASE = this.#usubaHost.origin; @@ -154,6 +152,7 @@ export class Runtime { ); return new PreparedModule( + id, instantiate as (imports: ImportMap) => Promise ); } @@ -170,9 +169,15 @@ export class Runtime { * always an implementation of the interface defined by the Module's WIT. */ export class PreparedModule { + #id; #instantiate: (imports: ImportMap) => Promise; - constructor(instantiate: any) { + id() { + return this.#id; + } + + constructor(id: string, instantiate: any) { + this.#id = id; this.#instantiate = instantiate; } @@ -214,6 +219,8 @@ export class PreparedModule { return map; }, {} as ImportMap); + console.log('Invoking instantiate on the module'); + return await this.#instantiate(imports); } }