Implement a prototype of CFC

deno.json

@@ -73,7 +73,8 @@
     "singleQuote": false,
     "proseWrap": "always",
     "exclude": [
-      "packages/jumble/integration/cache/llm-api-cache/"
+      "packages/jumble/integration/cache/llm-api-cache/",
+      "packages/seeder/templates/"
     ]
   },
   "imports": {
@@ -114,4 +115,4 @@
     "zod-to-json-schema": "npm:zod-to-json-schema@^3.24.1",
     "zod": "npm:zod@^3.24.1"
   }
-}
+}

packages/background-charm-service/src/schema.ts

@@ -25,7 +25,7 @@ export const BGCharmEntrySchema = {
     "lastRun",
     "status",
   ],
-} as const as JSONSchema;
+} as const satisfies JSONSchema;
 export type BGCharmEntry = Schema<typeof BGCharmEntrySchema>;
 
 export const BGCharmEntriesSchema = {

packages/builder/src/opaque-ref.ts

@@ -15,6 +15,7 @@ import { hasValueAtPath, setValueAtPath } from "./utils.ts";
 import { getTopFrame, recipe } from "./recipe.ts";
 import { createNodeFactory } from "./module.ts";
 import { SchemaWithoutCell } from "./schema-to-ts.ts";
+import { ContextualFlowControl } from "../../runner/src/index.ts";
 
 let mapFactory: NodeFactory<any, any>;
 
@@ -54,12 +55,13 @@ export function opaqueRef<T>(
   };
 
   let unsafe_binding: { recipe: Recipe; path: PropertyKey[] } | undefined;
+  const cfc = new ContextualFlowControl();
 
   function createNestedProxy(
     path: PropertyKey[],
-    target?: any,
-    schema?: JSONSchema,
-    rootSchema: JSONSchema | undefined = schema,
+    target: any,
+    nestedSchema: JSONSchema | undefined,
+    rootSchema: JSONSchema | undefined,
   ): OpaqueRef<any> {
     const methods: OpaqueRefMethods<any> = {
       get: () => unsafe_materialize(unsafe_binding, path),
@@ -70,23 +72,14 @@ export function opaqueRef<T>(
       },
       key: (key: PropertyKey) => {
         // Determine child schema when accessing a property
-        let childSchema: JSONSchema | undefined;
-
-        if (schema?.type === "object") {
-          // For root object properties
-          childSchema = schema.properties?.[key as string] ||
-            (typeof schema.additionalProperties === "object"
-              ? schema.additionalProperties
-              : undefined);
-        } else if (schema?.type === "array") {
-          // For root array elements
-          childSchema = schema.items;
-        }
+        const childSchema = key in methods
+          ? undefined
+          : cfc.getSchemaAtPath(nestedSchema, [key.toString()], rootSchema);
         return createNestedProxy(
           [...path, key],
           key in methods ? methods[key as keyof OpaqueRefMethods<any>] : store,
           childSchema,
-          childSchema ? rootSchema : undefined, // Only pass rootSchema if we have a child schema
+          childSchema === undefined ? undefined : rootSchema,
         );
       },
       setDefault: (newValue: Opaque<any>) => {
@@ -102,13 +95,13 @@ export function opaqueRef<T>(
       setSchema: (newSchema: JSONSchema) => {
         // This sets the schema of the nested proxy, but does not alter the parent store's
         // schema. Our schema variable shadows that one.
-        schema = newSchema;
+        nestedSchema = newSchema;
       },
       connect: (node: NodeRef) => store.nodes.add(node),
       export: () => {
         // Store's schema won't be the same as ours as a nested proxy
         // We also don't adjust the defaultValue to be relative to our path
-        return { cell: top, path, rootSchema, ...store, schema };
+        return { cell: top, ...store, path, rootSchema, schema: nestedSchema };
       },
       unsafe_bindToRecipeAndPath: (
         recipe: Recipe,
@@ -159,9 +152,17 @@ export function opaqueRef<T>(
                 "Can't use iterator over an opaque value in an unlimited loop.",
               );
             }
+            const childSchema = cfc.getSchemaAtPath(nestedSchema, [
+              index.toString(),
+            ], rootSchema);
             return {
               done: false,
-              value: createNestedProxy([...path, index++]),
+              value: createNestedProxy(
+                [...path, index++],
+                target,
+                childSchema,
+                childSchema === undefined ? undefined : rootSchema,
+              ),
             };
           },
         };
@@ -191,7 +192,7 @@ export function opaqueRef<T>(
     return proxy;
   }
 
-  const top = createNestedProxy([], store, schema) as OpaqueRef<T>;
+  const top = createNestedProxy([], store, schema, schema) as OpaqueRef<T>;
 
   store.frame.opaqueRefs.add(top);
 

packages/builder/src/recipe.ts

@@ -100,7 +100,9 @@ export function recipe<T, R>(
 
   const inputs = opaqueRef<Required<T>>(
     undefined,
-    argumentSchema as JSONSchema | undefined,
+    typeof argumentSchema === "string"
+      ? undefined
+      : argumentSchema as JSONSchema | undefined,
   );
 
   const outputs = fn!(inputs);
@@ -125,7 +127,9 @@ export function recipeFromFrame<T, R>(
 ): RecipeFactory<T, R> {
   const inputs = opaqueRef<Required<T>>(
     undefined,
-    argumentSchema as JSONSchema | undefined,
+    typeof argumentSchema === "string"
+      ? undefined
+      : argumentSchema as JSONSchema | undefined,
   );
   const outputs = fn(inputs);
   return factoryFromRecipe<T, R>(argumentSchema, resultSchema, inputs, outputs);

packages/builder/src/types.ts

@@ -145,6 +145,7 @@ export type JSONSchema = {
   readonly enum?: readonly string[];
   readonly items?: Readonly<JSONSchema>;
   readonly $ref?: string;
+  readonly $defs?: Readonly<Record<string, JSONSchema>>;
   readonly asCell?: boolean;
   readonly asStream?: boolean;
   readonly anyOf?: readonly JSONSchema[];

packages/builder/test/opaque-ref-schema.test.ts

@@ -215,7 +215,7 @@ describe("OpaqueRef Schema Support", () => {
       expect(exported.rootSchema).toEqual(schema);
     });
 
-    it("should return undefined schema for properties that aren't included in the schema", () => {
+    it("should return undefined schema for properties that aren't allowed by the schema", () => {
       // Set a schema with nested objects
       const schema = {
         type: "object",
@@ -228,6 +228,7 @@ describe("OpaqueRef Schema Support", () => {
             },
           },
         },
+        additionalProperties: false,
       } as const satisfies JSONSchema;
 
       // Create an opaque ref with nested objects, and a property that isn't in the schema

packages/builder/test/recipe.test.ts

@@ -380,18 +380,22 @@ describe("recipe with ifc property", () => {
         schema: { ifc: ArgumentSchema.properties.x.ifc },
       },
     });
-    // I don't like that we don't know our input is classified here
     expect(nodes[1].inputs).toEqual({
       x: {
         $alias: {
           path: ["internal", "__#0", "double"],
+          schema: {
+            ifc: ArgumentSchema.properties.x.ifc,
+          },
         },
       },
     });
-    // I don't like that we don't know our output is classified here
     expect(nodes[1].outputs).toEqual({
       $alias: {
         path: ["internal", "__#1"],
+        schema: {
+          ifc: ArgumentSchema.properties.x.ifc,
+        },
       },
     });
   });
@@ -459,7 +463,7 @@ describe("recipe with mixed ifc properties", () => {
   );
 
   it("has the correct classification in the schema of the ssn result", () => {
-    const { result, nodes, argumentSchema } = capitalizeSsnRecipe;
+    const { result, nodes, argumentSchema, resultSchema } = capitalizeSsnRecipe;
     expect(isRecipe(capitalizeSsnRecipe)).toBe(true);
     expect(argumentSchema).toMatchObject(UserSchema);
     expect(nodes).toHaveLength(1);
@@ -477,6 +481,10 @@ describe("recipe with mixed ifc properties", () => {
         },
       },
     });
+    expect(resultSchema).toEqual({
+      ...ResultSchema,
+      ...{ ifc: { classification: ["confidential"] } },
+    });
   });
 
   // Perhaps I should handle a similar recipe that only accesses the name

packages/charm/src/iterate.ts

@@ -222,7 +222,11 @@ export function scrub(data: any): any {
       // If this resulted in an empty schema, return without a schema
       return data.asSchema(
         Object.keys(scrubbed).length > 0
-          ? { ...data.schema, properties: scrubbed }
+          ? {
+            ...data.schema,
+            properties: scrubbed,
+            additionalProperties: false,
+          }
           : undefined,
       );
     } else {
@@ -238,7 +242,8 @@ export function scrub(data: any): any {
               (key) => [key, {}],
             ),
           ),
-        } satisfies JSONSchema;
+          additionalProperties: false,
+        } as const satisfies JSONSchema;
         console.log("scrubbed generated schema", scrubbed);
         // Only if we found any properties, return the scrubbed schema
         return Object.keys(scrubbed).length > 0

packages/cli/curl.ts

@@ -0,0 +1,137 @@
+// Load .env file
+import { parseArgs } from "@std/cli/parse-args";
+import { type DID, Identity } from "@commontools/identity";
+import { Provider as CachedStorageProvider } from "../runner/src/storage/cache.ts";
+// Some examples of how you can use this to play with the classification labels
+// Store the empty list
+// > deno task curl --spaceName robin --data '[]' ct://did:key:z6MkjMowGqCog2ZfvNBNrx32p2Fa2bKR1nT7pUWiPQFWzVAg/baedreih5ute2slgsylwtbszccarx6ky2ca3mtticxug6sfj3nwamacefmn/application/json
+// Mark the entity secret
+// > deno task curl --spaceName robin --raw --data '{"classification": ["secret"]}' ct://did:key:z6MkjMowGqCog2ZfvNBNrx32p2Fa2bKR1nT7pUWiPQFWzVAg/baedreih5ute2slgsylwtbszccarx6ky2ca3mtticxug6sfj3nwamacefmn/application/label+json
+// Check the classification labels (requires the classification labels)
+// > deno task curl --spaceName robin --raw --schema '{"ifc": {"classification": ["secret"]}}' ct://did:key:z6MkjMowGqCog2ZfvNBNrx32p2Fa2bKR1nT7pUWiPQFWzVAg/baedreih5ute2slgsylwtbszccarx6ky2ca3mtticxug6sfj3nwamacefmn/application/label+json
+// Get the original empty list back (requires the classification labels)
+// > deno task curl --spaceName robin --schema '{"ifc": {"classification": ["secret"]}}' ct://did:key:z6MkjMowGqCog2ZfvNBNrx32p2Fa2bKR1nT7pUWiPQFWzVAg/baedreih5ute2slgsylwtbszccarx6ky2ca3mtticxug6sfj3nwamacefmn/application/json
+const flags = parseArgs(Deno.args, {
+  string: [
+    "spaceName",
+    "key",
+    "data",
+    "schema",
+  ],
+  boolean: ["admin"],
+  default: { the: "application/json", admin: false, raw: false },
+});
+
+const toolshedUrl = Deno.env.get("TOOLSHED_API_URL") ??
+  "http://localhost:8000/";
+
+const ANYONE = "common user";
+const remoteStorageUrl = new URL(toolshedUrl);
+
+function usage() {
+  console.log(
+    "Usage: curl [--key <keyfile>] [--spaceName <spaceName>] [--admin] [--raw] [--schema <schema>] [--data <data>] url\n" +
+      "Example URL: ct://did:key:z6MkjMowGqCog2ZfvNBNrx32p2Fa2bKR1nT7pUWiPQFWzVAg/baedreihxpwcmhvzpf5weuf4ceow4zbahqikvu5ploox36ipeuvqnminyba/application/json\n" +
+      "If you provide a spaceDID in the URL, you must either be using the admin flag, or provide the --spaceName option.\n" +
+      "You can also provide a spaceName in the URL if it does not include any colon or slash characters. In this case, you do not need to provide the --spaceName option.",
+  );
+}
+async function main() {
+  const url = flags._[0];
+  if (!url || typeof url !== "string") {
+    console.error("Must provide an url");
+    usage();
+    Deno.exit(1);
+  }
+  // Parse the url like ct://spaceDID/entityID/attribute
+  // did key is base58btc; entity id is base32; attribute is mime type-ish
+  const urlRegex: RegExp =
+    /^(ct:\/\/)?((?<spaceDID>(did:key:[1-9A-HJ-NP-Za-km-z]+))|(?<spaceName>[^/:]+))\/(?<of>[a-z2-7]+)(\/(?<the>\w+\/[-+.\w]+))?$/;
+  const match = url.match(urlRegex);
+  if (match === null || match.groups === undefined) {
+    console.error("Invalid url");
+    Deno.exit(1);
+  }
+  const entityId = { "/": match.groups.of };
+  const the = (match.groups.the && match.groups.the !== "")
+    ? match.groups.the
+    : "application/json";
+  if (!match.groups.spaceName && !match.groups.spaceDID) {
+    console.error("No space name or space DID found");
+    Deno.exit(1);
+  }
+
+  let identity: Identity;
+  if (flags.key) {
+    try {
+      const pkcs8Key = await Deno.readFile(flags.key);
+      identity = await Identity.fromPkcs8(pkcs8Key);
+    } catch (e) {
+      console.error(
+        `Could not read key at ${flags.key}.`,
+      );
+      Deno.exit(1);
+    }
+  } else {
+    identity = await Identity.fromPassphrase(ANYONE);
+  }
+
+  // Actual identity is derived from space name if we don't provide an admin key
+  if (!flags.admin) {
+    const spaceName = match.groups.spaceName ?? flags.spaceName;
+    identity = await identity.derive(spaceName);
+  }
+  const spaceDID = match.groups.spaceDID
+    ? match.groups.spaceDID as DID
+    : identity.did();
+  const schema = flags.schema ? JSON.parse(flags.schema) : {};
+
+  // TODO(@ubik2) - this constrains us to values that are json
+  // need to revisit for image or blob support
+  const putData = flags.data ? JSON.parse(flags.data) : undefined;
+
+  const storageId = crypto.randomUUID();
+  const provider = new CachedStorageProvider({
+    id: storageId,
+    address: new URL("/api/storage/memory", remoteStorageUrl),
+    space: spaceDID,
+    as: identity,
+    the: the,
+    settings: {
+      maxSubscriptionsPerSpace: 50_000,
+      connectionTimeout: 30_000,
+      useSchemaQueries: true,
+    },
+  });
+  if (!putData) {
+    const result = await provider.sync(entityId, true, {
+      schema: schema,
+      rootSchema: schema,
+    });
+    if (result.error) {
+      console.log("Failed to sync object", result.error);
+    }
+    const storageValue = provider.get(entityId);
+    const data = flags.raw ? storageValue : storageValue?.value;
+
+    console.log(JSON.stringify(data));
+    Deno.exit(0);
+  } else {
+    // The entries in the database all get a value key and store their value there,
+    // so we need to do the same for the value we provide to StorageValue for send.
+    const result = await provider.send([{
+      entityId: entityId,
+      value: flags.raw ? putData : { value: putData },
+    }]);
+    if (result.ok) {
+      const putDataJSON = JSON.stringify(putData);
+      console.log(
+        `Stored ${putDataJSON} at ct://${spaceDID}/${entityId["/"]}/${the}`,
+      );
+    } else {
+      console.error("Failed to put data:", result.error);
+    }
+    Deno.exit(0);
+  }
+}
+main();

packages/cli/deno.json

@@ -4,6 +4,7 @@
     "google-importer": "deno run --allow-read --allow-env --allow-net google-importer.ts",
     "background-runner": "deno run --allow-read --allow-env --allow-net background-charm-runner.ts",
     "memorydemo": "deno run --allow-read --allow-env --allow-net memory_demo.ts",
+    "curl": "deno run --allow-read --allow-env --allow-net curl.ts",
     "test": "deno test",
     "charmdemo": "deno run --allow-read --allow-env --allow-net charm_demo.ts",
     "name2did": "deno run -A name2did.ts",