feat(runner): create data URIs for array element objects not marked asCell

packages/runner/src/link-utils.ts

@@ -511,7 +511,7 @@ export function findAndInlineDataURILinks(value: any): any {
 export function createDataCellURI(
   data: any,
   base?: Cell | NormalizedLink,
-): string {
+): URI {
   const baseId = isCell(base) ? base.getAsNormalizedFullLink().id : base?.id;
 
   function traverseAndAddBaseIdToRelativeLinks(
@@ -549,8 +549,8 @@ export function createDataCellURI(
   const json = JSON.stringify({
     value: traverseAndAddBaseIdToRelativeLinks(data, new Set()),
   });
-  const base64 = btoa(json);
-  return `data:application/json;charset=utf-8;base64,${base64}`;
+  // Use encodeURIComponent for UTF-8 safe encoding (matches runtime.ts pattern)
+  return `data:application/json,${encodeURIComponent(json)}` as URI;
 }
 
 /**

packages/runner/src/schema.ts

@@ -8,7 +8,11 @@ import { createCell, isCell, isStream } from "./cell.ts";
 import { readMaybeLink, resolveLink } from "./link-resolution.ts";
 import { type IExtendedStorageTransaction } from "./storage/interface.ts";
 import { type IRuntime } from "./runtime.ts";
-import { type NormalizedFullLink } from "./link-utils.ts";
+import {
+  createDataCellURI,
+  type NormalizedFullLink,
+  parseLink,
+} from "./link-utils.ts";
 import {
   createQueryResultProxy,
   isQueryResultForDereferencing,
@@ -737,20 +741,6 @@ export function validateAndTransform(
 
     // Now process elements after adding the array to seen
     for (let i = 0; i < value.length; i++) {
-      // If the element on the array is a link, we follow that link so the
-      // returned object is the current item at that location (otherwise the
-      // link would refer to "Nth element"). This is important when turning
-      // returned objects back into cells: We want to then refer to the actual
-      // object by default, not the array location.
-      //
-      // This makes
-      // ```ts
-      // const array = [...cell.get()];
-      // array.splice(index, 1);
-      // cell.set(array);
-      // ```
-      // work as expected.
-      // Handle boolean items values for element schema
       let elementSchema: JSONSchema;
       if (resolvedSchema.items === true) {
         // items: true means allow any item type
@@ -772,13 +762,56 @@ export function validateAndTransform(
         path: [...link.path, String(i)],
         schema: elementSchema,
       };
-      const maybeLink = readMaybeLink(tx ?? runtime.edit(), elementLink);
+
+      // If the element on the array is a link, we follow that link so the
+      // returned object is the current item at that location (otherwise the
+      // link would refer to "Nth element"). This is important when turning
+      // returned objects back into cells: We want to then refer to the actual
+      // object by default, not the array location.
+      //
+      // If the element is an object, but not a link, we create an immutable
+      // cell to hold the object, except when it is requested as Cell. While
+      // this means updates aren't propagated, it seems like the right trade-off
+      // for stability of links and the ability to mutate them without creating
+      // loops (see below).
+      //
+      // This makes
+      // ```ts
+      // const array = [...cell.get()];
+      // array.splice(index, 1);
+      // cell.set(array);
+      // ```
+      // work as expected. Handle boolean items values for element schema
+      const maybeLink = parseLink(value[i], link);
       if (maybeLink) {
         elementLink = {
           ...maybeLink,
           schema: elementLink.schema,
           rootSchema: elementLink.rootSchema,
         };
+      } else if (
+        isRecord(value[i]) &&
+        // TODO(seefeld): Should factor this out, but we should just fully
+        // normalize schemas, etc.
+        !(isObject(elementSchema) &&
+          (elementSchema.asCell || elementSchema.asStream ||
+            (Array.isArray(elementSchema?.anyOf) &&
+              elementSchema.anyOf.every((option) =>
+                option.asCell || option.asStream
+              )) ||
+            (Array.isArray(elementSchema?.oneOf) &&
+              elementSchema.oneOf.every((option) =>
+                option.asCell || option.asStream
+              ))))
+      ) {
+        elementLink = {
+          id: createDataCellURI(value[i], link),
+          path: [],
+          schema: elementSchema,
+          rootSchema: elementLink.rootSchema,
+          space: link.space,
+          type: "application/json",
+        } satisfies NormalizedFullLink;
       }
 
       result[i] = validateAndTransform(

packages/runner/src/uri-utils.ts

@@ -90,7 +90,19 @@ export function getJSONFromDataURI(uri: URI | string): any {
   // Check if data is base64 encoded
   const isBase64 = headerParts.some((part) => part === "base64");
 
-  const decodedData = isBase64 ? atob(data) : decodeURIComponent(data);
+  let decodedData: string;
+  if (isBase64) {
+    // Use TextDecoder to properly decode UTF-8 bytes from base64
+    const binaryString = atob(data);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+    const decoder = new TextDecoder();
+    decodedData = decoder.decode(bytes);
+  } else {
+    decodedData = decodeURIComponent(data);
+  }
 
   return decodedData.length > 0 ? JSON.parse(decodedData) : undefined;
 }

packages/runner/test/link-utils.test.ts

@@ -13,6 +13,7 @@ import {
   parseLinkOrThrow,
   sanitizeSchemaForLinks,
 } from "../src/link-utils.ts";
+import { getJSONFromDataURI } from "../src/uri-utils.ts";
 import { Identity } from "@commontools/identity";
 import { StorageManager } from "@commontools/runner/storage/cache.deno";
 import type { JSONSchema } from "../src/builder/types.ts";
@@ -773,10 +774,8 @@ describe("link-utils", () => {
         baseCell,
       );
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.link["/"][LINK_V1_TAG].path).toEqual([
         "nested",
@@ -814,10 +813,8 @@ describe("link-utils", () => {
 
       const dataURI = createDataCellURI(data, baseCell);
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.items[0]["/"][LINK_V1_TAG].id).toBe(baseId);
       expect(parsed.value.items[1].nested.link["/"][LINK_V1_TAG].id).toBe(
@@ -841,10 +838,8 @@ describe("link-utils", () => {
 
       const dataURI = createDataCellURI({ link: absoluteLink }, baseCell);
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       // Should remain unchanged
       expect(parsed.value.link["/"][LINK_V1_TAG].id).toBe(otherId);
@@ -867,14 +862,34 @@ describe("link-utils", () => {
       // Should not throw even though sharedObject is referenced multiple times
       const dataURI = createDataCellURI(data);
 
-      // Decode and verify
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode and verify using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.first.value).toBe(42);
       expect(parsed.value.second.value).toBe(42);
       expect(parsed.value.nested.third.value).toBe(42);
     });
+
+    it("should handle UTF-8 characters (emojis, special characters)", () => {
+      const data = {
+        emoji: "🚀 Hello World! 🌍",
+        chinese: "你好世界",
+        arabic: "مرحبا بالعالم",
+        special: "Ñoño™©®",
+        mixed: "Test 🎉 with ñ and 中文",
+      };
+
+      // Should not throw with UTF-8 characters
+      const dataURI = createDataCellURI(data);
+
+      // Decode and verify using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
+
+      expect(parsed.value.emoji).toBe("🚀 Hello World! 🌍");
+      expect(parsed.value.chinese).toBe("你好世界");
+      expect(parsed.value.arabic).toBe("مرحبا بالعالم");
+      expect(parsed.value.special).toBe("Ñoño™©®");
+      expect(parsed.value.mixed).toBe("Test 🎉 with ñ and 中文");
+    });
   });
 });

packages/runner/test/schema.test.ts

@@ -1397,7 +1397,7 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        plain.set({
+        plain.setRaw({
           type: "vnode",
           name: "div",
           props: { style: { color: "red" } },
@@ -1417,23 +1417,23 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        styleCell.set({ color: "red" });
+        styleCell.setRaw({ color: "red" });
 
         const innerTextCell = runtime.getCell<{ type: string; value: string }>(
           space,
           "should work for the vdom schema with $ref 4",
           undefined,
           tx,
         );
-        innerTextCell.set({ type: "text", value: "world" });
+        innerTextCell.setRaw({ type: "text", value: "world" });
 
         const childrenArrayCell = runtime.getCell<any[]>(
           space,
           "should work for the vdom schema with $ref 5",
           undefined,
           tx,
         );
-        childrenArrayCell.set([
+        childrenArrayCell.setRaw([
           { type: "text", value: "hello" },
           innerTextCell.getAsLink(),
         ]);
@@ -1451,15 +1451,15 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        withLinks.set({
+        withLinks.setRaw({
           type: "vnode",
           name: "div",
           props: {
             style: styleCell,
           },
           children: [
             { type: "text", value: "single" },
-            childrenArrayCell,
+            childrenArrayCell.getAsLink(),
             "or just text",
           ],
         });
@@ -2441,7 +2441,7 @@ describe("Schema Support", () => {
       expect(links[0].id).not.toBe(links[2].id);
     });
 
-    it("should resolve to array indices when elements are not nested documents", () => {
+    it("should create data URIs for plain objects not marked asCell", () => {
       const schema = {
         type: "object",
         properties: {
@@ -2482,14 +2482,17 @@ describe("Schema Support", () => {
       const itemCells = result.items.map((item: any) => item[toCell]());
       const links = itemCells.map((cell) => cell.getAsNormalizedFullLink());
 
-      // Without nested documents, links should point to array indices
-      expect(links[0].path).toEqual(["items", "0"]);
-      expect(links[1].path).toEqual(["items", "1"]);
-      expect(links[2].path).toEqual(["items", "2"]);
+      // Plain objects now get data URIs with empty paths
+      expect(links[0].id).toMatch(/^data:/);
+      expect(links[1].id).toMatch(/^data:/);
+      expect(links[2].id).toMatch(/^data:/);
+      expect(links[0].path).toEqual([]);
+      expect(links[1].path).toEqual([]);
+      expect(links[2].path).toEqual([]);
 
-      // They should all have the same ID (the parent cell)
-      expect(links[0].id).toBe(links[1].id);
-      expect(links[1].id).toBe(links[2].id);
+      // Each should have unique data URIs
+      expect(links[0].id).not.toBe(links[1].id);
+      expect(links[1].id).not.toBe(links[2].id);
     });
 
     it("should support array splice operations with nested documents", () => {
@@ -2614,15 +2617,19 @@ describe("Schema Support", () => {
       expect(links[0].path).toEqual([]);
       expect(links[2].path).toEqual([]);
 
-      // Plain objects have array index paths
-      expect(links[1].path).toEqual(["items", "1"]);
-      expect(links[3].path).toEqual(["items", "3"]);
+      // Plain objects now also have empty paths (data URIs)
+      expect(links[1].path).toEqual([]);
+      expect(links[3].path).toEqual([]);
 
-      // Nested documents should have unique IDs
+      // Nested documents should have unique IDs (of: format)
       expect(links[0].id).not.toBe(links[2].id);
+      expect(links[0].id).toMatch(/^of:/);
+      expect(links[2].id).toMatch(/^of:/);
 
-      // Plain objects should share the parent cell's ID
-      expect(links[1].id).toBe(links[3].id);
+      // Plain objects should have data URIs
+      expect(links[1].id).toMatch(/^data:/);
+      expect(links[3].id).toMatch(/^data:/);
+      expect(links[1].id).not.toBe(links[3].id); // Different data URIs
     });
 
     it("should preserve nested document references when reordering arrays", () => {