feat(runner): create data URIs for array element objects not marked asCell (#1920)

* feat(runner): create data URIs for array element objects not marked asCell

When getting array elements as cells, plain objects (not marked with asCell
or asStream) now receive immutable data URIs instead of array index paths.
This provides stable references while preventing mutation loops when
reassigning array elements.

Changes:
- Array element objects get data URIs with empty paths
- Nested documents (marked asCell) continue to use of: URIs as before
- Only create array index references when element is already a link
- Update tests to reflect new data URI behavior for plain objects
- Change createDataCellURI return type from string to URI for consistency

* make anyOf.every vs .same consistent with the rest (though actually currently incorrectly so)

* fix setRaw not automatically converting the cell

* fix(runner): use encodeURIComponent for UTF-8 safe data URIs

Author: seefeldb

packages/runner/src/link-utils.ts

@@ -511,7 +511,7 @@ export function findAndInlineDataURILinks(value: any): any {
 export function createDataCellURI(
   data: any,
   base?: Cell | NormalizedLink,
-): string {
+): URI {
   const baseId = isCell(base) ? base.getAsNormalizedFullLink().id : base?.id;
 
   function traverseAndAddBaseIdToRelativeLinks(
@@ -549,8 +549,8 @@ export function createDataCellURI(
   const json = JSON.stringify({
     value: traverseAndAddBaseIdToRelativeLinks(data, new Set()),
   });
-  const base64 = btoa(json);
-  return `data:application/json;charset=utf-8;base64,${base64}`;
+  // Use encodeURIComponent for UTF-8 safe encoding (matches runtime.ts pattern)
+  return `data:application/json,${encodeURIComponent(json)}` as URI;
 }
 
 /**

packages/runner/src/schema.ts

@@ -8,7 +8,11 @@ import { createCell, isCell, isStream } from "./cell.ts";
 import { readMaybeLink, resolveLink } from "./link-resolution.ts";
 import { type IExtendedStorageTransaction } from "./storage/interface.ts";
 import { type IRuntime } from "./runtime.ts";
-import { type NormalizedFullLink } from "./link-utils.ts";
+import {
+  createDataCellURI,
+  type NormalizedFullLink,
+  parseLink,
+} from "./link-utils.ts";
 import {
   createQueryResultProxy,
   isQueryResultForDereferencing,
@@ -737,20 +741,6 @@ export function validateAndTransform(
 
     // Now process elements after adding the array to seen
     for (let i = 0; i < value.length; i++) {
-      // If the element on the array is a link, we follow that link so the
-      // returned object is the current item at that location (otherwise the
-      // link would refer to "Nth element"). This is important when turning
-      // returned objects back into cells: We want to then refer to the actual
-      // object by default, not the array location.
-      //
-      // This makes
-      // ```ts
-      // const array = [...cell.get()];
-      // array.splice(index, 1);
-      // cell.set(array);
-      // ```
-      // work as expected.
-      // Handle boolean items values for element schema
       let elementSchema: JSONSchema;
       if (resolvedSchema.items === true) {
         // items: true means allow any item type
@@ -772,13 +762,56 @@ export function validateAndTransform(
         path: [...link.path, String(i)],
         schema: elementSchema,
       };
-      const maybeLink = readMaybeLink(tx ?? runtime.edit(), elementLink);
+
+      // If the element on the array is a link, we follow that link so the
+      // returned object is the current item at that location (otherwise the
+      // link would refer to "Nth element"). This is important when turning
+      // returned objects back into cells: We want to then refer to the actual
+      // object by default, not the array location.
+      //
+      // If the element is an object, but not a link, we create an immutable
+      // cell to hold the object, except when it is requested as Cell. While
+      // this means updates aren't propagated, it seems like the right trade-off
+      // for stability of links and the ability to mutate them without creating
+      // loops (see below).
+      //
+      // This makes
+      // ```ts
+      // const array = [...cell.get()];
+      // array.splice(index, 1);
+      // cell.set(array);
+      // ```
+      // work as expected. Handle boolean items values for element schema
+      const maybeLink = parseLink(value[i], link);
       if (maybeLink) {
         elementLink = {
           ...maybeLink,
           schema: elementLink.schema,
           rootSchema: elementLink.rootSchema,
         };
+      } else if (
+        isRecord(value[i]) &&
+        // TODO(seefeld): Should factor this out, but we should just fully
+        // normalize schemas, etc.
+        !(isObject(elementSchema) &&
+          (elementSchema.asCell || elementSchema.asStream ||
+            (Array.isArray(elementSchema?.anyOf) &&
+              elementSchema.anyOf.every((option) =>
+                option.asCell || option.asStream
+              )) ||
+            (Array.isArray(elementSchema?.oneOf) &&
+              elementSchema.oneOf.every((option) =>
+                option.asCell || option.asStream
+              ))))
+      ) {
+        elementLink = {
+          id: createDataCellURI(value[i], link),
+          path: [],
+          schema: elementSchema,
+          rootSchema: elementLink.rootSchema,
+          space: link.space,
+          type: "application/json",
+        } satisfies NormalizedFullLink;
       }
 
       result[i] = validateAndTransform(

packages/runner/src/uri-utils.ts

@@ -90,7 +90,19 @@ export function getJSONFromDataURI(uri: URI | string): any {
   // Check if data is base64 encoded
   const isBase64 = headerParts.some((part) => part === "base64");
 
-  const decodedData = isBase64 ? atob(data) : decodeURIComponent(data);
+  let decodedData: string;
+  if (isBase64) {
+    // Use TextDecoder to properly decode UTF-8 bytes from base64
+    const binaryString = atob(data);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+    const decoder = new TextDecoder();
+    decodedData = decoder.decode(bytes);
+  } else {
+    decodedData = decodeURIComponent(data);
+  }
 
   return decodedData.length > 0 ? JSON.parse(decodedData) : undefined;
 }

packages/runner/test/link-utils.test.ts

@@ -13,6 +13,7 @@ import {
   parseLinkOrThrow,
   sanitizeSchemaForLinks,
 } from "../src/link-utils.ts";
+import { getJSONFromDataURI } from "../src/uri-utils.ts";
 import { Identity } from "@commontools/identity";
 import { StorageManager } from "@commontools/runner/storage/cache.deno";
 import type { JSONSchema } from "../src/builder/types.ts";
@@ -773,10 +774,8 @@ describe("link-utils", () => {
         baseCell,
       );
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.link["/"][LINK_V1_TAG].path).toEqual([
         "nested",
@@ -814,10 +813,8 @@ describe("link-utils", () => {
 
       const dataURI = createDataCellURI(data, baseCell);
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.items[0]["/"][LINK_V1_TAG].id).toBe(baseId);
       expect(parsed.value.items[1].nested.link["/"][LINK_V1_TAG].id).toBe(
@@ -841,10 +838,8 @@ describe("link-utils", () => {
 
       const dataURI = createDataCellURI({ link: absoluteLink }, baseCell);
 
-      // Decode the data URI
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode the data URI using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       // Should remain unchanged
       expect(parsed.value.link["/"][LINK_V1_TAG].id).toBe(otherId);
@@ -867,14 +862,34 @@ describe("link-utils", () => {
       // Should not throw even though sharedObject is referenced multiple times
       const dataURI = createDataCellURI(data);
 
-      // Decode and verify
-      const base64 = dataURI.split(",")[1];
-      const json = atob(base64);
-      const parsed = JSON.parse(json);
+      // Decode and verify using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
 
       expect(parsed.value.first.value).toBe(42);
       expect(parsed.value.second.value).toBe(42);
       expect(parsed.value.nested.third.value).toBe(42);
     });
+
+    it("should handle UTF-8 characters (emojis, special characters)", () => {
+      const data = {
+        emoji: "🚀 Hello World! 🌍",
+        chinese: "你好世界",
+        arabic: "مرحبا بالعالم",
+        special: "Ñoño™©®",
+        mixed: "Test 🎉 with ñ and 中文",
+      };
+
+      // Should not throw with UTF-8 characters
+      const dataURI = createDataCellURI(data);
+
+      // Decode and verify using getJSONFromDataURI
+      const parsed = getJSONFromDataURI(dataURI);
+
+      expect(parsed.value.emoji).toBe("🚀 Hello World! 🌍");
+      expect(parsed.value.chinese).toBe("你好世界");
+      expect(parsed.value.arabic).toBe("مرحبا بالعالم");
+      expect(parsed.value.special).toBe("Ñoño™©®");
+      expect(parsed.value.mixed).toBe("Test 🎉 with ñ and 中文");
+    });
   });
 });

packages/runner/test/schema.test.ts

@@ -1397,7 +1397,7 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        plain.set({
+        plain.setRaw({
           type: "vnode",
           name: "div",
           props: { style: { color: "red" } },
@@ -1417,23 +1417,23 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        styleCell.set({ color: "red" });
+        styleCell.setRaw({ color: "red" });
 
         const innerTextCell = runtime.getCell<{ type: string; value: string }>(
           space,
           "should work for the vdom schema with $ref 4",
           undefined,
           tx,
         );
-        innerTextCell.set({ type: "text", value: "world" });
+        innerTextCell.setRaw({ type: "text", value: "world" });
 
         const childrenArrayCell = runtime.getCell<any[]>(
           space,
           "should work for the vdom schema with $ref 5",
           undefined,
           tx,
         );
-        childrenArrayCell.set([
+        childrenArrayCell.setRaw([
           { type: "text", value: "hello" },
           innerTextCell.getAsLink(),
         ]);
@@ -1451,15 +1451,15 @@ describe("Schema Support", () => {
           undefined,
           tx,
         );
-        withLinks.set({
+        withLinks.setRaw({
           type: "vnode",
           name: "div",
           props: {
             style: styleCell,
           },
           children: [
             { type: "text", value: "single" },
-            childrenArrayCell,
+            childrenArrayCell.getAsLink(),
             "or just text",
           ],
         });
@@ -2441,7 +2441,7 @@ describe("Schema Support", () => {
       expect(links[0].id).not.toBe(links[2].id);
     });
 
-    it("should resolve to array indices when elements are not nested documents", () => {
+    it("should create data URIs for plain objects not marked asCell", () => {
       const schema = {
         type: "object",
         properties: {
@@ -2482,14 +2482,17 @@ describe("Schema Support", () => {
       const itemCells = result.items.map((item: any) => item[toCell]());
       const links = itemCells.map((cell) => cell.getAsNormalizedFullLink());
 
-      // Without nested documents, links should point to array indices
-      expect(links[0].path).toEqual(["items", "0"]);
-      expect(links[1].path).toEqual(["items", "1"]);
-      expect(links[2].path).toEqual(["items", "2"]);
+      // Plain objects now get data URIs with empty paths
+      expect(links[0].id).toMatch(/^data:/);
+      expect(links[1].id).toMatch(/^data:/);
+      expect(links[2].id).toMatch(/^data:/);
+      expect(links[0].path).toEqual([]);
+      expect(links[1].path).toEqual([]);
+      expect(links[2].path).toEqual([]);
 
-      // They should all have the same ID (the parent cell)
-      expect(links[0].id).toBe(links[1].id);
-      expect(links[1].id).toBe(links[2].id);
+      // Each should have unique data URIs
+      expect(links[0].id).not.toBe(links[1].id);
+      expect(links[1].id).not.toBe(links[2].id);
     });
 
     it("should support array splice operations with nested documents", () => {
@@ -2614,15 +2617,19 @@ describe("Schema Support", () => {
       expect(links[0].path).toEqual([]);
       expect(links[2].path).toEqual([]);
 
-      // Plain objects have array index paths
-      expect(links[1].path).toEqual(["items", "1"]);
-      expect(links[3].path).toEqual(["items", "3"]);
+      // Plain objects now also have empty paths (data URIs)
+      expect(links[1].path).toEqual([]);
+      expect(links[3].path).toEqual([]);
 
-      // Nested documents should have unique IDs
+      // Nested documents should have unique IDs (of: format)
       expect(links[0].id).not.toBe(links[2].id);
+      expect(links[0].id).toMatch(/^of:/);
+      expect(links[2].id).toMatch(/^of:/);
 
-      // Plain objects should share the parent cell's ID
-      expect(links[1].id).toBe(links[3].id);
+      // Plain objects should have data URIs
+      expect(links[1].id).toMatch(/^data:/);
+      expect(links[3].id).toMatch(/^data:/);
+      expect(links[1].id).not.toBe(links[3].id); // Different data URIs
     });
 
     it("should preserve nested document references when reordering arrays", () => {