refactor(runner): create cells using memory addresses instead of direct references (#1347)

refactor(runner): create cells using memory addresses instead of direct references

- Updated cell creation logic to use memory addresses (URIs) for referencing cells, rather than direct DocImpl references.
- Refactored related modules (cell, doc, link-resolution, link-utils, query-result-proxy, runner, runtime, scheduler, schema) to support address-based cell instantiation and linking.
- Updated tests to reflect new cell creation and linking approach.

Author: seefeldb

packages/runner/src/cell.ts

@@ -34,6 +34,11 @@ import {
   type URI,
 } from "./sigil-types.ts";
 import { areLinksSame, isLink } from "./link-utils.ts";
+import { type IRuntime } from "./runtime.ts";
+import {
+  type NormalizedFullLink,
+  parseNormalizedFullLinktoLegacyDocCellLink,
+} from "./link-utils.ts";
 
 /**
  * This is the regular Cell interface, generated by DocImpl.asCell().
@@ -90,6 +95,9 @@ import { areLinksSame, isLink } from "./link-utils.ts";
  * @method getAsLegacyCellLink Returns a cell link for the cell (legacy format).
  * @returns {LegacyDocCellLink}
  *
+ * @method getAsNormalizedFullLink Returns a normalized full link for the cell.
+ * @returns {NormalizedFullLink}
+ *
  * @method getAsLink Returns a cell link for the cell (new sigil format).
  * @returns {SigilLink}
  *
@@ -170,6 +178,7 @@ declare module "@commontools/api" {
       log?: ReactivityLog,
     ): QueryResult<DeepKeyLookup<T, Path>>;
     getAsLegacyCellLink(): LegacyDocCellLink;
+    getAsNormalizedFullLink(): NormalizedFullLink;
     getAsLink(
       options?: {
         base?: Cell<any>;
@@ -255,20 +264,22 @@ export interface Stream<T> {
   send(event: T): void;
   sink(callback: (event: T) => Cancel | undefined | void): Cancel;
   getRaw(): any;
+  getAsNormalizedFullLink(): NormalizedFullLink;
   getDoc(): DocImpl<any>;
   schema?: JSONSchema;
   rootSchema?: JSONSchema;
   [isStreamMarker]: true;
 }
 
 export function createCell<T>(
-  doc: DocImpl<any>,
-  path: PropertyKey[] = [],
+  runtime: IRuntime,
+  link: NormalizedFullLink,
   log?: ReactivityLog,
-  schema?: JSONSchema,
-  rootSchema: JSONSchema | undefined = schema,
   noResolve = false,
 ): Cell<T> {
+  const doc = runtime.documentMap.getDocByEntityId(link.space, link.id, true);
+  let { path, schema, rootSchema } = link;
+
   // Resolve the path to check whether it's a stream. We're not logging this right now.
   // The corner case where during it's lifetime this changes from non-stream to stream
   // or vice versa will not be detected.
@@ -283,23 +294,26 @@ export function createCell<T>(
 
   if (isStreamValue(ref.cell.getAtPath(ref.path))) {
     return createStreamCell(
-      ref.cell,
-      ref.path,
+      runtime,
+      {
+        ...link,
+        space: ref.cell.space,
+        id: toURI(ref.cell.entityId),
+        path: ref.path as string[],
+        schema,
+        rootSchema,
+      },
       log,
-      schema,
-      rootSchema,
     ) as unknown as Cell<T>;
   } else {
-    return createRegularCell(doc, path, log, schema, rootSchema);
+    return createRegularCell(runtime, { ...link, schema, rootSchema }, log);
   }
 }
 
 function createStreamCell<T>(
-  doc: DocImpl<any>,
-  path: PropertyKey[],
+  runtime: IRuntime,
+  link: NormalizedFullLink,
   _log?: ReactivityLog,
-  schema?: JSONSchema,
-  rootSchema?: JSONSchema,
 ): Stream<T> {
   const listeners = new Set<(event: T) => Cancel | undefined>();
 
@@ -309,11 +323,7 @@ function createStreamCell<T>(
     // Implementing just the subset of Cell<T> that is needed for streams.
     send: (event: T) => {
       // Use runtime from doc if available
-      if (doc.runtime) {
-        doc.runtime.scheduler.queueEvent({ cell: doc, path }, event);
-      } else {
-        throw new Error("No runtime available for queueEvent");
-      }
+      runtime.scheduler.queueEvent(link, event);
 
       cleanup?.();
       const [cancel, addCancel] = useCancelGroup();
@@ -325,23 +335,25 @@ function createStreamCell<T>(
       listeners.add(callback);
       return () => listeners.delete(callback);
     },
-    getRaw: () => doc.getAtPath(path),
-    getDoc: () => doc,
-    schema,
-    rootSchema,
+    getRaw: () => self.getDoc().getAtPath(link.path),
+    getAsNormalizedFullLink: () => link,
+    getDoc: () => runtime.documentMap.getDocByEntityId(link.space, link.id),
+    schema: link.schema,
+    rootSchema: link.rootSchema,
     [isStreamMarker]: true,
   } satisfies Stream<T>;
 
   return self;
 }
 
 function createRegularCell<T>(
-  doc: DocImpl<any>,
-  path: PropertyKey[],
+  runtime: IRuntime,
+  link: NormalizedFullLink,
   log?: ReactivityLog,
-  schema?: JSONSchema,
-  rootSchema?: JSONSchema,
 ): Cell<T> {
+  const doc = runtime.documentMap.getDocByEntityId(link.space, link.id);
+  const { path, schema, rootSchema } = link;
+
   const self = {
     get: () => validateAndTransform(doc, path, schema, log, rootSchema),
     set: (newValue: Cellify<T>) =>
@@ -453,25 +465,31 @@ function createRegularCell<T>(
         rootSchema,
       );
       return createCell(
-        doc,
-        [...path, valueKey],
+        runtime,
+        {
+          ...link,
+          path: [...path, valueKey.toString()],
+          schema: childSchema,
+        },
         log,
-        childSchema,
-        rootSchema,
       ) as T extends Cell<infer S> ? Cell<S[K & keyof S]> : Cell<T[K]>;
     },
 
     asSchema: (newSchema?: JSONSchema) =>
-      createCell(doc, path, log, newSchema, newSchema),
-    withLog: (newLog: ReactivityLog) =>
-      createCell(doc, path, newLog, schema, rootSchema),
+      createCell(
+        runtime,
+        { ...link, schema: newSchema, rootSchema: newSchema },
+        log,
+      ),
+    withLog: (newLog: ReactivityLog) => createCell(runtime, link, newLog),
     sink: (callback: (value: T) => Cancel | undefined) =>
       subscribeToReferencedDocs(callback, doc, path, schema, rootSchema),
     getAsQueryResult: (subPath: PropertyKey[] = [], newLog?: ReactivityLog) =>
       createQueryResultProxy(doc, [...path, ...subPath], newLog ?? log),
     getAsLegacyCellLink: (): LegacyDocCellLink => {
       return { space: doc.space, cell: doc, path, schema, rootSchema };
     },
+    getAsNormalizedFullLink: () => link,
     getAsLink: (
       options?: {
         base?: Cell<any>;

packages/runner/src/doc.ts

@@ -20,6 +20,7 @@ import { type ReactivityLog } from "./scheduler.ts";
 import { type Cancel } from "./cancel.ts";
 import { Labels, MemorySpace } from "./storage.ts";
 import { arrayEqual } from "./path-utils.ts";
+import { toURI } from "./uri-utils.ts";
 
 /**
  * Lowest level cell implementation.
@@ -274,7 +275,15 @@ export function createDoc<T>(
       log?: ReactivityLog,
       schema?: JSONSchema,
       rootSchema?: JSONSchema,
-    ) => createCell<Q>(self, path || [], log, schema, rootSchema),
+    ) =>
+      createCell(runtime, {
+        space,
+        id: toURI(entityId),
+        path: path?.map(String) ?? [],
+        type: "application/json",
+        schema,
+        rootSchema,
+      }, log),
     send: (newValue: T, log?: ReactivityLog) =>
       self.setAtPath([], newValue, log),
     updates: (

packages/runner/src/link-resolution.ts

@@ -14,6 +14,7 @@ import {
   parseLink,
   parseToLegacyCellLink,
 } from "./link-utils.ts";
+import { toURI } from "./uri-utils.ts";
 
 /**
  * Track visited cell links and memoize results during path resolution
@@ -210,10 +211,13 @@ export function followLinks(
       ? parseToLegacyCellLink(
         target,
         createCell(
-          result.cell,
-          [], // Use empty path to reference the document itself
-          undefined,
-          undefined,
+          result.cell.runtime,
+          {
+            space: result.cell.space,
+            id: toURI(result.cell.entityId),
+            path: [], // Use empty path to reference to document itself
+            type: "application/json",
+          },
           undefined,
           true,
         ),

packages/runner/src/link-utils.ts

@@ -6,7 +6,7 @@ import { type IRuntime } from "./runtime.ts";
 import {
   type JSONCellLink,
   type LegacyAlias,
-  type LegacyDocCellLink as LegacyDocCellLink,
+  type LegacyDocCellLink,
   LINK_V1_TAG,
   type SigilLink,
   type SigilValue,
@@ -219,7 +219,8 @@ export function isLegacyAlias(value: any): value is LegacyAlias {
 export function parseLink(
   value:
     | Cell<any>
-    | DocImpl<any>,
+    | DocImpl<any>
+    | LegacyDocCellLink,
   base?: Cell | NormalizedLink,
 ): NormalizedFullLink;
 export function parseLink(

packages/runner/src/query-result-proxy.ts

@@ -6,6 +6,7 @@ import { type LegacyDocCellLink } from "./sigil-types.ts";
 import { type ReactivityLog } from "./scheduler.ts";
 import { diffAndUpdate, setNestedValue } from "./data-updating.ts";
 import { resolveLinkToValue } from "./link-resolution.ts";
+import { parseLink } from "./link-utils.ts";
 
 // Maximum recursion depth to prevent infinite loops
 const MAX_RECURSION_DEPTH = 100;
@@ -243,10 +244,15 @@ export function createQueryResultProxy<T>(
           ) {
             log?.writes.push({ cell: valueCell, path: [...valuePath, i] });
             if (valueCell.runtime) {
-              valueCell.runtime.scheduler.queueEvent({
-                cell: valueCell,
-                path: [...valuePath, i],
-              }, undefined);
+              valueCell.runtime.scheduler.queueEvent(
+                parseLink(
+                  {
+                    cell: valueCell,
+                    path: [...valuePath, i],
+                  } as LegacyDocCellLink,
+                ),
+                undefined,
+              );
             }
           }
         }

packages/runner/src/runner.ts

@@ -590,7 +590,9 @@ export class Runner implements IRunner {
         }
       };
 
-      addCancel(this.runtime.scheduler.addEventHandler(handler, stream));
+      addCancel(
+        this.runtime.scheduler.addEventHandler(handler, parseLink(streamRef)),
+      );
     } else {
       if (isRecord(inputs) && "$event" in inputs) {
         throw new Error(

packages/runner/src/runtime.ts

@@ -30,6 +30,7 @@ import {
   isLegacyCellLink,
   isLink,
   isNormalizedFullLink,
+  type NormalizedFullLink,
   type NormalizedLink,
   parseLink,
 } from "./link-utils.ts";
@@ -180,8 +181,8 @@ export interface IScheduler {
   unschedule(action: Action): void;
   onConsole(fn: ConsoleHandler): void;
   onError(fn: ErrorHandler): void;
-  queueEvent(eventRef: LegacyDocCellLink, event: any): void;
-  addEventHandler(handler: EventHandler, ref: LegacyDocCellLink): Cancel;
+  queueEvent(eventRef: NormalizedFullLink, event: any): void;
+  addEventHandler(handler: EventHandler, ref: NormalizedFullLink): Cancel;
   runningPromise: Promise<unknown> | undefined;
 }
 

packages/runner/src/scheduler.ts

@@ -16,6 +16,10 @@ import type {
   IScheduler,
   MemorySpace,
 } from "./runtime.ts";
+import {
+  areNormalizedLinksSame,
+  type NormalizedFullLink,
+} from "./link-utils.ts";
 
 // Re-export types that tests expect from scheduler
 export type { ErrorWithContext };
@@ -39,7 +43,7 @@ const MAX_ITERATIONS_PER_RUN = 100;
 export class Scheduler implements IScheduler {
   private pending = new Set<Action>();
   private eventQueue: (() => any)[] = [];
-  private eventHandlers: [LegacyDocCellLink, EventHandler][] = [];
+  private eventHandlers: [NormalizedFullLink, EventHandler][] = [];
   private dirty = new Set<DocImpl<any>>();
   private dependencies = new WeakMap<Action, ReactivityLog>();
   private cancels = new WeakMap<Action, Cancel[]>();
@@ -179,20 +183,16 @@ export class Scheduler implements IScheduler {
     });
   }
 
-  queueEvent(eventRef: LegacyDocCellLink, event: any): void {
-    for (const [ref, handler] of this.eventHandlers) {
-      if (
-        ref.cell === eventRef.cell &&
-        ref.path.length === eventRef.path.length &&
-        ref.path.every((p, i) => p === eventRef.path[i])
-      ) {
+  queueEvent(eventLink: NormalizedFullLink, event: any): void {
+    for (const [link, handler] of this.eventHandlers) {
+      if (areNormalizedLinksSame(link, eventLink)) {
         this.queueExecution();
         this.eventQueue.push(() => handler(event));
       }
     }
   }
 
-  addEventHandler(handler: EventHandler, ref: LegacyDocCellLink): Cancel {
+  addEventHandler(handler: EventHandler, ref: NormalizedFullLink): Cancel {
     this.eventHandlers.push([ref, handler]);
     return () => {
       const index = this.eventHandlers.findIndex(([r, h]) =>