feat(transform): rewrite .map function closures to pass in any closed over cells

packages/api/index.ts

@@ -65,6 +65,10 @@ export interface OpaqueRefMethods<T> {
       array: OpaqueRef<T>,
     ) => Opaque<S>,
   ): OpaqueRef<S[]>;
+  mapWithPattern<S>(
+    op: Recipe,
+    params: Record<string, any>,
+  ): OpaqueRef<S[]>;
 }
 
 // Factory types

packages/runner/src/builder/opaque-ref.ts

@@ -18,7 +18,7 @@ import { hasValueAtPath, setValueAtPath } from "../path-utils.ts";
 import { getTopFrame, recipe } from "./recipe.ts";
 import { createNodeFactory } from "./module.ts";
 
-let mapFactory: NodeFactory<any, any>;
+let mapFactory: NodeFactory<any, any> | undefined;
 
 // A opaque ref factory that creates future cells with optional default values.
 //
@@ -140,6 +140,22 @@ export function opaqueRef<T>(
           ),
         });
       },
+      mapWithPattern: <S>(
+        op: Recipe,
+        params: Record<string, any>,
+      ) => {
+        // Create the factory if it doesn't exist. Doing it here to avoid
+        // circular dependency.
+        mapFactory ||= createNodeFactory({
+          type: "ref",
+          implementation: "map",
+        });
+        return mapFactory({
+          list: proxy,
+          op: op,
+          params: params,
+        });
+      },
       toJSON: () => null, // TODO(seefeld): Merge with Cell and cover doc-less case
       /**
        * We assume the cell is an array and will provide an infinite iterator.

packages/runner/src/builder/types.ts

@@ -127,7 +127,11 @@ declare module "@commontools/api" {
         index: OpaqueRef<number>,
         array: OpaqueRef<T>,
       ) => Opaque<S>,
-    ): OpaqueRef<S[]>;
+    ): Opaque<S[]>;
+    mapWithPattern<S>(
+      op: Recipe,
+      params: Record<string, any>,
+    ): Opaque<S[]>;
     toJSON(): unknown;
     [Symbol.iterator](): Iterator<T>;
     [Symbol.toPrimitive](hint: string): T;

packages/runner/src/builtins/map.ts

@@ -6,16 +6,21 @@ import type { IRuntime } from "../runtime.ts";
 import type { IExtendedStorageTransaction } from "../storage/interface.ts";
 
 /**
- * Implemention of built-in map module. Unlike regular modules, this will be
+ * Implementation of built-in map module. Unlike regular modules, this will be
  * called once at setup and thus sets up its own actions for the scheduler.
  *
+ * This supports both legacy map calls and closure-transformed map calls:
+ * - Legacy mode (params === undefined): Passes { element, index, array } to recipe
+ * - Closure mode (params !== undefined): Passes { element, index, array, params } to recipe
+ *
  * The goal is to keep the output array current without recomputing too much.
  *
  * Approach:
  * 1. Create a doc to store the result.
  * 2. Create a handler to update the result doc when the input doc changes.
  * 3. Create a handler to update the result doc when the op doc changes.
- * 4. For each value in the input doc, create a handler to update the result
+ * 4. Create a handler to update the result doc when the params doc changes (closure mode).
+ * 5. For each value in the input doc, create a handler to update the result
  *    doc when the value changes.
  *
  * TODO: Optimization depends on javascript objects and not lookslike objects.
@@ -24,12 +29,14 @@ import type { IExtendedStorageTransaction } from "../storage/interface.ts";
  *
  * @param list - A doc containing an array of values to map over.
  * @param op - A recipe to apply to each value.
+ * @param params - Optional object containing captured variables from outer scope (closure mode).
  * @returns A doc containing the mapped values.
  */
 export function map(
   inputsCell: Cell<{
     list: any[];
     op: Recipe;
+    params?: Record<string, any>;
   }>,
   sendResult: (tx: IExtendedStorageTransaction, result: any) => void,
   addCancel: AddCancel,
@@ -62,12 +69,13 @@ export function map(
       sendResult(tx, result);
     }
     const resultWithLog = result.withTx(tx);
-    const { list, op } = inputsCell.asSchema(
+    const { list, op, params } = inputsCell.asSchema(
       {
         type: "object",
         properties: {
           list: { type: "array", items: { asCell: true } },
           op: { asCell: true },
+          params: { type: "object" },
         },
         required: ["list", "op"],
         additionalProperties: false,
@@ -111,14 +119,26 @@ export function map(
         undefined,
         tx,
       );
-      runtime.runner.run(
-        tx,
-        opRecipe,
-        {
+      // Determine which mode we're in based on presence of params
+      const recipeInputs = params !== undefined
+        ? {
+          // Closure mode: include params
           element: inputsCell.key("list").key(initializedUpTo),
           index: initializedUpTo,
           array: inputsCell.key("list"),
-        },
+          params: inputsCell.key("params"),
+        }
+        : {
+          // Legacy mode: no params
+          element: inputsCell.key("list").key(initializedUpTo),
+          index: initializedUpTo,
+          array: inputsCell.key("list"),
+        };
+
+      runtime.runner.run(
+        tx,
+        opRecipe,
+        recipeInputs,
         resultCell,
       );
       resultCell.getSourceCell()!.setSourceCell(parentCell);

packages/schema-generator/src/formatters/common-tools-formatter.ts

@@ -34,6 +34,11 @@ export class CommonToolsFormatter implements TypeFormatter {
       return true;
     }
 
+    // Check if this is an Opaque<T> union (T | OpaqueRef<T>)
+    if (this.isOpaqueUnion(type, context.typeChecker)) {
+      return true;
+    }
+
     // Check if this is a wrapper type (Cell/Stream/OpaqueRef) via type structure
     const wrapperInfo = this.getWrapperTypeInfo(type);
     return wrapperInfo !== undefined;
@@ -42,6 +47,26 @@ export class CommonToolsFormatter implements TypeFormatter {
   formatType(type: ts.Type, context: GenerationContext): SchemaDefinition {
     const n = context.typeNode;
 
+    // Check if this is an Opaque<T> union and handle it first
+    // This prevents the UnionFormatter from creating an anyOf
+    const opaqueUnionInfo = this.getOpaqueUnionInfo(type, context.typeChecker);
+    if (opaqueUnionInfo) {
+      // Format the base type T and add asOpaque: true
+      const innerSchema = this.schemaGenerator.formatChildType(
+        opaqueUnionInfo.baseType,
+        context,
+        undefined, // Don't pass typeNode since we're working with the unwrapped type
+      );
+
+      // Handle boolean schemas
+      if (typeof innerSchema === "boolean") {
+        return innerSchema === false
+          ? { asOpaque: true, not: true } as SchemaDefinition // false = "no value is valid"
+          : { asOpaque: true } as SchemaDefinition; // true = "any value is valid"
+      }
+      return { ...innerSchema, asOpaque: true } as SchemaDefinition;
+    }
+
     // Check via typeNode for all wrapper types (handles both direct usage and aliases)
     const resolvedWrapper = n
       ? resolveWrapperNode(n, context.typeChecker)
@@ -191,6 +216,133 @@ export class CommonToolsFormatter implements TypeFormatter {
     return { ...innerSchema, [propertyName]: true };
   }
 
+  /**
+   * Check if a type is an Opaque<T> union (T | OpaqueRef<T>)
+   */
+  private isOpaqueUnion(type: ts.Type, checker: ts.TypeChecker): boolean {
+    return this.getOpaqueUnionInfo(type, checker) !== undefined;
+  }
+
+  /**
+   * Extract information from an Opaque<T> union type.
+   * Opaque<T> is defined as: T | OpaqueRef<T>
+   * This function detects this pattern and returns the base type T.
+   */
+  private getOpaqueUnionInfo(
+    type: ts.Type,
+    checker: ts.TypeChecker,
+  ): { baseType: ts.Type } | undefined {
+    // Must be a union type
+    if (!(type.flags & ts.TypeFlags.Union)) {
+      return undefined;
+    }
+
+    const unionType = type as ts.UnionType;
+    const members = unionType.types;
+
+    // Must have exactly 2 members
+    if (members.length !== 2) {
+      return undefined;
+    }
+
+    // One member should be OpaqueRef<T>, the other should be T
+    let opaqueRefMember: ts.Type | undefined;
+    let baseMember: ts.Type | undefined;
+
+    for (const member of members) {
+      // Check if this member is an OpaqueRef type (it will be an intersection)
+      const isOpaqueRef = this.isOpaqueRefType(member);
+      if (isOpaqueRef) {
+        opaqueRefMember = member;
+      } else {
+        baseMember = member;
+      }
+    }
+
+    // Both members must be present for this to be an Opaque<T> union
+    if (!opaqueRefMember || !baseMember) {
+      return undefined;
+    }
+
+    // Verify that the OpaqueRef's type argument matches the base type
+    // Extract T from OpaqueRef<T>
+    const opaqueRefInnerType = this.extractOpaqueRefTypeArgument(
+      opaqueRefMember,
+      checker,
+    );
+    if (!opaqueRefInnerType) {
+      return undefined;
+    }
+
+    // The inner type of OpaqueRef should match the base member
+    // Use type equality check
+    const innerTypeString = checker.typeToString(opaqueRefInnerType);
+    const baseTypeString = checker.typeToString(baseMember);
+
+    if (innerTypeString !== baseTypeString) {
+      // Not a matching Opaque<T> pattern
+      return undefined;
+    }
+
+    return { baseType: baseMember };
+  }
+
+  /**
+   * Check if a type is an OpaqueRef type (intersection with OpaqueRefMethods)
+   */
+  private isOpaqueRefType(type: ts.Type): boolean {
+    // OpaqueRef types are intersection types
+    if (!(type.flags & ts.TypeFlags.Intersection)) {
+      return false;
+    }
+
+    const intersectionType = type as ts.IntersectionType;
+    for (const constituent of intersectionType.types) {
+      if (constituent.flags & ts.TypeFlags.Object) {
+        const objectType = constituent as ts.ObjectType;
+        if (objectType.objectFlags & ts.ObjectFlags.Reference) {
+          const typeRef = objectType as ts.TypeReference;
+          const name = typeRef.target?.symbol?.name;
+          if (name === "OpaqueRefMethods") {
+            return true;
+          }
+        }
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Extract the type argument T from OpaqueRef<T>
+   */
+  private extractOpaqueRefTypeArgument(
+    type: ts.Type,
+    checker: ts.TypeChecker,
+  ): ts.Type | undefined {
+    if (!(type.flags & ts.TypeFlags.Intersection)) {
+      return undefined;
+    }
+
+    const intersectionType = type as ts.IntersectionType;
+    for (const constituent of intersectionType.types) {
+      if (constituent.flags & ts.TypeFlags.Object) {
+        const objectType = constituent as ts.ObjectType;
+        if (objectType.objectFlags & ts.ObjectFlags.Reference) {
+          const typeRef = objectType as ts.TypeReference;
+          const name = typeRef.target?.symbol?.name;
+          if (name === "OpaqueRefMethods") {
+            // Found OpaqueRefMethods<T>, extract T
+            const typeArgs = checker.getTypeArguments(typeRef);
+            if (typeArgs && typeArgs.length > 0) {
+              return typeArgs[0];
+            }
+          }
+        }
+      }
+    }
+    return undefined;
+  }
+
   /**
    * Get wrapper type information (Cell/Stream/OpaqueRef)
    * Handles both direct references and intersection types (e.g., OpaqueRef<"literal">)

packages/schema-generator/src/interface.ts

@@ -67,4 +67,18 @@ export interface SchemaGenerator {
     checker: ts.TypeChecker,
     typeNode?: ts.TypeNode,
   ): SchemaDefinition;
+
+  /**
+   * Generate schema from a synthetic TypeNode that doesn't resolve to a proper Type.
+   * Used by transformers that create synthetic type structures programmatically.
+   *
+   * @param typeNode - Synthetic TypeNode to analyze
+   * @param checker - TypeScript type checker
+   * @param typeRegistry - Optional WeakMap of Node → Type for registered synthetic nodes
+   */
+  generateSchemaFromSyntheticTypeNode(
+    typeNode: ts.TypeNode,
+    checker: ts.TypeChecker,
+    typeRegistry?: WeakMap<ts.Node, ts.Type>,
+  ): SchemaDefinition;
 }

packages/schema-generator/src/plugin.ts

@@ -2,18 +2,32 @@ import ts from "typescript";
 import { SchemaGenerator } from "./schema-generator.ts";
 
 /**
- * Plugin function that matches the existing typeToJsonSchema signature
- * This allows our new system to be a drop-in replacement
+ * Plugin function that creates a schema transformer with access to both
+ * Type-based and synthetic TypeNode-based schema generation
  */
-export function createSchemaTransformerV2(): (
-  type: ts.Type,
-  checker: ts.TypeChecker,
-  typeArg?: ts.TypeNode,
-) => any {
+export function createSchemaTransformerV2() {
   const generator = new SchemaGenerator();
 
-  return (type: ts.Type, checker: ts.TypeChecker, typeArg?: ts.TypeNode) => {
-    return generator.generateSchema(type, checker, typeArg);
+  return {
+    generateSchema(
+      type: ts.Type,
+      checker: ts.TypeChecker,
+      typeArg?: ts.TypeNode,
+    ) {
+      return generator.generateSchema(type, checker, typeArg);
+    },
+
+    generateSchemaFromSyntheticTypeNode(
+      typeNode: ts.TypeNode,
+      checker: ts.TypeChecker,
+      typeRegistry?: WeakMap<ts.Node, ts.Type>,
+    ) {
+      return generator.generateSchemaFromSyntheticTypeNode(
+        typeNode,
+        checker,
+        typeRegistry,
+      );
+    },
   };
 }