Speed up recipe sync (#610)

* Do not re-create merkle ref every time we try to load a recipe

We already know the ID!

* Refactor addRecipe to registerNewRecipe and registerRecipe

Be clear about when we do and do not wish to generate a reference

* Export registerNewRecipe

Author: bfollington

charm/src/iterate.ts

@@ -1,4 +1,4 @@
-import { addRecipe, Cell, EntityId } from "@commontools/runner";
+import { registerNewRecipe, Cell, EntityId } from "@commontools/runner";
 import { LLMClient } from "@commontools/llm-client";
 import { createJsonSchema, JSONSchema } from "@commontools/builder";
 
@@ -178,7 +178,7 @@ export async function compileRecipe(
     return;
   }
   const parentsIds = parents?.map((id) => id.toString());
-  addRecipe(recipe, recipeSrc, spec, parentsIds);
+  registerNewRecipe(recipe, recipeSrc, spec, parentsIds);
   return recipe;
 }
 

runner/src/index.ts

@@ -33,7 +33,8 @@ export {
   getEntityId,
 } from "./doc-map.ts";
 export {
-  addRecipe,
+  registerRecipe,
+  registerNewRecipe,
   allRecipesByName,
   getRecipe,
   getRecipeId,

runner/src/recipe-map.ts

@@ -9,17 +9,30 @@ const srcById = new Map<string, string>();
 const specById = new Map<string, string>();
 const parentsById = new Map<string, string[]>();
 
-export function addRecipe(
-  recipe: Recipe | Module,
+export function registerNewRecipe(
+  recipe: Recipe,
   src?: string,
   spec?: string,
-  parents?: string[],
+  parents?: string[]
 ): string {
   if (idByRecipe.has(recipe)) return idByRecipe.get(recipe)!;
 
   const id = src
     ? createRef(src, "recipe source").toString()
     : createRef(recipe, "recipe").toString();
+
+  return registerRecipe(id, recipe, src, spec, parents);
+}
+
+export function registerRecipe(
+  id: string,
+  recipe: Recipe,
+  src?: string,
+  spec?: string,
+  parents?: string[]
+): string {
+  if (idByRecipe.has(recipe)) return idByRecipe.get(recipe)!;
+
   recipeById.set(id, recipe);
   idByRecipe.set(recipe, id);
 

runner/src/recipe-sync.ts

@@ -1,5 +1,5 @@
 import {
-  addRecipe,
+  registerRecipe,
   getRecipe,
   getRecipeName,
   getRecipeParents,
@@ -49,11 +49,8 @@ export async function syncRecipeBlobby(id: string) {
   const { recipe, errors } = await buildRecipe(src);
   if (errors) throw new Error(errors);
 
-  const recipeId = addRecipe(recipe!, src, spec, parents);
-  if (id !== recipeId) {
-    throw new Error(`Recipe ID mismatch: ${id} !== ${recipeId}`);
-  }
-  recipesKnownToStorage.add(recipeId);
+  registerRecipe(id, recipe!, src, spec, parents);
+  recipesKnownToStorage.add(id);
 }
 
 function saveRecipe(

runner/src/runner.ts

@@ -38,7 +38,7 @@ import {
 import { getModuleByRef } from "./module.ts";
 import { type AddCancel, type Cancel, useCancelGroup } from "./cancel.ts";
 import "./builtins/index.ts";
-import { addRecipe, getRecipe, getRecipeId } from "./recipe-map.ts";
+import { registerRecipe, getRecipe, getRecipeId, registerNewRecipe } from "./recipe-map.ts";
 import { isCell } from "./cell.ts";
 import { isQueryResultForDereferencing } from "./query-result-proxy.ts";
 import { getDocLinkOrThrow } from "./query-result-proxy.ts";
@@ -138,7 +138,7 @@ export function run<T, R = any>(
     recipe = recipeOrModule as Recipe;
   }
 
-  recipeId ??= addRecipe(recipe);
+  recipeId ??= registerNewRecipe(recipe);
 
   if (cancels.has(resultCell)) {
     // If it's already running and no new recipe or argument are given,
@@ -175,10 +175,10 @@ export function run<T, R = any>(
     const ref = isDocLink(argument)
       ? argument
       : isCell(argument)
-      ? argument.getAsDocLink()
-      : isQueryResultForDereferencing(argument)
-      ? getDocLinkOrThrow(argument)
-      : ({ cell: argument, path: [] } satisfies DocLink);
+        ? argument.getAsDocLink()
+        : isQueryResultForDereferencing(argument)
+          ? getDocLinkOrThrow(argument)
+          : ({ cell: argument, path: [] } satisfies DocLink);
 
     // Get value, but just to get the keys. Throw if it isn't an object.
     const value = ref.cell.getAsQueryResult(ref.path);
@@ -464,11 +464,11 @@ function instantiateJavaScriptNode(
           resultRecipe,
           undefined,
           resultCell ??
-            getDoc(
-              undefined,
-              { resultFor: { inputs, outputs, fn: fn.toString() } },
-              processCell.space,
-            ),
+          getDoc(
+            undefined,
+            { resultFor: { inputs, outputs, fn: fn.toString() } },
+            processCell.space,
+          ),
         );
         addCancel(cancels.get(resultCell));