charm-cleanup-again (#608)

* add some sleep for ci in github
* try to output logs in integration
* fix logging in integartions
* break single bdd into steps
* trying to shortcut integration tests
* don't throw execption on non-iframe charms
* remove node type that we have now via deno.json types
* remove old code / update comments
* types
* more cleanup
* stop the loading stuff as soon as we show one

Author: anotherjesse

charm/src/charm.ts

@@ -161,14 +161,17 @@ export class CharmManager {
     return this.pinnedCharms;
   }
 
+  // FIXME(ja): this says it returns a list of charm, but it isn't! you will
+  // have to call .get() to get the actual charm (this is missing the schema)
+  // how can we fix the type here?
   getCharms(): Cell<Cell<Charm>[]> {
     // Start syncing if not already syncing. Will trigger a change to the list
     // once loaded.
     storage.syncCell(this.charmsDoc);
     return this.charms;
   }
 
-  async add(newCharms: Cell<Charm>[]) {
+  private async add(newCharms: Cell<Charm>[]) {
     await storage.syncCell(this.charmsDoc);
     await idle();
 
@@ -181,6 +184,8 @@ export class CharmManager {
     await idle();
   }
 
+  // FIXME(ja): if we are already running the charm, can we just return it?
+  // if a charm has sideeffects we might multiple versions...
   async get<T = Charm>(
     id: string | Cell<Charm>,
     runIt: boolean = true,
@@ -291,46 +296,6 @@ export class CharmManager {
   ): Promise<Cell<Charm>> {
     await idle();
 
-    // Fill in missing parameters from other charms. It's a simple match on
-    // hashtags: For each top-level argument prop that has a hashtag in the
-    // description, look for a charm that has a top-level output prop with the
-    // same hashtag in the description, or has the hashtag in its own description.
-    // If there is a match, assign the first one to the input property.
-
-    // TODO(seefeld,ben): This should be in spellcaster.
-    /*
-    if (
-      !isDoc(inputs) && // Adding to a cell input is not supported yet
-      !isDocLink(inputs) && // Neither for cell reference
-      recipe.argumentSchema &&
-      (recipe.argumentSchema as any).type === "object"
-    ) {
-      const properties = (recipe.argumentSchema as any).properties;
-      const inputProperties =
-        typeof inputs === "object" && inputs !== null ? Object.keys(inputs) : [];
-      for (const key in properties) {
-        if (!(key in inputProperties) && properties[key].description?.includes("#")) {
-          const hashtag = properties[key].description.match(/#(\w+)/)?.[1];
-          if (hashtag) {
-            this.charms.get().forEach((charm) => {
-              const type = charm.getAsDocLink().cell?.sourceCell?.get()?.[TYPE];
-              const recipe = getRecipe(type);
-              const charmProperties = (recipe?.resultSchema as any)?.properties as any;
-              const matchingProperty = Object.keys(charmProperties ?? {}).find((property) =>
-                charmProperties[property].description?.includes(`#${hashtag}`),
-              );
-              if (matchingProperty) {
-                inputs = {
-                  ...inputs,
-                  [key]: { $alias: { cell: charm.getAsDocLink().cell, path: [matchingProperty] } },
-                };
-              }
-            });
-          }
-        }
-      }
-    }*/
-
     const syncAllMentionedCells = (
       value: any,
       promises: any[] = [],
@@ -360,7 +325,7 @@ export class CharmManager {
   }
 
   // FIXME(JA): this really really really needs to be revisited
-  async syncRecipe(charm: Cell<Charm>): Promise<string | undefined> {
+  async syncRecipe(charm: Cell<Charm>): Promise<string> {
     const recipeId = charm.getSourceCell()?.get()?.[TYPE];
     if (!recipeId) throw new Error("charm missing recipe ID");
 
@@ -372,7 +337,7 @@ export class CharmManager {
   }
 
   async syncRecipeCells(recipeId: string) {
-    // NOTE(ja): I don't think this actually syncs the recipe
+    // NOTE(ja): this doesn't sync recipe to storage
     if (recipeId) await storage.syncCellById(this.space, { "/": recipeId });
   }
 

charm/src/iframe/index.ts

@@ -41,54 +41,28 @@ export const buildFullRecipe = (iframe: IFrameRecipe) => {
   `;
 };
 
-function parseIframeRecipe(source: string): IFrameRecipe | undefined {
+function parseIframeRecipe(source: string): IFrameRecipe {
   // Extract content between IFRAME-V0 comments
   const match = source.match(
     /\/\* IFRAME-V0 \*\/([\s\S]*?)\/\* IFRAME-V0 \*\//,
   );
-  if (!match) {
-    console.warn("no IFRAME-V0 section in source");
-    return undefined;
+
+  if (!match || !match[1]) {
+    throw new Error("Could not find IFRAME-V0 recipe content in source");
   }
 
   return JSON.parse(match[1]) as IFrameRecipe;
 }
 
 export const getIframeRecipe = (charm: Cell<Charm>) => {
-  const recipeId = charm.getSourceCell(processSchema)?.get()?.[TYPE];
-  if (!recipeId) {
-    console.error("FIXME, no recipeId, what should we do?");
-    return {};
-  }
-
-  const recipe = getRecipe(recipeId);
-  if (!recipe) {
-    console.error("FIXME, no recipe, what should we do?");
-    return {};
-  }
-  const src = getRecipeSrc(recipeId);
-  if (!src) {
-    console.error("FIXME, no src, what should we do?");
-    return {};
-  }
-
+  const { src, recipeId } = getRecipeFrom(charm);
   return { recipeId, iframe: parseIframeRecipe(src) };
 };
 
 export const getRecipeFrom = (charm: Cell<Charm>) => {
   const recipeId = charm.getSourceCell(processSchema)?.get()?.[TYPE];
-  if (!recipeId) {
-    throw new Error("No recipeId found");
-  }
-
-  const recipe = getRecipe(recipeId);
-  if (!recipe) {
-    throw new Error("No recipe found for recipeId");
-  }
-  const src = getRecipeSrc(recipeId);
-  if (!src) {
-    throw new Error("No source found for recipeId");
-  }
+  const recipe = getRecipe(recipeId)!;
+  const src = getRecipeSrc(recipeId)!;
 
   return { recipeId, recipe, src };
 };

charm/src/iframe/recipe.ts

@@ -4,8 +4,7 @@ export {
   castNewRecipe,
   compileAndRunRecipe,
   compileRecipe,
-  extend,
+  generateNewRecipeVersion,
   iterate,
-  saveNewRecipeVersion,
 } from "./iterate.ts";
 export { getIframeRecipe, type IFrameRecipe } from "./iframe/recipe.ts";

charm/src/index.ts

@@ -7,6 +7,7 @@ import { Charm, CharmManager } from "./charm.ts";
 import { buildFullRecipe, getIframeRecipe } from "./iframe/recipe.ts";
 import { buildPrompt, RESPONSE_PREFILL } from "./iframe/prompt.ts";
 import { injectUserCode } from "./iframe/static.ts";
+import { isCell } from "../../runner/src/cell.ts";
 
 const llm = new LLMClient(LLMClient.DEFAULT_URL);
 
@@ -32,31 +33,25 @@ const genSrc = async ({
     response = RESPONSE_PREFILL + response;
   }
 
-  const source = injectUserCode(response.split(RESPONSE_PREFILL)[1].split("\n```")[0]);
+  const source = injectUserCode(
+    response.split(RESPONSE_PREFILL)[1].split("\n```")[0],
+  );
   return source;
 };
 
 export async function iterate(
   charmManager: CharmManager,
-  charm: Cell<Charm> | null,
-  value: string,
+  charm: Cell<Charm>,
+  spec: string,
   shiftKey: boolean,
   model?: string,
-): Promise<EntityId | undefined> {
-  if (!charm) {
-    console.error("FIXME, no charm, what should we do?");
-    return;
-  }
-
+): Promise<Cell<Charm>> {
   const { iframe } = getIframeRecipe(charm);
   if (!iframe) {
-    console.error(
-      "Cannot iterate on a non-iframe. Must extend instead.",
-    );
-    return;
+    throw new Error("Cannot iterate on a non-iframe. Must extend instead.");
   }
 
-  const newSpec = shiftKey ? iframe.spec + "\n" + value : value;
+  const newSpec = shiftKey ? iframe.spec + "\n" + spec : spec;
 
   const newIFrameSrc = await genSrc({
     src: iframe.src,
@@ -66,21 +61,7 @@ export async function iterate(
     model: model,
   });
 
-  return saveNewRecipeVersion(charmManager, charm, newIFrameSrc, newSpec);
-}
-
-export async function extend(
-  charmManager: CharmManager,
-  charm: Cell<Charm> | null,
-  value: string,
-  model?: string,
-): Promise<EntityId | undefined> {
-  if (!charm) {
-    console.error("FIXME, no charm, what should we do?");
-    return;
-  }
-
-  return await castRecipeOnCell(charmManager, charm, value);
+  return generateNewRecipeVersion(charmManager, charm, newIFrameSrc, newSpec);
 }
 
 export function extractTitle(src: string, defaultTitle: string): string {
@@ -89,7 +70,7 @@ export function extractTitle(src: string, defaultTitle: string): string {
   return htmlTitleMatch || jsTitleMatch || defaultTitle;
 }
 
-export const saveNewRecipeVersion = async (
+export const generateNewRecipeVersion = (
   charmManager: CharmManager,
   charm: Cell<Charm>,
   newIFrameSrc: string,
@@ -98,19 +79,18 @@ export const saveNewRecipeVersion = async (
   const { recipeId, iframe } = getIframeRecipe(charm);
 
   if (!recipeId || !iframe) {
-    console.error("FIXME, no recipeId or iframe, what should we do?");
-    return;
+    throw new Error("FIXME, no recipeId or iframe, what should we do?");
   }
 
-  const name = extractTitle(newIFrameSrc, '<unknown>');
+  const name = extractTitle(newIFrameSrc, "<unknown>");
   const newRecipeSrc = buildFullRecipe({
     ...iframe,
     src: newIFrameSrc,
     spec: newSpec,
     name,
   });
 
-  return await compileAndRunRecipe(
+  return compileAndRunRecipe(
     charmManager,
     newRecipeSrc,
     newSpec,
@@ -119,38 +99,17 @@ export const saveNewRecipeVersion = async (
   );
 };
 
-export async function castRecipeOnCell(
-  charmManager: CharmManager,
-  cell: Cell<any>,
-  newSpec: string,
-): Promise<EntityId | undefined> {
-  const schema = { ...cell.schema, description: newSpec };
-  console.log("schema", schema);
-
-  const newIFrameSrc = await genSrc({ newSpec, schema });
-  const name = extractTitle(newIFrameSrc, '<unknown>');
-  const newRecipeSrc = buildFullRecipe({
-    src: newIFrameSrc,
-    spec: newSpec,
-    argumentSchema: schema,
-    resultSchema: {},
-    name,
-  });
-
-  return await compileAndRunRecipe(charmManager, newRecipeSrc, newSpec, cell);
-}
-
 export async function castNewRecipe(
   charmManager: CharmManager,
   data: any,
   newSpec: string,
-): Promise<EntityId | undefined> {
-  const schema = createJsonSchema({}, data);
+): Promise<Cell<Charm>> {
+  const schema = isCell(data) ? { ...data.schema } : createJsonSchema({}, data);
   schema.description = newSpec;
   console.log("schema", schema);
 
   const newIFrameSrc = await genSrc({ newSpec, schema });
-  const name = extractTitle(newIFrameSrc, '<unknown>');
+  const name = extractTitle(newIFrameSrc, "<unknown>");
   const newRecipeSrc = buildFullRecipe({
     src: newIFrameSrc,
     spec: newSpec,
@@ -159,7 +118,7 @@ export async function castNewRecipe(
     name,
   });
 
-  return await compileAndRunRecipe(charmManager, newRecipeSrc, newSpec, data);
+  return compileAndRunRecipe(charmManager, newRecipeSrc, newSpec, data);
 }
 
 export async function compileRecipe(
@@ -169,13 +128,11 @@ export async function compileRecipe(
 ) {
   const { exports, errors } = await tsToExports(recipeSrc);
   if (errors) {
-    console.error("Compilation errors in recipe:", errors);
-    return;
+    throw new Error("Compilation errors in recipe");
   }
   const recipe = exports.default;
   if (!recipe) {
-    console.error("No default recipe found in the compiled exports.");
-    return;
+    throw new Error("No default recipe found in the compiled exports.");
   }
   const parentsIds = parents?.map((id) => id.toString());
   registerNewRecipe(recipe, recipeSrc, spec, parentsIds);
@@ -188,15 +145,11 @@ export async function compileAndRunRecipe(
   spec: string,
   runOptions: any,
   parents?: string[],
-): Promise<EntityId | undefined> {
+): Promise<Cell<Charm>> {
   const recipe = await compileRecipe(recipeSrc, spec, parents);
   if (!recipe) {
-    return;
+    throw new Error("Failed to compile recipe");
   }
 
-  const newCharm = await charmManager.runPersistent(recipe, runOptions);
-  await charmManager.add([newCharm]);
-  await charmManager.syncRecipe(newCharm);
-
-  return newCharm.entityId;
+  return charmManager.runPersistent(recipe, runOptions);
 }

charm/src/iterate.ts

@@ -58,8 +58,8 @@ async function main() {
   );
 
   // let's add the cell to the charmManager
-  await charmManager.add([cell]);
-  log(charmManager, "charmmanager after adding cell");
+  // await charmManager.add([cell]);
+  // log(charmManager, "charmmanager after adding cell");
 }
 
 main();

cli/charm_demo.ts

@@ -72,8 +72,6 @@ async function main() {
       const recipeSrc = await Deno.readTextFile(recipeFile);
       const recipe = await compileRecipe(recipeSrc, "recipe", []);
       const charm = await manager.runPersistent(recipe, undefined, cause);
-      await manager.syncRecipe(charm);
-      manager.add([charm]);
       const charmWithSchema = (await manager.get(charm))!;
       charmWithSchema.sink((value) => {
         console.log("running charm:", getEntityId(charm), value);