Prompt engineering grab bag (#1114)

* XML tags for prompts

* Present steps as XML

* Rename `useDoc` -> `useReactiveCell`

* Tweak API to be more 'intuitive' for LLM

This is a guess, llm has been broken into two

* Improve robustness of JSON parsing

* Fix suggested charms behaviour

* Update LLM cache

* Backwards compatible, expose useDoc alias

* Add missing types

* Stringify search error

Author: bfollington

charm/src/iframe/prompt.ts

@@ -59,7 +59,9 @@ ${newSpec}
         ? `
 
 by following the following steps:
-${steps.map((step, index) => `${index + 1}. ${step}`).join("\n")}`
+<steps>
+${steps.map((step, index) => `<step>${step}</step>`).join("\n")}
+</steps>`
         : ""
     }`,
   });

charm/src/iframe/static.ts

@@ -1271,7 +1271,7 @@ export class CharmManager {
     const recipeId = sourceCell.get()?.[TYPE];
     if (!recipeId) throw new Error("charm missing recipe ID");
 
-    await this.syncRecipeById(recipeId);
+    return await this.syncRecipeById(recipeId);
   }
 
   async syncRecipeById(recipeId: string) {

charm/src/manager.ts

@@ -26,23 +26,34 @@ export async function searchCharms(
     await charmManager.sync(charms);
     const results = await Promise.all(
       charms.get().map(async (charm) => {
-        const data = charm.get();
-        const title = data?.[NAME] ?? "Untitled";
+        try {
+          const data = charm.get();
+          const title = data?.[NAME] ?? "Untitled";
 
-        const recipeId = await charmManager.syncRecipe(charm);
-        const recipe = recipeManager.recipeById(recipeId!)!;
+          const recipe = await charmManager.syncRecipe(charm);
 
-        return {
-          title: title + ` (#${charmId(charm.entityId!)!.slice(-4)})`,
-          description: (recipe as Recipe).argumentSchema.description,
-          id: charmId(charm.entityId!)!,
-          value: charm.entityId!,
-        };
+          return {
+            title: title + ` (#${charmId(charm.entityId!)!.slice(-4)})`,
+            description: (recipe as Recipe).argumentSchema.description,
+            id: charmId(charm.entityId!)!,
+            value: charm.entityId!,
+          };
+        } catch (error) {
+          console.error(`Error processing charm:`, error);
+          // Return a minimal viable object to keep the array intact
+          return {
+            title: "Error loading charm",
+            description: "Failed to load charm details",
+            id: charm.entityId ? charmId(charm.entityId)! : "unknown",
+            value: charm.entityId || "unknown",
+          };
+        }
       }),
     );
 
     // Early return if no charms are found
     if (!results.length) {
+      console.warn("No charms are available to search through.");
       return {
         thinking: "No charms are available to search through.",
         charms: [],
@@ -85,7 +96,11 @@ export async function searchCharms(
       /<charm id="([^"]+)" name="([^"]+)">([\s\S]*?)<\/charm>/g,
     );
 
-    const selectedCharms = [];
+    const selectedCharms: {
+      charm: Cell<Charm>;
+      name: string;
+      reason: string;
+    }[] = [];
     if (charmMatches) {
       for (const match of charmMatches) {
         const charmId = match[1];
@@ -109,8 +124,11 @@ export async function searchCharms(
       thinking,
       charms: selectedCharms,
     };
-  } catch (error) {
-    console.error("Search charms error:", error);
+  } catch (error: any) {
+    console.error(
+      "Search charms error:",
+      error?.message ?? JSON.stringify(error),
+    );
 
     return {
       thinking: "An error occurred while searching for charms.",

charm/src/search.ts

@@ -0,0 +1,16 @@
+{
+  "model": "anthropic:claude-3-7-sonnet-latest",
+  "system": "\nYou are creating a simple MVP based on the user's goal, using an existing data schema. Focus on the simplest implementation that works with the provided schema.\n\n<task>\nGiven a user's feature request and an existing data schema, you will:\n1. Create a short title (2-5 words) that names the artifact\n2. Create a one-sentence description in the format \"A <artifact> to <goal>\"\n3. Create a concise specification (3-5 sentences max) that works with the existing schema\n4. Generate a brief implementation plan (3 steps max)\n5. Design a minimal JSON schema that represents the core data model\n</task>\n\n<output_structure>\nYour response must be structured as follows:\n\n<thinking>Freeform reasoning for you to consider the solution.</thinking>\n\n<title>\n[Short title for the artifact, 2-5 words]\n</title>\n\n<description>\n[One-sentence description in the format \"A <artifact> to <goal>\"]\n</description>\n\n<spec>\n[Concise specification that captures only the essential requirements]\n</spec>\n\n<plan>\n[Brief 3-step implementation plan using the existing schema]\n</plan>\n\n<result_schema>\n[Minimal JSON Schema in valid JSON format that represents data created by the artifact]\n</result_schema>\n</output_structure>\n\n<schema_guidelines>\n1. Keep it minimal:\n   - Include only essential fields (5-7 properties max)\n   - Focus on the core functionality\n   - If user requested complex features, simplify for this first version\n\n2. Each property should have:\n   - A descriptive \"title\" field\n   - A brief \"description\" field\n   - A sensible default value where appropriate\n\n3. Example of a simple schema:\n```json\n{\n  \"type\": \"object\",\n  \"title\": \"Note List\",\n  \"description\": \"A simple note list for the user\",\n  \"properties\": {\n    \"notes\": {\n      \"type\": \"array\",\n      \"title\": \"Notes\",\n      \"description\": \"List of user notes\",\n      \"default\": [],\n      \"items\": {\n        \"type\": \"object\",\n        \"properties\": {\n          \"title\": {\n            \"type\": \"string\",\n            \"title\": \"Title\",\n            \"description\": \"Title of the note\",\n            \"default\": \"New Note\"\n          },\n          \"content\": {\n            \"type\": \"string\",\n            \"title\": \"Content\",\n            \"description\": \"Content of the note\"\n          },\n          \"created\": {\n            \"type\": \"string\",\n            \"format\": \"date-time\",\n            \"title\": \"Created Date\",\n            \"description\": \"When the note was created\"\n          }\n        },\n        \"required\": [\"title\", \"content\"]\n      }\n    }\n  },\n  \"required\": [\"title\", \"content\"]\n}\n```\n\nGUIDELINES:\n- Aim for the simplest possible solution that works with the existing schema\n- The specification should take into account the existing schema structure\n- Focus on what can be achieved quickly with the existing data model\n- Avoid suggesting modifications to the schema if possible\n</schema_guidelines>\n\n<high_level>\nIMPORTANT:\n- Focus on the simplest working version\n- Aim for fewer fields rather than more\n- But still capture all the important state the user is creating\n- The user can always iterate and improve the solution later\n</high_level>\n",
+  "stream": false,
+  "messages": [
+    {
+      "role": "user",
+      "content": "\n<user_input>\n\n<goal>show the data from `simpleValue2`</goal>\n\n<description>Show the data from the `simpleValue2` charm, displaying its current value and any associated metadata.</description>\n\n</user_input>\n\n<existing_schema>\n```json\n{\n  \"type\": \"object\",\n  \"properties\": {\n    \"simplevalue2\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"values\": {\n          \"type\": \"array\",\n          \"items\": {\n            \"type\": \"string\"\n          }\n        }\n      }\n    }\n  }\n}\n```\n</existing_schema>\n\nBased on this goal and the existing schema, please provide a title, description, any additional schema, detailed specification, and implementation plan.\n"
+    },
+    {
+      "role": "assistant",
+      "content": "<thinking>\nLet's analyze what we have:\n\n1. The user wants to display data from a `simpleValue2` charm.\n2. The existing schema shows that `simpleValue2` contains an object with a property called `values`, which is an array of strings.\n3. The user wants to see the current value and any associated metadata.\n\nFrom the schema, it seems like the main data is in the `values` array. There doesn't appear to be explicit metadata in the schema, just the array of string values.\n\nFor this MVP, I'll create a simple viewer that displays the values from the array. Since there's no explicit metadata in the schema, I'll focus on displaying the values themselves in a clean format.\n\nThe artifact will be a viewer component that renders the values from the `simpleValue2` charm.\n</thinking>\n\n<title>\nSimpleValue2 Viewer\n</title>\n\n<description>\nA viewer component to display data from the simpleValue2 charm.\n</description>\n\n<spec>\nThe SimpleValue2 Viewer will display all string values stored in the simpleValue2.values array. It will render each value as a list item, showing the index and the string content. If the array is empty, it will display a message indicating no values are present.\n</spec>\n\n<plan>\n1. Access the simpleValue2 object from the existing schema\n2. Extract the values array from the simpleValue2 object\n3. Render each string in the values array as a list item with its index\n</plan>\n\n<result_schema>\n{\n  \"type\": \"object\",\n  \"title\": \"SimpleValue2 Viewer\",\n  \"description\": \"Displays the values from the simpleValue2 charm\",\n  \"properties\": {\n    \"displayData\": {\n      \"type\": \"object\",\n      \"title\": \"Display Data\",\n      \"description\": \"Data prepared for display\",\n      \"properties\": {\n        \"values\": {\n          \"type\": \"array\",\n          \"title\": \"Values\",\n          \"description\": \"List of string values from simpleValue2\",\n          \"default\": [],\n          \"items\": {\n            \"type\": \"object\",\n            \"properties\": {\n              \"index\": {\n                \"type\": \"integer\",\n                \"title\": \"Index\",\n                \"description\": \"Position in the values array\"\n              },\n              \"value\": {\n                \"type\": \"string\",\n                \"title\": \"Value\",\n                \"description\": \"String value from the array\"\n              }\n            },\n            \"required\": [\"index\", \"value\"]\n          }\n        },\n        \"isEmpty\": {\n          \"type\": \"boolean\",\n          \"title\": \"Is Empty\",\n          \"description\": \"Indicates if there are no values to display\",\n          \"default\": true\n        }\n      },\n      \"required\": [\"values\", \"isEmpty\"]\n    }\n  },\n  \"required\": [\"displayData\"]\n}\n```"
+    }
+  ],
+  "maxTokens": 64000
+}