Charm mentions in prompt composer (#1858)

* Add `@` mentions to `ct-prompt-input`

* Pills and files

* Implement `ct-chip` and track attachments in `chatbot.tsx`

* Checkpoint, pulling handlers off charm

* Working (yet janky) dynamic tools

* Dice rolling experiment

* Ensure charm is running before invoking handler

* Remove excess code

* Update comment

* Remove injected result cell from schema

* Add/remove attachments before submission

* Format pass

* Fix type errors

* Remove .md plan

* Remove manual schemas now that we trim result cell

* Remove commented line

* Format pass

* Remove TODO

* Address PR review

* Allow removal of attachments

* Format pass

* Optional result cell

Author: bfollington

packages/api/index.ts

@@ -314,6 +314,7 @@ export interface BuiltInLLMDialogState {
   error: unknown;
   cancelGeneration: Stream<void>;
   addMessage: Stream<BuiltInLLMMessage>;
+  flattenedTools: Record<string, any>;
 }
 
 export interface BuiltInGenerateObjectParams {

packages/patterns/chatbot-note-composed.tsx

@@ -262,116 +262,48 @@ export default recipe<ChatbotNoteInput, ChatbotNoteResult>(
       },
       editActiveNote: {
         description: "Modify the shared note.",
-        inputSchema: {
-          type: "object",
-          properties: {
-            body: {
-              type: "string",
-              description: "The content of the note.",
-            },
-          },
-          required: ["body"],
-        } as JSONSchema,
         handler: editNote({ content }),
       },
       readActiveNote: {
         description: "Read the currently focused note.",
-        inputSchema: {
-          type: "object",
-          properties: {},
-          required: [],
-        } as JSONSchema,
         handler: readNote({ content }),
       },
       listNotes: {
         description:
           "List all mentionable note titles (read the body with readNoteByIndex).",
-        inputSchema: {
-          type: "object",
-          properties: {},
-          required: [],
-        } as JSONSchema,
         handler: listMentionable({
           allCharms: allCharms as unknown as OpaqueRef<MentionableCharm[]>,
         }),
       },
       readNoteByIndex: {
         description:
           "Read the body of a note by its index in the listNotes() list.",
-        inputSchema: {
-          type: "object",
-          properties: {
-            index: {
-              type: "number",
-              description: "The index of the note in the notes list.",
-            },
-          },
-          required: ["index"],
-        } as JSONSchema,
         handler: readNoteByIndex({
           allCharms: allCharms as unknown as OpaqueRef<MentionableCharm[]>,
         }),
       },
       editNoteByIndex: {
         description:
           "Edit the body of a note by its index in the listNotes() list.",
-        inputSchema: {
-          type: "object",
-          properties: {
-            index: {
-              type: "number",
-              description: "The index of the note in the notes list.",
-            },
-            body: {
-              type: "string",
-              description: "The new content of the note.",
-            },
-          },
-          required: ["index", "body"],
-        } as JSONSchema,
         handler: editNoteByIndex({
           allCharms: allCharms as unknown as OpaqueRef<MentionableCharm[]>,
         }),
       },
       navigateToNote: {
         description: "Navigate to a note by its index in the listNotes() list.",
-        inputSchema: {
-          type: "object",
-          properties: {
-            index: {
-              type: "number",
-              description: "The index of the note in the notes list.",
-            },
-          },
-          required: ["index"],
-        } as JSONSchema,
         handler: navigateToNote({
           allCharms: allCharms as unknown as OpaqueRef<MentionableCharm[]>,
         }),
       },
       newNote: {
-        description: "Read the shared note.",
-        inputSchema: {
-          type: "object",
-          properties: {
-            title: {
-              type: "string",
-              description: "The title of the note.",
-            },
-            content: {
-              type: "string",
-              description: "The content of the note.",
-            },
-          },
-          required: ["title"],
-        } as JSONSchema,
+        description: "Create a new note instance",
         handler: newNote({
           allCharms: allCharms as unknown as OpaqueRef<MentionableCharm[]>,
         }),
       },
     };
 
-    const chat = Chat({ messages, tools });
+    const chat = Chat({ messages, tools, mentionable: allCharms });
     const note = Note({ title, content, allCharms });
 
     return {

packages/patterns/chatbot-outliner.tsx

@@ -140,7 +140,7 @@ export default recipe<LLMTestInput, LLMTestResult>(
       },
     };
 
-    const chat = Chat({ messages, tools });
+    const chat = Chat({ messages, tools, mentionable: allCharms });
     const { addMessage, cancelGeneration, pending } = chat;
 
     return {

packages/patterns/chatbot.tsx

@@ -20,16 +20,87 @@ import {
   Stream,
   UI,
 } from "commontools";
+import { MentionableCharm } from "./chatbot-list-view.tsx";
+
+const addAttachment = handler<
+  {
+    detail: {
+      attachment: PromptAttachment;
+    };
+  },
+  {
+    allAttachments: Cell<Array<PromptAttachment>>;
+  }
+>((event, { allAttachments }) => {
+  const { attachment } = event.detail;
+  const current = allAttachments.get() || [];
+  allAttachments.set([...current, attachment]);
+});
+
+const removeAttachment = handler<
+  {
+    detail: {
+      id: string;
+    };
+  },
+  {
+    allAttachments: Cell<Array<PromptAttachment>>;
+  }
+>((event, { allAttachments }) => {
+  const { id } = event.detail;
+  const current = allAttachments.get() || [];
+  allAttachments.set(current.filter((a) => a.id !== id));
+});
 
 const sendMessage = handler<
-  { detail: { message: string } },
+  {
+    detail: {
+      text: string;
+      attachments: Array<PromptAttachment>;
+      mentions: Array<any>;
+      message: string; // Backward compatibility
+    };
+  },
   {
     addMessage: Stream<BuiltInLLMMessage>;
+    allAttachments: Cell<Array<PromptAttachment>>;
+  }
+>((event, { addMessage, allAttachments }) => {
+  const { text } = event.detail;
+
+  // Build content array from text and attachments
+  const contentParts = [{ type: "text" as const, text }];
+
+  // Get current attachments from the global list
+  const attachments = allAttachments.get() || [];
+
+  // Compute mentions from mention attachments so they are available to consumers
+  const mentions = attachments
+    .filter((a) => a.type === "mention" && a.charm)
+    .map((a) => a.charm);
+
+  // Process attachments
+  for (const attachment of attachments) {
+    if (attachment.type === "file" && attachment.data) {
+      // For now, add a text reference
+      contentParts.push({
+        type: "text" as const,
+        text: `[Attached file: ${attachment.name}]`,
+      });
+    } else if (attachment.type === "clipboard" && attachment.data) {
+      // Append clipboard content as additional context
+      contentParts.push({
+        type: "text" as const,
+        text: `\n\n--- Pasted content ---\n${attachment.data}`,
+      });
+    }
+    // Note: mentions are already in the text as clean names
+    // The charm references are available in attachment.charm if needed
   }
->((event, { addMessage }) => {
+
   addMessage.send({
     role: "user",
-    content: [{ type: "text", text: event.detail.message }],
+    content: contentParts,
   });
 });
 
@@ -50,6 +121,15 @@ type ChatInput = {
   messages: Default<Array<BuiltInLLMMessage>, []>;
   tools: any;
   theme?: any;
+  mentionable: Cell<MentionableCharm[]>;
+};
+
+type PromptAttachment = {
+  id: string;
+  name: string;
+  type: "file" | "clipboard" | "mention";
+  data?: any; // File | Blob | string
+  charm?: any;
 };
 
 type ChatOutput = {
@@ -58,6 +138,8 @@ type ChatOutput = {
   addMessage: Stream<BuiltInLLMMessage>;
   cancelGeneration: Stream<void>;
   title?: string;
+  attachments: Array<PromptAttachment>;
+  tools: any;
 };
 
 export const TitleGenerator = recipe<
@@ -100,16 +182,45 @@ export const TitleGenerator = recipe<
 
 export default recipe<ChatInput, ChatOutput>(
   "Chat",
-  ({ messages, tools, theme }) => {
+  ({ messages, tools, theme, mentionable }) => {
     const model = cell<string>("anthropic:claude-sonnet-4-5");
+    const allAttachments = cell<Array<PromptAttachment>>([]);
 
-    const { addMessage, cancelGeneration, pending } = llmDialog({
-      system: "You are a helpful assistant with some tools.",
-      messages,
-      tools,
-      model,
+    // Derive tools from attachments
+    const dynamicTools = derive(allAttachments, (attachments) => {
+      const tools: Record<string, any> = {};
+
+      for (const attachment of attachments || []) {
+        if (attachment.type === "mention" && attachment.charm) {
+          const charmName = attachment.charm[NAME] || "Charm";
+          tools[charmName] = {
+            charm: attachment.charm,
+            description: `Handlers from ${charmName}`,
+          };
+        }
+      }
+
+      return tools;
     });
 
+    // Merge static and dynamic tools
+    const mergedTools = derive(
+      [tools, dynamicTools],
+      ([staticTools, dynamic]: [any, any]) => ({
+        ...staticTools,
+        ...dynamic,
+      }),
+    );
+
+    const { addMessage, cancelGeneration, pending, flattenedTools } = llmDialog(
+      {
+        system: "You are a helpful assistant with some tools.",
+        messages,
+        tools: mergedTools,
+        model,
+      },
+    );
+
     const { result } = fetchData({
       url: "/api/ai/llm/models",
       mode: "json",
@@ -130,26 +241,36 @@ export default recipe<ChatInput, ChatOutput>(
       [NAME]: title,
       [UI]: (
         <ct-screen>
-          <ct-hstack justify="between" slot="header">
+          <ct-vstack slot="header">
             <ct-heading level={4}>{title}</ct-heading>
-            <ct-tools-chip tools={tools} />
-          </ct-hstack>
+            <ct-hstack gap="normal">
+              <ct-attachments-bar
+                attachments={allAttachments}
+                removable
+                onct-remove={removeAttachment({ allAttachments })}
+              />
+              <ct-tools-chip tools={flattenedTools} />
+            </ct-hstack>
+          </ct-vstack>
 
           <ct-vscroll flex showScrollbar fadeEdges snapToBottom>
             <ct-chat
               theme={theme}
               $messages={messages}
               pending={pending}
-              tools={tools}
+              tools={flattenedTools}
             />
           </ct-vscroll>
 
           <div slot="footer">
             <ct-prompt-input
               placeholder="Ask the LLM a question..."
               pending={pending}
-              onct-send={sendMessage({ addMessage })}
+              $mentionable={mentionable}
+              onct-send={sendMessage({ addMessage, allAttachments })}
               onct-stop={cancelGeneration}
+              onct-attachment-add={addAttachment({ allAttachments })}
+              onct-attachment-remove={removeAttachment({ allAttachments })}
             />
             <ct-select
               items={items}
@@ -163,6 +284,8 @@ export default recipe<ChatInput, ChatOutput>(
       addMessage,
       cancelGeneration,
       title,
+      attachments: allAttachments,
+      tools: flattenedTools,
     };
   },
 );

packages/patterns/counter-handlers.ts

@@ -1,15 +1,25 @@
 /// <cts-enable />
 import { Cell, derive, handler } from "commontools";
 
-export const increment = handler<unknown, { value: Cell<number> }>(
-  (_, state) => {
+export const increment = handler<
+  { result?: Cell<string> },
+  { value: Cell<number> }
+>(
+  (args, state) => {
     state.value.set(state.value.get() + 1);
+    args.result?.set(`Incremented to ${state.value.get()}`);
   },
 );
 
-export const decrement = handler((_, state: { value: Cell<number> }) => {
-  state.value.set(state.value.get() - 1);
-});
+export const decrement = handler<
+  { result?: Cell<string> },
+  { value: Cell<number> }
+>(
+  (args, state) => {
+    state.value.set(state.value.get() - 1);
+    args.result?.set(`Decremented to ${state.value.get()}`);
+  },
+);
 
 export function nth(value: number) {
   if (value === 1) {

packages/patterns/counter.tsx

@@ -23,5 +23,7 @@ export default recipe<RecipeState>("Counter", (state) => {
       </div>
     ),
     value: state.value,
+    increment: increment(state),
+    decrement: decrement(state),
   };
 });