Use tool calling API to build graph

bfollington · bfollington · commit 668f783bbf24 · 2024-06-06T16:30:05.000-07:00
diff --git a/typescript/packages/lookslike-prototype/src/components/com-app.ts b/typescript/packages/lookslike-prototype/src/components/com-app.ts
@@ -3,21 +3,21 @@ import { customElement, state } from 'lit/decorators.js'
 import { base } from '../styles'
 
 import { Recipe, emptyGraph, todoAppMockup, RecipeNode } from '../data'
-import { doLLM, grabJson } from '../llm'
+import { doLLM, grabJson, processUserInput } from '../llm'
 import { collectSymbols } from '../graph'
 import { listKeys } from '../state'
 
 const codePrompt = `
   Your task is to take a user description or request and produce a series of nodes for a computation graph. Nodes can be code blocks or UI components and they communicate with named ports.
-  you will provide the required edges to connect data from the environment to the inputs of the node. The keys of \`in\` are the names of local inputs and the values are NodePaths (of the form [context, nodeId], where context is typically '.' meaning local namespace).
+
+  You will construct the graph using the available tools to add, remove, replace and list nodes.
+  You will provide the required edges to connect data from the environment to the inputs of the node. The keys of \`in\` are the names of local inputs and the values are NodePaths (of the form [context, nodeId], where context is typically '.' meaning local namespace).
 
   "Imagine some todos" ->
 
-  \`\`\`json
-  [
-    {
-      "id": "todos",
-      "contentType": "text/javascript",
+  addCodeNode({
+    "id": "todos",
+    "node": {
       "in": {},
       "outputType": {
         "type": "array",
@@ -29,10 +29,11 @@ const codePrompt = `
           }
         }
       },
-      "body": "return [{ label: 'Water my plants', checked: false }, { label: 'Buy milk', checked: true }];"
-    }
-  ]
-  \`\`\`
+    },
+    "code": \`
+      return [{ label: 'Water my plants', checked: false }, { label: 'Buy milk', checked: true }];
+    \`
+  })
 
   Tasks that take no inputs require no edges.
   All function bodies must take zero parameters. Inputs can be accessed via 'read' and 'deref'.
@@ -41,28 +42,40 @@ const codePrompt = `
 
   "Remind me to water the plants" ->
 
-  \`\`\`json
-  [
-    {
-      "id": "addReminder",
-      "contentType": "text/javascript",
+  addCodeNode({
+    "id": "addReminder",
+    "node": {
       "in": {},
-      "outputType": {},
-      "body": "const todos = input('todos'); const newTodo = { label: 'water the plants', checked: false }; cost newTodos = [...todos, newTodo]; return newTodos;"
+      "outputType": {
+        "type": "array",
+        "items": {
+          "type": "object",
+          "properties": {
+            "label": { "type": "string" },
+            "checked": { "type": "boolean" }
+          }
+        }
+      },
+    },
+    "code": \`
+      const todos = input('todos');
+      const newTodo = { label: 'water the plants', checked: false };
+      const newTodos = [...todos, newTodo];
+      return newTodos;
+    \`
     }
-  ]
-  \`\`\`
+  )
 
   Tasks that take no inputs require no edges.
 
   ---
 
+
   "Take the existing todos and filter to unchecked" ->
-  \`\`\`json
-  [
-    {
-      "id": "filteredTodos",
-      "contentType": "text/javascript",
+
+  addCodeNode({
+    "id": "filteredTodos",
+    "node": {
       "in": {
         "todos": [".", "todos"]
       },
@@ -76,113 +89,108 @@ const codePrompt = `
           }
         }
       },
-      "body": "const todos = input('todos'); return todos.filter(todo => todo.checked);"
+    },
+    "code": \`
+      const todos = input('todos');
+      return todos.filter(todo => todo.checked);
+    \`
     }
-  ]
-  \`\`\`
+  )
 
   Tasks that filter other data must pipe the data through the edges.
   All function bodies must take zero parameters. Inputs can be accessed via 'input()', values may be null.
-
-  ContentType should be "text/javascript" for code.
   Always respond with code, even for static data. Wrap your response in a json block. Respond with nothing else.
 
   ---
 
   "render each image by url" ->
   images is an array of strings (URLs)
 
-  \`\`\`json
-  [
-    {
-      "id": "imageUi",
-      "contentType": "application/json+vnd.common.ui",
+  addUiNode({
+    "id": "imageUi",
+    "node": {
       "in": {
         "images": [".", "images"]
       },
       "outputType": {
         "$id": "https://common.tools/ui.schema.json"
       },
-      "body": {
-        "tag": "ul",
-        "props": {
-          "className": "image"
-        },
-        "children": [
-          "type": "repeat",
-          "binding": "images",
-          "template": {
-            "tag": "li",
-            "props": {},
-            "children": [
-              {
-                "tag": "img",
-                "props": {
-                  "src": { type: 'string', binding: null },
-                }
+    },
+    "body": {
+      "tag": "ul",
+      "props": {
+        "className": "image"
+      },
+      "children": [
+        "type": "repeat",
+        "binding": "images",
+        "template": {
+          "tag": "li",
+          "props": {},
+          "children": [
+            {
+              "tag": "img",
+              "props": {
+                "src": { type: 'string', binding: null },
               }
-            ],
-          }
-        ]
-      }
+            }
+          ],
+        }
+      ]
     }
-  ]
+  })
 
   Raw values can be passed through by setting binding to null.
 
   ---
 
   "render my todos" ->
 
-  \`\`\`json
-  [
-    {
-      "id": "todoUi",
-      "contentType": "application/json+vnd.common.ui",
+  addUiNode({
+    "id": "todoUi",
+    "node": {
       "in": {
         "todos": [".", "todos"]
       },
       "outputType": {
         "$id": "https://common.tools/ui.schema.json"
       },
-      "body": {
-        "tag": "ul",
-        "props": {
-          "className": "todo"
-        },
-        "children": {
-          "type": "repeat",
-          "binding": "todos",
-          "template": {
-            "tag": "li",
-            "props": {},
-            "children": [
-              {
-                "tag": "input",
-                "props": {
-                  "type": "checkbox",
-                  "checked": { type: 'boolean', binding: 'checked' }
-                }
-              },
-              {
-                "tag": "span",
-                "props": {
-                  "className": "todo-label"
-                },
-                "children": [
-                  { type: 'string', binding: 'label' }
-                ]
+    },
+    "body": {
+      "tag": "ul",
+      "props": {
+        "className": "todo"
+      },
+      "children": {
+        "type": "repeat",
+        "binding": "todos",
+        "template": {
+          "tag": "li",
+          "props": {},
+          "children": [
+            {
+              "tag": "input",
+              "props": {
+                "type": "checkbox",
+                "checked": { type: 'boolean', binding: 'checked' }
               }
-            ]
-          }
+            },
+            {
+              "tag": "span",
+              "props": {
+                "className": "todo-label"
+              },
+              "children": [
+                { type: 'string', binding: 'label' }
+              ]
+            }
+          ]
         }
       }
     }
-  ]
-  \`\`\`
-
-  ContentType should be "application/json+vnd.common.ui" for UI. UI trees cannot use any javascript methods, code blocks must prepare the data for the UI to consume.
+  })
 
+  UI trees cannot use any javascript methods, code blocks must prepare the data for the UI to consume.
   notalk;justgo
 `
 
@@ -220,35 +228,46 @@ export class ComApp extends LitElement {
     this.graph = newGraph;
     this.userInput = '';
 
-    const localContext = `
-      The following nodes are available in the current context, these can be referenced by name when wiring the graph.
-
-      \`\`\`json
-      ${JSON.stringify(symbols, null, 2)}
-      \`\`\`
+    const systemContext = `
+      Ensure you list the current state of the graph and make a detailed step-by-step plan for updating the graph to match the user's request.'
 
-      The keys at the top of the list are the most recently created by the user, prefer these when making connections.
-      Ensure the you use the name you assign to the input in the body of the code for any read calls.
-      `
-
-    const systemContext = ``
-
-    const result = await doLLM(input + localContext, codePrompt + systemContext, null)
-    const message = result?.choices[0]?.message
-    if (message) {
-      const data = grabJson(message?.content)
-      for (const node of data) {
-        node.messages = []
-        newGraph.push(node)
-      }
+      Prefer to send tool calls in serial rather than in one large block, this way we can show the user the nodes as they are created.
+    `
 
-      // newNode.id = data.id || id
-      // newNode.contentType = data.contentType || 'text/javascript'
-      // newNode.outputType = data.outputType || {}
-      // newNode.body = data.body || '...'
-      // newNode.in = data.in || {}
-      // newNode.messages = data.messages || newNode.messages
-    }
+    const availableFunctions = {
+      listNodes: () => {
+        console.log('listNodes', this.graph)
+        return JSON.stringify(this.graph)
+      },
+      addCodeNode: ({ id, node, code }) => {
+        console.log('addCodeNode', id, node, code)
+        newGraph.push({ id, contentType: 'text/javascript', ...node, body: code })
+        this.graph = JSON.parse(JSON.stringify(newGraph));
+        this.requestUpdate();
+        return `Added node: ${id}`
+      },
+      addUiNode: ({ id, node, body }) => {
+        console.log('addUiNode', id, node, body)
+        newGraph.push({ id, contentType: 'application/json+vnd.common.ui', ...node, body })
+        this.graph = JSON.parse(JSON.stringify(newGraph));
+        this.requestUpdate();
+        return `Added node: ${id}`
+      },
+      replaceNode: () => 'hello replace',
+      deleteNode: () => 'hello delete'
+    };
+    const result = await processUserInput(input, codePrompt + systemContext, availableFunctions);
+    console.log('result', result);
+
+    // const result = await doLLM(input + localContext, codePrompt + systemContext, null)
+    // const message = result?.choices[0]?.message
+    // if (message) {
+    //   const data = grabJson(message?.content)
+    //   for (const node of data) {
+    //     node.messages = []
+    //     newGraph.push(node)
+    //   }
+    // }
 
     this.graph = JSON.parse(JSON.stringify(newGraph));
     console.log('graph updated', this.graph);
diff --git a/typescript/packages/lookslike-prototype/src/components/com-data.ts b/typescript/packages/lookslike-prototype/src/components/com-data.ts
@@ -44,7 +44,11 @@ class CodeMirrorDataViewer extends LitElement {
   }
 
   updated() {
-    this.editor.dispatch({ changes: { from: 0, to: this.state.doc.length, insert: this.data } });
+    this.state = EditorState.create({
+      doc: this.data,
+      extensions: [basicSetup, json()]
+    })
+    this.editor.setState(this.state);
     this.foldAll();
   }
 
diff --git a/typescript/packages/lookslike-prototype/src/components/com-thread.ts b/typescript/packages/lookslike-prototype/src/components/com-thread.ts
@@ -95,7 +95,7 @@ export class ComThread extends LitElement {
       (node) => html`
           <com-thread-group>
             ${repeat(
-        node.messages.filter(m => m.role === 'user'),
+        node.messages?.filter(m => m.role === 'user') || [],
         (node) => {
           return html`<com-prompt slot="prompt">
                       ${node.content}
diff --git a/typescript/packages/lookslike-prototype/src/data.ts b/typescript/packages/lookslike-prototype/src/data.ts
@@ -4,7 +4,7 @@ export type InputMap = { [port: string]: NodePath };
 
 export type RecipeNode = {
   id: string, // an actual unique ID
-  messages: Message[], // could be empty if a user hand-authored the node etc.
+  messages?: Message[], // could be empty if a user hand-authored the node etc.
   contentType: string,
   outputType: any,
   in: InputMap,
diff --git a/typescript/packages/lookslike-prototype/src/llm.ts b/typescript/packages/lookslike-prototype/src/llm.ts

Original file line number	Diff line number	Diff line change
`@@ -44,7 +44,11 @@ class CodeMirrorDataViewer extends LitElement {`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`updated() {`
`47`		`- this.editor.dispatch({ changes: { from: 0, to: this.state.doc.length, insert: this.data } });`
	`47`	`+ this.state = EditorState.create({`
	`48`	`+ doc: this.data,`
	`49`	`+ extensions: [basicSetup, json()]`
	`50`	`+ })`
	`51`	`+ this.editor.setState(this.state);`
`48`	`52`	`this.foldAll();`
`49`	`53`	`}`
`50`	`54`