generateData -> llm + lift methods (#195)

generateData -> llm

simplified generate code to just be about LLM conversations
moves all the logic around prep / parsing / ... to recipe
ergo - json / html mode is now handled by lift inside recipes
example of using Zod / schemas to parse json results
llm improvements:

streaming
stop sequences
support for multiple messages (user -> assistant -> user -> assistant)
support for putting words into LLMs mouth (forcing it to generate json)
support for (base64) images
common file input:

added ability to load images / json data from file input

Author: anotherjesse

typescript/package-lock.json

@@ -1,20 +1,20 @@
 import { lift, createNodeFactory } from "./module.js";
 import { Value, NodeFactory, CellProxy } from "./types.js";
 
-export function generateData<T>(
+export function llm(
   params: Value<{
-    prompt: string;
-    result?: T;
-    schema?: any;
+    messages?: string[];
+    prompt?: string;
     system?: string;
-    mode?: "json" | "html";
+    stop?: string;
+    max_tokens?: number;
   }>
-): CellProxy<{ pending: boolean; result: T; partial: any; error: any }> {
-  generateDataFactory ||= createNodeFactory({
+): CellProxy<{ pending: boolean; result?: string; partial?: string; error: any }> {
+  llmFactory ||= createNodeFactory({
     type: "builtin",
-    implementation: "generateData",
+    implementation: "llm",
   });
-  return generateDataFactory(params);
+  return llmFactory(params);
 }
 
 export function fetchData<T>(
@@ -74,10 +74,10 @@ export function ifElse<T, U, V>(
 
 let ifElseFactory: NodeFactory<[any, any, any], any> | undefined = undefined;
 
-let generateDataFactory:
+let llmFactory:
   | NodeFactory<
-      { prompt: string; result?: any; schema?: any; system?: string },
-      { pending: boolean; result: any; partial: any; error: any }
+      { messages?: string[]; prompt?: string; system?: string; stop?: string; max_tokens?: number },
+      { pending: boolean; result?: string; partial?: string; error: any }
     >
   | undefined = undefined;
 

typescript/packages/common-builder/src/built-in.ts

@@ -6,7 +6,7 @@ export {
   isolated,
 } from "./module.js";
 export { recipe } from "./recipe.js";
-export { streamData, fetchData, generateData, ifElse, str } from "./built-in.js";
+export { streamData, fetchData, llm, ifElse, str } from "./built-in.js";
 export {
   ID,
   TYPE,

typescript/packages/common-builder/src/index.ts

@@ -3,14 +3,14 @@ import { type CellImpl } from "../cell.js";
 import { map } from "./map.js";
 import { fetchData } from "./fetch-data.js";
 import { streamData } from "./stream-data.js";
-import { generateData } from "./generate-data.js";
+import { llm } from "./llm.js";
 import { ifElse } from "./if-else.js";
 export const builtins: {
   [key: string]: (recipeCell: CellImpl<any>, node: Node) => void;
 } = {
   map,
   fetchData,
   streamData,
-  generateData,
+  llm,
   ifElse,
 };

typescript/packages/common-runner/src/builtins/generate-data.ts

@@ -0,0 +1,103 @@
+import { type Node } from "@commontools/common-builder";
+import { cell, CellImpl, ReactivityLog } from "../cell.js";
+import { sendValueToBinding, findAllAliasedCells } from "../utils.js";
+import { schedule, Action } from "../scheduler.js";
+import { mapBindingsToCell } from "../utils.js";
+import { makeClient, SimpleMessage, SimpleContent } from "../llm-client.js";
+
+// TODO(ja): investigate if generateText should be replaced by 
+// fetchData with streaming support
+
+/**
+ * Generate data via an LLM.
+ *
+ * Returns the complete result as `result` and the incremental result as
+ * `partial`. `pending` is true while a request is pending.
+ *
+ * @param prompt - A cell to store the prompt message - if you only have a single message
+ * @param messages - list of messages to send to the LLM. - alternating user and assistant messages.
+ *  - if you end with an assistant message, the LLM will continue from there.
+ *  - if both prompt and messages are empty, no LLM call will be made,
+ *    result and partial will be undefined.
+ * @param system - A cell to store the system message.
+ * @param stop - A cell to store (optional) stop sequence.
+ * @param max_tokens - A cell to store the maximum number of tokens to generate.
+ * 
+ * @returns { pending: boolean, result?: string, partial?: string } - As individual
+ *   cells, representing `pending` state, final `result` and incrementally
+ *   updating `partial` result.
+ */
+export function llm(
+  recipeCell: CellImpl<any>,
+  { inputs, outputs }: Node
+) {
+  const inputBindings = mapBindingsToCell(inputs, recipeCell) as {
+    messages?: SimpleContent[] | SimpleMessage[];
+    prompt?: SimpleContent;
+    stop?: string;
+    system?: string;
+    max_tokens?: number;
+  };
+  const inputsCell = cell(inputBindings);
+
+  const pending = cell(false);
+  const result = cell<string | undefined>(undefined);
+  const partial = cell<string | undefined>(undefined);
+
+  const outputCell = cell({ pending, result, partial });
+
+  const outputBindings = mapBindingsToCell(outputs, recipeCell) as any[];
+  sendValueToBinding(recipeCell, outputBindings, outputCell);
+
+  let currentRun = 0;
+
+  const startGeneration: Action = (log: ReactivityLog) => {
+    const thisRun = ++currentRun;
+
+    const { system, messages, prompt, stop, max_tokens } = inputsCell.getAsProxy([], log) ?? {};
+
+    result.setAtPath([], undefined, log);
+    partial.setAtPath([], undefined, log);
+
+    if (((prompt === undefined || prompt.length === 0) && (messages === undefined || messages.length === 0)) || system === undefined) {
+      pending.setAtPath([], false, log);
+      return;
+    }
+    pending.setAtPath([], true, log);
+
+    const updatePartial = (text: string) => {
+      if (thisRun != currentRun) return;
+      partial.setAtPath([], text, log);
+    }
+
+    let resultPromise = makeClient().sendRequest({
+      messages: messages || [prompt as SimpleContent],
+      system,
+      model: "claude-3-5-sonnet-20240620",
+      max_tokens: max_tokens || 4096,
+      stop
+    }, updatePartial)
+
+    resultPromise
+      .then((text) => {
+        if (thisRun !== currentRun) return;
+        // normalizeToCells(result, undefined, log);
+
+        pending.setAtPath([], false, log);
+        result.setAtPath([], text, log);
+        partial.setAtPath([], text, log);
+      }).catch((error) => {
+        if (thisRun !== currentRun) return;
+
+        console.error("Error generating data", error);
+        pending.setAtPath([], false, log);
+        result.setAtPath([], undefined, log);
+        partial.setAtPath([], undefined, log);
+      });
+  };
+
+  schedule(startGeneration, {
+    reads: findAllAliasedCells(inputBindings, recipeCell),
+    writes: findAllAliasedCells(outputBindings, recipeCell),
+  });
+}

typescript/packages/common-runner/src/builtins/index.ts

@@ -1,23 +1,28 @@
 import { LLMClient } from "@commontools/llm-client";
-export const LLM_SERVER_URL =
-  window.location.protocol + "//" + window.location.host + "/api/llm";
 
-export const suggestionClient = new LLMClient({
-  serverUrl: LLM_SERVER_URL,
-  system:
-    "You are an assistant that helps match user queries to relevant data gems based on their names and types.",
-  tools: [],
-});
+export { SimpleMessage, SimpleContent } from "@commontools/llm-client";
 
-export const mockResultClient = new LLMClient({
-  serverUrl: LLM_SERVER_URL,
-  system: `Generate dummy data as JSON as per the provided spec. Use the input to imagine what an API response would look like for a request.`,
-  tools: [],
-});
+export const suggestSystem = "You are an assistant that helps match user queries to relevant data gems based on their names and types."
+export const jsonDataRequest = `Generate dummy data as JSON as per the provided spec. Use the input to imagine what an API response would look like for a request.`
 
-export const makeClient = (system: string) =>
-  new LLMClient({
-    serverUrl: LLM_SERVER_URL,
-    system,
-    tools: [],
-  });
+export const LLM_SERVER_URL = window.location.protocol + "//" + window.location.host + "/api/llm";
+export const makeClient = (url?: string) => new LLMClient(url || LLM_SERVER_URL);
+
+export function dataRequest({
+  description,
+  inputData,
+  jsonSchema,
+}: {
+  description: string;
+  inputData: any;
+  jsonSchema: any;
+}) {
+  return `You specialize in generating believable and useful data for testing applications during development. Take the provided input parameters and use them to hallucinate a plausible result that conforms to the following JSON schema:
+
+  <schema>${JSON.stringify(jsonSchema)}</schema>
+
+  <description>${description}</description>
+  <input>${JSON.stringify(inputData)}</input>
+
+  Respond with only the generated data in a JSON block.`;
+}