From 6ecfbf33f485e2bd46f34942caedc8f2d160b36e Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:37:55 +1000 Subject: [PATCH 1/6] Implement `generateText` built-in --- packages/api/index.ts | 19 ++ packages/patterns/note.tsx | 11 +- packages/runner/src/builder/built-in.ts | 9 + packages/runner/src/builder/factory.ts | 2 + packages/runner/src/builder/types.ts | 2 + packages/runner/src/builtins/index.ts | 16 +- packages/runner/src/builtins/llm.ts | 268 ++++++++++++++++++------ 7 files changed, 253 insertions(+), 74 deletions(-) diff --git a/packages/api/index.ts b/packages/api/index.ts index 8d7c7331b..ea720c4f0 100644 --- a/packages/api/index.ts +++ b/packages/api/index.ts @@ -704,6 +704,20 @@ export interface BuiltInGenerateObjectParams { metadata?: Record; } +export interface BuiltInGenerateTextParams { + prompt: string; + system?: string; + model?: string; + maxTokens?: number; +} + +export interface BuiltInGenerateTextState { + pending: boolean; + result?: string; + partial?: string; + requestHash?: string; +} + export interface BuiltInCompileAndRunParams { files: Array<{ name: string; contents: string }>; main: string; @@ -883,6 +897,10 @@ export type GenerateObjectFunction = ( params: Opaque, ) => OpaqueRef>; +export type GenerateTextFunction = ( + params: Opaque, +) => OpaqueRef; + export type FetchOptions = { body?: JSONValue; headers?: Record; @@ -989,6 +1007,7 @@ export declare const ifElse: IfElseFunction; export declare const llm: LLMFunction; export declare const llmDialog: LLMDialogFunction; export declare const generateObject: GenerateObjectFunction; +export declare const generateText: GenerateTextFunction; export declare const fetchData: FetchDataFunction; export declare const streamData: StreamDataFunction; export declare const compileAndRun: CompileAndRunFunction; diff --git a/packages/patterns/note.tsx b/packages/patterns/note.tsx index 95992645a..246a28538 100644 --- a/packages/patterns/note.tsx +++ b/packages/patterns/note.tsx @@ -4,6 +4,7 @@ import { type Cell, cell, type Default, + generateText, derive, handler, llm, @@ -175,15 +176,9 @@ const Note = recipe( content: string; }, ) => { - const result = llm({ + const result = generateText({ system: str`Translate the content to ${language}.`, - messages: derive(content, (c) => - [ - { - role: "user", - content: c, - }, - ] satisfies BuiltInLLMMessage[]), + prompt: content, }); return derive(result, ({ pending, result }) => { diff --git a/packages/runner/src/builder/built-in.ts b/packages/runner/src/builder/built-in.ts index e0b63de0b..c12d10e34 100644 --- a/packages/runner/src/builder/built-in.ts +++ b/packages/runner/src/builder/built-in.ts @@ -15,6 +15,8 @@ import type { BuiltInCompileAndRunParams, BuiltInCompileAndRunState, BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, + BuiltInGenerateTextState, BuiltInLLMGenerateObjectState, BuiltInLLMParams, BuiltInLLMState, @@ -50,6 +52,13 @@ export const generateObject = createNodeFactory({ params: Opaque, ) => OpaqueRef>; +export const generateText = createNodeFactory({ + type: "ref", + implementation: "generateText", +}) as ( + params: Opaque, +) => OpaqueRef; + export const fetchData = createNodeFactory({ type: "ref", implementation: "fetchData", diff --git a/packages/runner/src/builder/factory.ts b/packages/runner/src/builder/factory.ts index 181e9ff02..fbcdc45db 100644 --- a/packages/runner/src/builder/factory.ts +++ b/packages/runner/src/builder/factory.ts @@ -26,6 +26,7 @@ import { compileAndRun, fetchData, generateObject, + generateText, ifElse, llm, llmDialog, @@ -117,6 +118,7 @@ export const createBuilder = ( llm, llmDialog, generateObject, + generateText, fetchData, streamData, compileAndRun, diff --git a/packages/runner/src/builder/types.ts b/packages/runner/src/builder/types.ts index c1766ef8c..130f9883c 100644 --- a/packages/runner/src/builder/types.ts +++ b/packages/runner/src/builder/types.ts @@ -12,6 +12,7 @@ import type { DeriveFunction, FetchDataFunction, GenerateObjectFunction, + GenerateTextFunction, GetRecipeEnvironmentFunction, HandlerFunction, HFunction, @@ -282,6 +283,7 @@ export interface BuilderFunctionsAndConstants { llm: LLMFunction; llmDialog: LLMDialogFunction; generateObject: GenerateObjectFunction; + generateText: GenerateTextFunction; fetchData: FetchDataFunction; streamData: StreamDataFunction; compileAndRun: CompileAndRunFunction; diff --git a/packages/runner/src/builtins/index.ts b/packages/runner/src/builtins/index.ts index 62721ca3f..a71e147bf 100644 --- a/packages/runner/src/builtins/index.ts +++ b/packages/runner/src/builtins/index.ts @@ -2,14 +2,17 @@ import { raw } from "../module.ts"; import { map } from "./map.ts"; import { fetchData } from "./fetch-data.ts"; import { streamData } from "./stream-data.ts"; -import { generateObject, llm } from "./llm.ts"; +import { generateObject, generateText, llm } from "./llm.ts"; import { ifElse } from "./if-else.ts"; import type { IRuntime } from "../runtime.ts"; import { compileAndRun } from "./compile-and-run.ts"; import { navigateTo } from "./navigate-to.ts"; import { wish } from "./wish.ts"; import type { Cell } from "../cell.ts"; -import type { BuiltInGenerateObjectParams } from "@commontools/api"; +import type { + BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, +} from "@commontools/api"; import { llmDialog } from "./llm-dialog.ts"; /** @@ -34,6 +37,15 @@ export function registerBuiltins(runtime: IRuntime) { requestHash: Cell; }>(generateObject), ); + moduleRegistry.addModuleByRef( + "generateText", + raw; + result: Cell; + partial: Cell; + requestHash: Cell; + }>(generateText), + ); moduleRegistry.addModuleByRef("navigateTo", raw(navigateTo)); moduleRegistry.addModuleByRef("wish", raw(wish)); } diff --git a/packages/runner/src/builtins/llm.ts b/packages/runner/src/builtins/llm.ts index bc69219a3..cd16168b4 100644 --- a/packages/runner/src/builtins/llm.ts +++ b/packages/runner/src/builtins/llm.ts @@ -9,6 +9,7 @@ import { } from "@commontools/llm"; import { BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, BuiltInLLMParams, } from "@commontools/api"; import { refer } from "merkle-reference/json"; @@ -22,6 +23,54 @@ const client = new LLMClient(); // TODO(ja): investigate if generateText should be replaced by // fetchData with streaming support +/** + * Helper function to initialize cells for LLM built-ins. + * Reduces code duplication across llm, generateText, and generateObject. + */ +function initializeCells( + runtime: IRuntime, + parentCell: Cell, + cause: any, + tx: IExtendedStorageTransaction, + builtinName: "llm" | "generateText" | "generateObject", +): { + pending: Cell; + result: Cell; + partial: Cell; + requestHash: Cell; +} { + const pending = runtime.getCell( + parentCell.space, + { [builtinName]: { pending: cause } }, + undefined, + tx, + ); + pending.send(false); + + const result = runtime.getCell( + parentCell.space, + { [builtinName]: { result: cause } }, + undefined, + tx, + ); + + const partial = runtime.getCell( + parentCell.space, + { [builtinName]: { partial: cause } }, + undefined, + tx, + ); + + const requestHash = runtime.getCell( + parentCell.space, + { [builtinName]: { requestHash: cause } }, + undefined, + tx, + ); + + return { pending, result, partial, requestHash }; +} + /** * Generate data via an LLM. * @@ -59,40 +108,17 @@ export function llm( return (tx: IExtendedStorageTransaction) => { if (!cellsInitialized) { - pending = runtime.getCell( - parentCell.space, - { llm: { pending: cause } }, - undefined, - tx, - ); - pending.send(false); - - result = runtime.getCell( - parentCell.space, - { - llm: { result: cause }, - }, - undefined, - tx, - ); - - partial = runtime.getCell( - parentCell.space, - { - llm: { partial: cause }, - }, - undefined, - tx, - ); - - requestHash = runtime.getCell( - parentCell.space, - { - llm: { requestHash: cause }, - }, - undefined, + const cells = initializeCells( + runtime, + parentCell, + cause, tx, + "llm", ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; sendResult(tx, { pending, result, partial, requestHash }); cellsInitialized = true; @@ -182,6 +208,143 @@ export function llm( }; } +/** + * Generate text via an LLM. + * + * A simplified alternative to `llm` that takes a single prompt string and + * optional system message, returning plain text rather than a structured + * content array. + * + * Returns the complete result as `result` (string) and the incremental result + * as `partial` (string). `pending` is true while a request is pending. + * + * @param prompt - The user prompt/message to send to the LLM. + * @param system - Optional system message. + * @param model - Model to use (defaults to DEFAULT_MODEL_NAME). + * @param maxTokens - Maximum number of tokens to generate (defaults to 4096). + * + * @returns { pending: boolean, result?: string, partial?: string, requestHash?: string } - + * As individual docs, representing `pending` state, final `result` and + * incrementally updating `partial` result. + */ +export function generateText( + inputsCell: Cell, + sendResult: (tx: IExtendedStorageTransaction, result: any) => void, + _addCancel: (cancel: () => void) => void, + cause: any, + parentCell: Cell, + runtime: IRuntime, +): Action { + let currentRun = 0; + let previousCallHash: string | undefined = undefined; + let cellsInitialized = false; + let pending: Cell; + let result: Cell; + let partial: Cell; + let requestHash: Cell; + + return (tx: IExtendedStorageTransaction) => { + if (!cellsInitialized) { + const cells = initializeCells( + runtime, + parentCell, + cause, + tx, + "generateText", + ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; + + sendResult(tx, { pending, result, partial, requestHash }); + cellsInitialized = true; + } + const thisRun = ++currentRun; + const pendingWithLog = pending.withTx(tx); + const resultWithLog = result.withTx(tx); + const partialWithLog = partial.withTx(tx); + const requestHashWithLog = requestHash.withTx(tx); + + const { system, prompt, model, maxTokens } = + inputsCell.getAsQueryResult([], tx) ?? {}; + + // If no prompt is provided, don't make a request + if (!prompt) { + pendingWithLog.set(false); + return; + } + + // Convert simple prompt to messages array format for LLM client + const llmParams: LLMRequest = { + system: system ?? "", + messages: [{ role: "user", content: prompt }], + stop: "", + maxTokens: maxTokens ?? 4096, + stream: true, + model: model ?? DEFAULT_MODEL_NAME, + metadata: { + context: "charm", + }, + cache: true, + }; + + const hash = refer(llmParams).toString(); + + // Return if the same request is being made again + if (hash === previousCallHash || hash === requestHashWithLog.get()) return; + previousCallHash = hash; + + resultWithLog.set(undefined); + partialWithLog.set(undefined); + pendingWithLog.set(true); + + const updatePartial = (text: string) => { + if (thisRun != currentRun) return; + const status = tx.status(); + if (status.status !== "ready") return; + + partialWithLog.set(text); + }; + + const resultPromise = client.sendRequest(llmParams, updatePartial); + + resultPromise + .then(async (llmResult) => { + if (thisRun !== currentRun) return; + + await runtime.idle(); + + // Extract text from the LLM response + const textResult = extractTextFromLLMResponse(llmResult); + + await runtime.editWithRetry((tx) => { + pending.withTx(tx).set(false); + result.withTx(tx).set(textResult); + partial.withTx(tx).set(textResult); + requestHash.withTx(tx).set(hash); + }); + }) + .catch(async (error) => { + if (thisRun !== currentRun) return; + + console.error("Error generating text", error); + + await runtime.idle(); + + await runtime.editWithRetry((tx) => { + pending.withTx(tx).set(false); + result.withTx(tx).set(undefined); + partial.withTx(tx).set(undefined); + }); + + // TODO(seefeld): Not writing now, so we retry the request after failure. + // Replace this with more fine-grained retry logic. + // requestHash.setAtPath([], hash, log); + }); + }; +} + /** * Generate structured data via an LLM using JSON mode. * @@ -223,40 +386,17 @@ export function generateObject>( return (tx: IExtendedStorageTransaction) => { if (!cellsInitialized) { - pending = runtime.getCell( - parentCell.space, - { generateObject: { pending: cause } }, - undefined, - tx, - ); - pending.send(false); - - result = runtime.getCell( - parentCell.space, - { - generateObject: { result: cause }, - }, - undefined, - tx, - ); - - partial = runtime.getCell( - parentCell.space, - { - generateObject: { partial: cause }, - }, - undefined, - tx, - ); - - requestHash = runtime.getCell( - parentCell.space, - { - generateObject: { requestHash: cause }, - }, - undefined, + const cells = initializeCells( + runtime, + parentCell, + cause, tx, + "generateObject", ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; sendResult(tx, { pending, result, partial, requestHash }); cellsInitialized = true; From c2bc91cf035bc9b4b9212b160669656c045bd958 Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:49:46 +1000 Subject: [PATCH 2/6] Usage example in `note.tsx` --- packages/patterns/note.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/patterns/note.tsx b/packages/patterns/note.tsx index 246a28538..502ad4c62 100644 --- a/packages/patterns/note.tsx +++ b/packages/patterns/note.tsx @@ -178,7 +178,7 @@ const Note = recipe( ) => { const result = generateText({ system: str`Translate the content to ${language}.`, - prompt: content, + prompt: str`${content}`, }); return derive(result, ({ pending, result }) => { From 2ddce697b57d64968948619e446883fa0a222448 Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:50:59 +1000 Subject: [PATCH 3/6] Add `mise.toml` to specify `deno` version Probably only useful for me... for now --- mise.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 mise.toml diff --git a/mise.toml b/mise.toml new file mode 100644 index 000000000..668ef33e8 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +deno = "2.5.2" From e8d3671aa42201cb4d7e123b1736d771f8187c5e Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 07:51:58 +1000 Subject: [PATCH 4/6] Lint + format --- packages/patterns/note.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/patterns/note.tsx b/packages/patterns/note.tsx index 502ad4c62..3e4602fdd 100644 --- a/packages/patterns/note.tsx +++ b/packages/patterns/note.tsx @@ -7,7 +7,6 @@ import { generateText, derive, handler, - llm, NAME, navigateTo, type Opaque, From f0fade4259a08ad0a1e6d6db558b642d0ddcdc35 Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 08:11:23 +1000 Subject: [PATCH 5/6] Lint + format again --- packages/patterns/note.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/patterns/note.tsx b/packages/patterns/note.tsx index 3e4602fdd..41492f99c 100644 --- a/packages/patterns/note.tsx +++ b/packages/patterns/note.tsx @@ -1,11 +1,10 @@ /// import { - type BuiltInLLMMessage, type Cell, cell, type Default, - generateText, derive, + generateText, handler, NAME, navigateTo, From 5a6694a2e2f89459bebdfc04494fb4794d53172d Mon Sep 17 00:00:00 2001 From: Ben Follington <5009316+bfollington@users.noreply.github.com> Date: Wed, 5 Nov 2025 08:21:34 +1000 Subject: [PATCH 6/6] Clear intermediate results to prevent stale output --- packages/runner/src/builtins/llm.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/runner/src/builtins/llm.ts b/packages/runner/src/builtins/llm.ts index cd16168b4..262f9971d 100644 --- a/packages/runner/src/builtins/llm.ts +++ b/packages/runner/src/builtins/llm.ts @@ -155,13 +155,15 @@ export function llm( if (hash === previousCallHash || hash === requestHashWithLog.get()) return; previousCallHash = hash; - resultWithLog.set(undefined); - partialWithLog.set(undefined); - if (!Array.isArray(messages) || messages.length === 0) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(false); return; } + + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(true); const updatePartial = (text: string) => { @@ -201,6 +203,9 @@ export function llm( partial.withTx(tx).set(undefined); }); + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + // TODO(seefeld): Not writing now, so we retry the request after failure. // Replace this with more fine-grained retry logic. // requestHash.setAtPath([], hash, log); @@ -271,6 +276,8 @@ export function generateText( // If no prompt is provided, don't make a request if (!prompt) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(false); return; } @@ -338,6 +345,9 @@ export function generateText( partial.withTx(tx).set(undefined); }); + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + // TODO(seefeld): Not writing now, so we retry the request after failure. // Replace this with more fine-grained retry logic. // requestHash.setAtPath([], hash, log); @@ -411,6 +421,8 @@ export function generateObject>( inputsCell.getAsQueryResult([], tx) ?? {}; if (!prompt || !schema) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(false); return; } @@ -476,6 +488,9 @@ export function generateObject>( partial.withTx(tx).set(undefined); }); + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + // TODO(seefeld): Not writing now, so we retry the request after failure. // Replace this with more fine-grained retry logic. // requestHash.setAtPath([], hash, log);