diff --git a/mise.toml b/mise.toml new file mode 100644 index 000000000..668ef33e8 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +deno = "2.5.2" diff --git a/packages/api/index.ts b/packages/api/index.ts index 8d7c7331b..ea720c4f0 100644 --- a/packages/api/index.ts +++ b/packages/api/index.ts @@ -704,6 +704,20 @@ export interface BuiltInGenerateObjectParams { metadata?: Record; } +export interface BuiltInGenerateTextParams { + prompt: string; + system?: string; + model?: string; + maxTokens?: number; +} + +export interface BuiltInGenerateTextState { + pending: boolean; + result?: string; + partial?: string; + requestHash?: string; +} + export interface BuiltInCompileAndRunParams { files: Array<{ name: string; contents: string }>; main: string; @@ -883,6 +897,10 @@ export type GenerateObjectFunction = ( params: Opaque, ) => OpaqueRef>; +export type GenerateTextFunction = ( + params: Opaque, +) => OpaqueRef; + export type FetchOptions = { body?: JSONValue; headers?: Record; @@ -989,6 +1007,7 @@ export declare const ifElse: IfElseFunction; export declare const llm: LLMFunction; export declare const llmDialog: LLMDialogFunction; export declare const generateObject: GenerateObjectFunction; +export declare const generateText: GenerateTextFunction; export declare const fetchData: FetchDataFunction; export declare const streamData: StreamDataFunction; export declare const compileAndRun: CompileAndRunFunction; diff --git a/packages/patterns/note.tsx b/packages/patterns/note.tsx index 95992645a..41492f99c 100644 --- a/packages/patterns/note.tsx +++ b/packages/patterns/note.tsx @@ -1,12 +1,11 @@ /// import { - type BuiltInLLMMessage, type Cell, cell, type Default, derive, + generateText, handler, - llm, NAME, navigateTo, type Opaque, @@ -175,15 +174,9 @@ const Note = recipe( content: string; }, ) => { - const result = llm({ + const result = generateText({ system: str`Translate the content to ${language}.`, - messages: derive(content, (c) => - [ - { - role: "user", - content: c, - }, - ] satisfies BuiltInLLMMessage[]), + prompt: str`${content}`, }); return derive(result, ({ pending, result }) => { diff --git a/packages/runner/src/builder/built-in.ts b/packages/runner/src/builder/built-in.ts index e0b63de0b..c12d10e34 100644 --- a/packages/runner/src/builder/built-in.ts +++ b/packages/runner/src/builder/built-in.ts @@ -15,6 +15,8 @@ import type { BuiltInCompileAndRunParams, BuiltInCompileAndRunState, BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, + BuiltInGenerateTextState, BuiltInLLMGenerateObjectState, BuiltInLLMParams, BuiltInLLMState, @@ -50,6 +52,13 @@ export const generateObject = createNodeFactory({ params: Opaque, ) => OpaqueRef>; +export const generateText = createNodeFactory({ + type: "ref", + implementation: "generateText", +}) as ( + params: Opaque, +) => OpaqueRef; + export const fetchData = createNodeFactory({ type: "ref", implementation: "fetchData", diff --git a/packages/runner/src/builder/factory.ts b/packages/runner/src/builder/factory.ts index 181e9ff02..fbcdc45db 100644 --- a/packages/runner/src/builder/factory.ts +++ b/packages/runner/src/builder/factory.ts @@ -26,6 +26,7 @@ import { compileAndRun, fetchData, generateObject, + generateText, ifElse, llm, llmDialog, @@ -117,6 +118,7 @@ export const createBuilder = ( llm, llmDialog, generateObject, + generateText, fetchData, streamData, compileAndRun, diff --git a/packages/runner/src/builder/types.ts b/packages/runner/src/builder/types.ts index c1766ef8c..130f9883c 100644 --- a/packages/runner/src/builder/types.ts +++ b/packages/runner/src/builder/types.ts @@ -12,6 +12,7 @@ import type { DeriveFunction, FetchDataFunction, GenerateObjectFunction, + GenerateTextFunction, GetRecipeEnvironmentFunction, HandlerFunction, HFunction, @@ -282,6 +283,7 @@ export interface BuilderFunctionsAndConstants { llm: LLMFunction; llmDialog: LLMDialogFunction; generateObject: GenerateObjectFunction; + generateText: GenerateTextFunction; fetchData: FetchDataFunction; streamData: StreamDataFunction; compileAndRun: CompileAndRunFunction; diff --git a/packages/runner/src/builtins/index.ts b/packages/runner/src/builtins/index.ts index 62721ca3f..a71e147bf 100644 --- a/packages/runner/src/builtins/index.ts +++ b/packages/runner/src/builtins/index.ts @@ -2,14 +2,17 @@ import { raw } from "../module.ts"; import { map } from "./map.ts"; import { fetchData } from "./fetch-data.ts"; import { streamData } from "./stream-data.ts"; -import { generateObject, llm } from "./llm.ts"; +import { generateObject, generateText, llm } from "./llm.ts"; import { ifElse } from "./if-else.ts"; import type { IRuntime } from "../runtime.ts"; import { compileAndRun } from "./compile-and-run.ts"; import { navigateTo } from "./navigate-to.ts"; import { wish } from "./wish.ts"; import type { Cell } from "../cell.ts"; -import type { BuiltInGenerateObjectParams } from "@commontools/api"; +import type { + BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, +} from "@commontools/api"; import { llmDialog } from "./llm-dialog.ts"; /** @@ -34,6 +37,15 @@ export function registerBuiltins(runtime: IRuntime) { requestHash: Cell; }>(generateObject), ); + moduleRegistry.addModuleByRef( + "generateText", + raw; + result: Cell; + partial: Cell; + requestHash: Cell; + }>(generateText), + ); moduleRegistry.addModuleByRef("navigateTo", raw(navigateTo)); moduleRegistry.addModuleByRef("wish", raw(wish)); } diff --git a/packages/runner/src/builtins/llm.ts b/packages/runner/src/builtins/llm.ts index bc69219a3..262f9971d 100644 --- a/packages/runner/src/builtins/llm.ts +++ b/packages/runner/src/builtins/llm.ts @@ -9,6 +9,7 @@ import { } from "@commontools/llm"; import { BuiltInGenerateObjectParams, + BuiltInGenerateTextParams, BuiltInLLMParams, } from "@commontools/api"; import { refer } from "merkle-reference/json"; @@ -22,6 +23,54 @@ const client = new LLMClient(); // TODO(ja): investigate if generateText should be replaced by // fetchData with streaming support +/** + * Helper function to initialize cells for LLM built-ins. + * Reduces code duplication across llm, generateText, and generateObject. + */ +function initializeCells( + runtime: IRuntime, + parentCell: Cell, + cause: any, + tx: IExtendedStorageTransaction, + builtinName: "llm" | "generateText" | "generateObject", +): { + pending: Cell; + result: Cell; + partial: Cell; + requestHash: Cell; +} { + const pending = runtime.getCell( + parentCell.space, + { [builtinName]: { pending: cause } }, + undefined, + tx, + ); + pending.send(false); + + const result = runtime.getCell( + parentCell.space, + { [builtinName]: { result: cause } }, + undefined, + tx, + ); + + const partial = runtime.getCell( + parentCell.space, + { [builtinName]: { partial: cause } }, + undefined, + tx, + ); + + const requestHash = runtime.getCell( + parentCell.space, + { [builtinName]: { requestHash: cause } }, + undefined, + tx, + ); + + return { pending, result, partial, requestHash }; +} + /** * Generate data via an LLM. * @@ -59,40 +108,17 @@ export function llm( return (tx: IExtendedStorageTransaction) => { if (!cellsInitialized) { - pending = runtime.getCell( - parentCell.space, - { llm: { pending: cause } }, - undefined, - tx, - ); - pending.send(false); - - result = runtime.getCell( - parentCell.space, - { - llm: { result: cause }, - }, - undefined, - tx, - ); - - partial = runtime.getCell( - parentCell.space, - { - llm: { partial: cause }, - }, - undefined, - tx, - ); - - requestHash = runtime.getCell( - parentCell.space, - { - llm: { requestHash: cause }, - }, - undefined, + const cells = initializeCells( + runtime, + parentCell, + cause, tx, + "llm", ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; sendResult(tx, { pending, result, partial, requestHash }); cellsInitialized = true; @@ -129,13 +155,15 @@ export function llm( if (hash === previousCallHash || hash === requestHashWithLog.get()) return; previousCallHash = hash; - resultWithLog.set(undefined); - partialWithLog.set(undefined); - if (!Array.isArray(messages) || messages.length === 0) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(false); return; } + + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(true); const updatePartial = (text: string) => { @@ -175,6 +203,151 @@ export function llm( partial.withTx(tx).set(undefined); }); + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + + // TODO(seefeld): Not writing now, so we retry the request after failure. + // Replace this with more fine-grained retry logic. + // requestHash.setAtPath([], hash, log); + }); + }; +} + +/** + * Generate text via an LLM. + * + * A simplified alternative to `llm` that takes a single prompt string and + * optional system message, returning plain text rather than a structured + * content array. + * + * Returns the complete result as `result` (string) and the incremental result + * as `partial` (string). `pending` is true while a request is pending. + * + * @param prompt - The user prompt/message to send to the LLM. + * @param system - Optional system message. + * @param model - Model to use (defaults to DEFAULT_MODEL_NAME). + * @param maxTokens - Maximum number of tokens to generate (defaults to 4096). + * + * @returns { pending: boolean, result?: string, partial?: string, requestHash?: string } - + * As individual docs, representing `pending` state, final `result` and + * incrementally updating `partial` result. + */ +export function generateText( + inputsCell: Cell, + sendResult: (tx: IExtendedStorageTransaction, result: any) => void, + _addCancel: (cancel: () => void) => void, + cause: any, + parentCell: Cell, + runtime: IRuntime, +): Action { + let currentRun = 0; + let previousCallHash: string | undefined = undefined; + let cellsInitialized = false; + let pending: Cell; + let result: Cell; + let partial: Cell; + let requestHash: Cell; + + return (tx: IExtendedStorageTransaction) => { + if (!cellsInitialized) { + const cells = initializeCells( + runtime, + parentCell, + cause, + tx, + "generateText", + ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; + + sendResult(tx, { pending, result, partial, requestHash }); + cellsInitialized = true; + } + const thisRun = ++currentRun; + const pendingWithLog = pending.withTx(tx); + const resultWithLog = result.withTx(tx); + const partialWithLog = partial.withTx(tx); + const requestHashWithLog = requestHash.withTx(tx); + + const { system, prompt, model, maxTokens } = + inputsCell.getAsQueryResult([], tx) ?? {}; + + // If no prompt is provided, don't make a request + if (!prompt) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); + pendingWithLog.set(false); + return; + } + + // Convert simple prompt to messages array format for LLM client + const llmParams: LLMRequest = { + system: system ?? "", + messages: [{ role: "user", content: prompt }], + stop: "", + maxTokens: maxTokens ?? 4096, + stream: true, + model: model ?? DEFAULT_MODEL_NAME, + metadata: { + context: "charm", + }, + cache: true, + }; + + const hash = refer(llmParams).toString(); + + // Return if the same request is being made again + if (hash === previousCallHash || hash === requestHashWithLog.get()) return; + previousCallHash = hash; + + resultWithLog.set(undefined); + partialWithLog.set(undefined); + pendingWithLog.set(true); + + const updatePartial = (text: string) => { + if (thisRun != currentRun) return; + const status = tx.status(); + if (status.status !== "ready") return; + + partialWithLog.set(text); + }; + + const resultPromise = client.sendRequest(llmParams, updatePartial); + + resultPromise + .then(async (llmResult) => { + if (thisRun !== currentRun) return; + + await runtime.idle(); + + // Extract text from the LLM response + const textResult = extractTextFromLLMResponse(llmResult); + + await runtime.editWithRetry((tx) => { + pending.withTx(tx).set(false); + result.withTx(tx).set(textResult); + partial.withTx(tx).set(textResult); + requestHash.withTx(tx).set(hash); + }); + }) + .catch(async (error) => { + if (thisRun !== currentRun) return; + + console.error("Error generating text", error); + + await runtime.idle(); + + await runtime.editWithRetry((tx) => { + pending.withTx(tx).set(false); + result.withTx(tx).set(undefined); + partial.withTx(tx).set(undefined); + }); + + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + // TODO(seefeld): Not writing now, so we retry the request after failure. // Replace this with more fine-grained retry logic. // requestHash.setAtPath([], hash, log); @@ -223,40 +396,17 @@ export function generateObject>( return (tx: IExtendedStorageTransaction) => { if (!cellsInitialized) { - pending = runtime.getCell( - parentCell.space, - { generateObject: { pending: cause } }, - undefined, - tx, - ); - pending.send(false); - - result = runtime.getCell( - parentCell.space, - { - generateObject: { result: cause }, - }, - undefined, - tx, - ); - - partial = runtime.getCell( - parentCell.space, - { - generateObject: { partial: cause }, - }, - undefined, - tx, - ); - - requestHash = runtime.getCell( - parentCell.space, - { - generateObject: { requestHash: cause }, - }, - undefined, + const cells = initializeCells( + runtime, + parentCell, + cause, tx, + "generateObject", ); + pending = cells.pending; + result = cells.result; + partial = cells.partial; + requestHash = cells.requestHash; sendResult(tx, { pending, result, partial, requestHash }); cellsInitialized = true; @@ -271,6 +421,8 @@ export function generateObject>( inputsCell.getAsQueryResult([], tx) ?? {}; if (!prompt || !schema) { + resultWithLog.set(undefined); + partialWithLog.set(undefined); pendingWithLog.set(false); return; } @@ -336,6 +488,9 @@ export function generateObject>( partial.withTx(tx).set(undefined); }); + // Reset previousCallHash to allow retry after error + previousCallHash = undefined; + // TODO(seefeld): Not writing now, so we retry the request after failure. // Replace this with more fine-grained retry logic. // requestHash.setAtPath([], hash, log);