diff --git a/packages/runner/src/builtins/llm-dialog.ts b/packages/runner/src/builtins/llm-dialog.ts index 56dada79d..8a86a288a 100644 --- a/packages/runner/src/builtins/llm-dialog.ts +++ b/packages/runner/src/builtins/llm-dialog.ts @@ -20,6 +20,10 @@ import { ID, NAME, type Recipe, TYPE } from "../builder/types.ts"; import type { Action } from "../scheduler.ts"; import type { IRuntime } from "../runtime.ts"; import type { IExtendedStorageTransaction } from "../storage/interface.ts"; +import { + debugTransactionWrites, + formatTransactionSummary, +} from "../storage/transaction-summary.ts"; import { parseLink } from "../link-utils.ts"; // Avoid importing from @commontools/charm to prevent circular deps in tests @@ -811,8 +815,13 @@ async function invokeToolCall( ...toolCall.input, result, // doesn't HAVE to be used, but can be }, (completedTx: IExtendedStorageTransaction) => { - resolve(result.withTx(completedTx).get()); // withTx likely superfluous - }); // TODO(bf): why any needed? + logger.info("Handler tx:", debugTransactionWrites(completedTx)); + + const summary = formatTransactionSummary(completedTx, space); + const value = result.withTx(completedTx).get(); + + resolve({ value, summary }); + }); } else { throw new Error("Tool has neither pattern nor handler"); } diff --git a/packages/runner/src/index.ts b/packages/runner/src/index.ts index 490690c39..78c648d0e 100644 --- a/packages/runner/src/index.ts +++ b/packages/runner/src/index.ts @@ -19,6 +19,12 @@ export type { IExtendedStorageTransaction, MemorySpace, } from "./storage/interface.ts"; +export { + debugTransactionWrites, + formatTransactionSummary, + summarizeTransaction, + type TransactionSummary, +} from "./storage/transaction-summary.ts"; export { convertCellsToLinks, isCell, isStream } from "./cell.ts"; export { getCellOrThrow, diff --git a/packages/runner/src/storage/transaction-summary.ts b/packages/runner/src/storage/transaction-summary.ts new file mode 100644 index 000000000..863046af7 --- /dev/null +++ b/packages/runner/src/storage/transaction-summary.ts @@ -0,0 +1,365 @@ +/** + * Transaction Summary - Condense transaction details for LLM consumption + * + * This module provides functions to extract and condense transaction information + * into concise summaries suitable for LLMs to help humans debug software behavior. + */ + +import type { + IExtendedStorageTransaction, + IMemorySpaceAddress, + ITransactionJournal, +} from "./interface.ts"; +import type { MemorySpace } from "../runtime.ts"; + +/** + * Condensed summary of a transaction suitable for LLM consumption + */ +export interface TransactionSummary { + /** Human-readable one-line summary */ + summary: string; + + /** Activity statistics */ + activity: { + reads: number; + writes: number; + }; + + /** Actual writes with values */ + writes: WriteDetail[]; +} + +/** + * Details of what was actually written + */ +export interface WriteDetail { + /** Object ID (shortened) */ + objectId: string; + + /** Full object ID */ + fullObjectId: string; + + /** Path that was written to */ + path: string; + + /** The value that was written */ + value: unknown; + + /** The previous value (if available) */ + previousValue?: unknown; + + /** Whether this was a deletion */ + isDeleted: boolean; +} + +/** + * Create a condensed transaction summary from an IExtendedStorageTransaction + * + * @param tx - The completed transaction + * @param space - Optional memory space to filter changes (defaults to first space found) + * @returns Condensed summary for LLM consumption + */ +export function summarizeTransaction( + tx: IExtendedStorageTransaction, + space?: MemorySpace, +): TransactionSummary { + const status = tx.status(); + const journal = status.journal; + + // Summarize activity + const activity = summarizeActivity(journal); + + // Extract actual writes with values + const writes = space ? extractWrites(journal, space) : []; + + // Generate summary + const summary = generateSummary(activity, writes, status.status); + + return { + summary, + activity, + writes, + }; +} + +/** + * Format transaction summary as a string for LLM consumption + * + * @param tx - The completed transaction + * @param space - Optional memory space to filter changes + * @returns Formatted string summary + */ +export function formatTransactionSummary( + tx: IExtendedStorageTransaction, + space?: MemorySpace, +): string { + const summary = summarizeTransaction(tx, space); + + const parts: string[] = []; + + // If there are detailed writes, format them grouped by object + if (summary.writes.length > 0) { + // Group writes by object + const byObject = new Map(); + for (const write of summary.writes) { + const existing = byObject.get(write.fullObjectId) || []; + existing.push(write); + byObject.set(write.fullObjectId, existing); + } + + const objectIds = Array.from(byObject.keys()); + + // If single object, skip the header + if (objectIds.length === 1) { + const writes = byObject.get(objectIds[0])!; + for (const write of writes) { + parts.push(formatWrite(write)); + } + } else { + // Multiple objects, show headers + for (const objectId of objectIds) { + const writes = byObject.get(objectId)!; + parts.push(`Object ${shortenId(objectId)}:`); + for (const write of writes) { + parts.push(` ${formatWrite(write)}`); + } + } + } + } else if (summary.activity.writes > 0 && !space) { + // Hint that we need the space parameter + parts.push("(pass space parameter to see what was written)"); + } else { + // No writes or writes occurred elsewhere - show generic summary + parts.push(summary.summary); + } + + // Add read count if significant + if (summary.activity.reads > 10) { + parts.push(`(${summary.activity.reads} reads for context)`); + } + + return parts.join("\n"); +} + +/** + * Format a single write as "path: old → new" or "path = value" + */ +function formatWrite(write: WriteDetail): string { + if (write.isDeleted) { + return `${write.path}: deleted`; + } + + const newVal = formatValueForSummary(write.value); + + // If we have previous value, show before → after + if (write.previousValue !== undefined) { + const oldVal = formatValueForSummary(write.previousValue); + return `${write.path}: ${oldVal} → ${newVal}`; + } + + // No previous value, just show assignment + return `${write.path} = ${newVal}`; +} + +/** + * Debug helper to see all write operations regardless of space + * Useful for understanding what's happening when writes aren't showing up + */ +export function debugTransactionWrites( + tx: IExtendedStorageTransaction, +): string { + const status = tx.status(); + const journal = status.journal; + + const parts: string[] = []; + parts.push("=== Transaction Debug ==="); + + // List all write operations from activity + const writes: IMemorySpaceAddress[] = []; + for (const activity of journal.activity()) { + if ("write" in activity && activity.write) { + writes.push(activity.write); + } + } + + parts.push(`Total writes in activity: ${writes.length}`); + + for (const write of writes) { + const pathStr = write.path.join("."); + parts.push(` Write to: ${write.id}/${pathStr} (space: ${write.space})`); + } + + // List all spaces that have novelty + parts.push("\nSpaces with novelty:"); + const spaces = new Set(); + for (const write of writes) { + spaces.add(write.space); + } + + for (const space of spaces) { + const noveltyCount = Array.from(journal.novelty(space)).length; + parts.push(` ${space}: ${noveltyCount} attestation(s)`); + } + + return parts.join("\n"); +} + +/** + * Summarize activity from transaction journal + */ +function summarizeActivity(journal: ITransactionJournal): { + reads: number; + writes: number; +} { + let reads = 0; + let writes = 0; + + for (const activity of journal.activity()) { + if ("read" in activity) { + reads++; + } else if ("write" in activity) { + writes++; + } + } + + return { reads, writes }; +} + +/** + * Extract actual writes with their values from novelty attestations + */ +function extractWrites( + journal: ITransactionJournal, + space: MemorySpace, +): WriteDetail[] { + // Build a map of previous values from history + const previousValues = new Map(); + for (const attestation of journal.history(space)) { + const key = `${attestation.address.id}:${ + attestation.address.path.join(".") + }`; + previousValues.set(key, attestation.value); + } + + const writes: WriteDetail[] = []; + + for (const attestation of journal.novelty(space)) { + const fullObjectId = attestation.address.id; + const path = attestation.address.path.join("."); + const value = attestation.value; + const isDeleted = value === undefined; + + const key = `${fullObjectId}:${path}`; + const previousValue = previousValues.get(key); + + writes.push({ + objectId: shortenId(fullObjectId), + fullObjectId, + path, + value: truncateValue(value, 100), + previousValue: truncateValue(previousValue, 100), + isDeleted, + }); + } + + return writes; +} + +/** + * Truncate a value for display + */ +function truncateValue(value: unknown, maxLength: number): unknown { + if (value === undefined) return undefined; + if (value === null) return null; + + if (typeof value === "string") { + return value.length > maxLength + ? value.substring(0, maxLength) + "..." + : value; + } + + if (Array.isArray(value)) { + return `[Array: ${value.length} items]`; + } + + if (typeof value === "object") { + const str = JSON.stringify(value); + return str.length > maxLength ? str.substring(0, maxLength) + "..." : value; + } + + return value; +} + +/** + * Generate a human-readable summary + */ +function generateSummary( + activity: { writes: number; reads: number }, + writes: WriteDetail[], + status: string, +): string { + if (status === "error") { + return "Transaction failed"; + } + + if (activity.writes === 0 && activity.reads === 0) { + return "Empty transaction"; + } + + if (activity.writes === 0) { + return "Read-only transaction"; + } + + if (writes.length === 0) { + return `${activity.writes} write(s) (details unavailable without space parameter)`; + } + + // Describe the actual writes + const parts: string[] = []; + + for (const write of writes.slice(0, 3)) { + if (write.isDeleted) { + parts.push(`Deleted ${write.path}`); + } else { + const valueStr = formatValueForSummary(write.value); + parts.push(`${write.path} = ${valueStr}`); + } + } + + if (writes.length > 3) { + parts.push(`... and ${writes.length - 3} more`); + } + + return parts.join("; "); +} + +/** + * Format a value for the summary line + */ +function formatValueForSummary(value: unknown): string { + if (value === undefined) return "undefined"; + if (value === null) return "null"; + if (typeof value === "string") { + const truncated = value.length > 50 + ? value.substring(0, 50) + "..." + : value; + return `"${truncated}"`; + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return JSON.stringify(value); +} + +/** + * Shorten an ID for display + */ +function shortenId(id: string): string { + if (id.startsWith("of:")) { + return id.substring(3, 15) + "..."; + } + if (id.length > 20) { + return id.substring(0, 20) + "..."; + } + return id; +} diff --git a/packages/runner/test/transaction-summary.test.ts b/packages/runner/test/transaction-summary.test.ts new file mode 100644 index 000000000..0a3a16829 --- /dev/null +++ b/packages/runner/test/transaction-summary.test.ts @@ -0,0 +1,156 @@ +import { assertEquals } from "@std/assert"; +import { describe, it } from "@std/testing/bdd"; +import { + formatTransactionSummary, + summarizeTransaction, +} from "../src/storage/transaction-summary.ts"; +import type { + IAttestation, + IExtendedStorageTransaction, + ITransactionJournal, +} from "../src/storage/interface.ts"; + +// Simple test transaction journal +class TestJournal implements ITransactionJournal { + constructor( + private noveltyData: IAttestation[] = [], + private historyData: IAttestation[] = [], + ) {} + + activity(): Iterable { + return []; + } + + novelty(_space: any): Iterable { + return this.noveltyData; + } + + history(_space: any): Iterable { + return this.historyData; + } +} + +// Helper to create test attestations +function attestation(id: string, path: string[], value?: any): IAttestation { + return { + address: { + id: id as any, // Cast to URI type + type: "application/json", + path, + }, + value, + }; +} + +// Simple test transaction +function createTestTransaction( + novelty: IAttestation[], + history: IAttestation[] = [], +): IExtendedStorageTransaction { + const journal = new TestJournal(novelty, history); + return { + journal, + status: () => ({ status: "done" as const, journal }), + } as any; +} + +describe("transaction-summary", () => { + it("should extract a single write with no previous value", () => { + const tx = createTestTransaction([ + attestation("of:abc123", ["value", "count"], 42), + ]); + + const summary = summarizeTransaction(tx, "did:key:test" as any); + + assertEquals(summary.writes.length, 1); + assertEquals(summary.writes[0].path, "value.count"); + assertEquals(summary.writes[0].value, 42); + assertEquals(summary.writes[0].previousValue, undefined); + assertEquals(summary.writes[0].isDeleted, false); + }); + + it("should extract a write with previous value (before → after)", () => { + const tx = createTestTransaction( + [attestation("of:abc123", ["value", "count"], 42)], + [attestation("of:abc123", ["value", "count"], 10)], + ); + + const summary = summarizeTransaction(tx, "did:key:test" as any); + + assertEquals(summary.writes.length, 1); + assertEquals(summary.writes[0].value, 42); + assertEquals(summary.writes[0].previousValue, 10); + }); + + it("should format a single new value without arrow", () => { + const tx = createTestTransaction([ + attestation("of:abc123", ["value", "count"], 1), + ]); + + const formatted = formatTransactionSummary(tx, "did:key:test" as any); + + assertEquals(formatted, "value.count = 1"); + }); + + it("should format a changed value with arrow", () => { + const tx = createTestTransaction( + [attestation("of:abc123", ["value", "count"], 5)], + [attestation("of:abc123", ["value", "count"], 3)], + ); + + const formatted = formatTransactionSummary(tx, "did:key:test" as any); + + assertEquals(formatted, "value.count: 3 → 5"); + }); + + it("should group multiple writes to same object", () => { + const tx = createTestTransaction( + [ + attestation("of:abc123", ["value", "count"], 5), + attestation("of:abc123", ["value", "title"], "Test"), + ], + [ + attestation("of:abc123", ["value", "count"], 3), + ], + ); + + const formatted = formatTransactionSummary(tx, "did:key:test" as any); + + assertEquals(formatted.includes("value.count: 3 → 5"), true); + assertEquals(formatted.includes('value.title = "Test"'), true); + }); + + it("should show object headers for multiple objects", () => { + const tx = createTestTransaction([ + attestation("of:abc123", ["value"], 1), + attestation("of:def456", ["value"], 2), + ]); + + const formatted = formatTransactionSummary(tx, "did:key:test" as any); + + assertEquals(formatted.includes("Object abc123"), true); + assertEquals(formatted.includes("Object def456"), true); + }); + + it("should handle deletions", () => { + const tx = createTestTransaction( + [attestation("of:abc123", ["value", "old"], undefined)], + [attestation("of:abc123", ["value", "old"], "previous")], + ); + + const summary = summarizeTransaction(tx, "did:key:test" as any); + + assertEquals(summary.writes.length, 1); + assertEquals(summary.writes[0].isDeleted, true); + }); + + it("should handle empty transaction", () => { + const tx = createTestTransaction([]); + + const summary = summarizeTransaction(tx, "did:key:test" as any); + assertEquals(summary.writes.length, 0); + + const formatted = formatTransactionSummary(tx, "did:key:test" as any); + assertEquals(formatted, "Empty transaction"); + }); +}); diff --git a/packages/shell/mise.toml b/packages/shell/mise.toml deleted file mode 100644 index ac7a209fc..000000000 --- a/packages/shell/mise.toml +++ /dev/null @@ -1,2 +0,0 @@ -[tools] -deno = "latest"