From 552ab591e6f5e056834fea2af6e8981d356824f9 Mon Sep 17 00:00:00 2001 From: jakedahn
Date: Fri, 11 Apr 2025 12:59:32 -0600 Subject: [PATCH 01/12] getting started on user feedback --- toolshed/env.ts | 1 + toolshed/routes/ai/llm/generateText.ts | 8 +++ toolshed/routes/ai/llm/instrumentation.ts | 8 ++- toolshed/routes/ai/llm/llm.handlers.ts | 68 +++++++++++++++++++++-- toolshed/routes/ai/llm/llm.index.ts | 3 +- toolshed/routes/ai/llm/llm.routes.ts | 61 ++++++++++++++++++++ 6 files changed, 141 insertions(+), 8 deletions(-) diff --git a/toolshed/env.ts b/toolshed/env.ts index f3c49a9856..71d7623834 100644 --- a/toolshed/env.ts +++ b/toolshed/env.ts @@ -44,6 +44,7 @@ const EnvSchema = z.object({ // LLM Observability Tool CTTS_AI_LLM_PHOENIX_PROJECT: z.string().default(""), CTTS_AI_LLM_PHOENIX_URL: z.string().default(""), + CTTS_AI_LLM_PHOENIX_API_URL: z.string().default(""), CTTS_AI_LLM_PHOENIX_API_KEY: z.string().default(""), // =========================================================================== diff --git a/toolshed/routes/ai/llm/generateText.ts b/toolshed/routes/ai/llm/generateText.ts index ec6b90f106..90cd2cf8de 100644 --- a/toolshed/routes/ai/llm/generateText.ts +++ b/toolshed/routes/ai/llm/generateText.ts @@ -1,4 +1,5 @@ import { streamText } from "ai"; +import { trace } from "@opentelemetry/api"; import { findModel, TASK_MODELS } from "./models.ts"; @@ -36,6 +37,7 @@ export interface GenerateTextResult { message: { role: "user" | "assistant"; content: string }; messages: { role: "user" | "assistant"; content: string }[]; stream?: ReadableStream; + spanId?: string; } // Configure the model parameters for JSON mode based on provider @@ -202,8 +204,12 @@ export async function generateText( metadata: params.metadata, }; + // This is where the LLM API call is made const llmStream = await streamText(streamParams as any); + // Get the active span ID from OpenTelemetry + const spanId = trace.getActiveSpan()?.spanContext().spanId; + // If not streaming, handle regular response if (!params.stream) { let result = ""; @@ -237,6 +243,7 @@ export async function generateText( return { message: messages[messages.length - 1], messages: [...messages], + spanId, }; } @@ -300,5 +307,6 @@ export async function generateText( message: messages[messages.length - 1], messages: [...messages], stream, + spanId, }; } diff --git a/toolshed/routes/ai/llm/instrumentation.ts b/toolshed/routes/ai/llm/instrumentation.ts index b4e7866118..61d98c8887 100644 --- a/toolshed/routes/ai/llm/instrumentation.ts +++ b/toolshed/routes/ai/llm/instrumentation.ts @@ -27,7 +27,13 @@ export function register() { }, }), spanFilter: (span) => { - return isOpenInferenceSpan(span); + // console.log("SPAN", span); + const includeSpanCriteria = [ + isOpenInferenceSpan(span), + span.attributes["http.route"] == "/api/ai/llm", // Include the root span, which is in the hono app + span.instrumentationLibrary.name.includes("@vercel/otel"), // Include the actual LLM API fetch span + ]; + return includeSpanCriteria.some((c) => c); }, }), ], diff --git a/toolshed/routes/ai/llm/llm.handlers.ts b/toolshed/routes/ai/llm/llm.handlers.ts index b55e7df907..51afe63b08 100644 --- a/toolshed/routes/ai/llm/llm.handlers.ts +++ b/toolshed/routes/ai/llm/llm.handlers.ts @@ -1,11 +1,16 @@ import * as HttpStatusCodes from "stoker/http-status-codes"; import type { AppRouteHandler } from "@/lib/types.ts"; -import type { GenerateTextRoute, GetModelsRoute } from "./llm.routes.ts"; +import type { + FeedbackRoute, + GenerateTextRoute, + GetModelsRoute, +} from "./llm.routes.ts"; import { ALIAS_NAMES, ModelList, MODELS, TASK_MODELS } from "./models.ts"; import * as cache from "./cache.ts"; import type { Context } from "@hono/hono"; import { generateText as generateTextCore } from "./generateText.ts"; import { findModel } from "./models.ts"; +import env from "@/env.ts"; const withoutMetadata = (obj: any) => { const { metadata, ...rest } = obj; @@ -113,10 +118,10 @@ export const generateText: AppRouteHandler
= async (c) => { JSON.stringify(withoutMetadata(payload)), ); const cachedResult = await cache.loadItem(cacheKey); - if (cachedResult) { - const lastMessage = cachedResult.messages[cachedResult.messages.length - 1]; - return c.json(lastMessage); - } + // if (cachedResult) { + // const lastMessage = cachedResult.messages[cachedResult.messages.length - 1]; + // return c.json(lastMessage); + // } const persistCache = async ( messages: { role: string; content: string }[], @@ -160,7 +165,11 @@ export const generateText: AppRouteHandler
= async (c) => { // If response is not streaming, save to cache and return the message if (!payload.stream) { await persistCache(result.messages); - return c.json(result.message); + const response = c.json(result.message); + if (result.spanId) { + response.headers.set("x-ct-llm-trace-id", result.spanId); + } + return response; } return new Response(result.stream, { @@ -169,6 +178,7 @@ export const generateText: AppRouteHandler
= async (c) => { "Transfer-Encoding": "chunked", "Cache-Control": "no-cache", Connection: "keep-alive", + ...(result.spanId ? { "x-ct-llm-trace-id": result.spanId } : {}), }, }); } catch (error) { @@ -177,3 +187,49 @@ export const generateText: AppRouteHandler
= async (c) => { return c.json({ error: message }, HttpStatusCodes.BAD_REQUEST); } }; + +/** + * Handler for POST /feedback endpoint + * Submits user feedback on an LLM response to Phoenix + */ +export const submitFeedback: AppRouteHandler
= async (c) => { + const payload = await c.req.json(); + + try { + const phoenixPayload = { + data: [ + { + span_id: payload.span_id, + name: payload.name || "user feedback", + annotator_kind: payload.annotator_kind || "HUMAN", + result: payload.result, + metadata: payload.metadata || {}, + }, + ], + }; + + const response = await fetch( + `${env.CTTS_AI_LLM_PHOENIX_API_URL}/span_annotations?sync=false`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + "api_key": env.CTTS_AI_LLM_PHOENIX_API_KEY, + }, + body: JSON.stringify(phoenixPayload), + }, + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Phoenix API error: ${response.status} ${errorText}`); + } + + return c.json({ success: true }); + } catch (error) { + console.error("Error submitting feedback:", error); + const message = error instanceof Error ? error.message : "Unknown error"; + return c.json({ error: message }, HttpStatusCodes.BAD_REQUEST); + } +}; diff --git a/toolshed/routes/ai/llm/llm.index.ts b/toolshed/routes/ai/llm/llm.index.ts index dddcf21baf..43579f367c 100644 --- a/toolshed/routes/ai/llm/llm.index.ts +++ b/toolshed/routes/ai/llm/llm.index.ts @@ -11,7 +11,8 @@ if (env.CTTS_AI_LLM_PHOENIX_PROJECT) { const router = createRouter() .openapi(routes.getModels, handlers.getModels) - .openapi(routes.generateText, handlers.generateText); + .openapi(routes.generateText, handlers.generateText) + .openapi(routes.feedback, handlers.submitFeedback); router.use( "/api/ai/llm/*", diff --git a/toolshed/routes/ai/llm/llm.routes.ts b/toolshed/routes/ai/llm/llm.routes.ts index 2ff1b2345a..7c1d556c1d 100644 --- a/toolshed/routes/ai/llm/llm.routes.ts +++ b/toolshed/routes/ai/llm/llm.routes.ts @@ -72,6 +72,18 @@ export type GetModelsRouteQueryParams = z.infer< typeof GetModelsRouteQueryParams >; +export const FeedbackSchema = z.object({ + span_id: z.string(), + name: z.string().default("user feedback"), + annotator_kind: z.enum(["HUMAN", "LLM"]).default("HUMAN"), + result: z.object({ + label: z.string().optional(), + score: z.number().optional(), + explanation: z.string().optional(), + }), + metadata: z.record(z.unknown()).optional(), +}); + // Route definitions export const getModels = createRoute({ path: "/api/ai/llm/models", @@ -157,5 +169,54 @@ export const generateText = createRoute({ }, }); +export const feedback = createRoute({ + path: "/api/ai/llm/feedback", + method: "post", + tags, + request: { + body: { + content: { + "application/json": { + schema: FeedbackSchema.openapi({ + example: { + span_id: "67f6740bbe1ddc3f", + name: "correctness", + annotator_kind: "HUMAN", + result: { + label: "correct", + score: 1, + explanation: "The response answered the question I asked", + }, + }, + }), + }, + }, + }, + }, + responses: { + [HttpStatusCodes.OK]: jsonContent( + z.object({ + success: z.boolean(), + }).openapi({ + example: { + success: true, + }, + }), + "Feedback submitted successfully", + ), + [HttpStatusCodes.BAD_REQUEST]: { + content: { + "application/json": { + schema: z.object({ + error: z.string(), + }), + }, + }, + description: "Invalid request parameters", + }, + }, +}); + export type GetModelsRoute = typeof getModels; export type GenerateTextRoute = typeof generateText; +export type FeedbackRoute = typeof feedback; From d451a2cbcda48157a83e798b9884014ef904a852 Mon Sep 17 00:00:00 2001 From: jakedahn
Date: Mon, 14 Apr 2025 12:15:53 -0600 Subject: [PATCH 02/12] fixing feedback endpoint schema --- toolshed/routes/ai/llm/llm.handlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolshed/routes/ai/llm/llm.handlers.ts b/toolshed/routes/ai/llm/llm.handlers.ts index 51afe63b08..3718be8510 100644 --- a/toolshed/routes/ai/llm/llm.handlers.ts +++ b/toolshed/routes/ai/llm/llm.handlers.ts @@ -226,7 +226,7 @@ export const submitFeedback: AppRouteHandler
= async (c) => { throw new Error(`Phoenix API error: ${response.status} ${errorText}`); } - return c.json({ success: true }); + return c.json({ success: true }, HttpStatusCodes.OK); } catch (error) { console.error("Error submitting feedback:", error); const message = error instanceof Error ? error.message : "Unknown error"; From fcb2e22db7b64d29c1f1a1881cca785055cb7b6e Mon Sep 17 00:00:00 2001 From: jakedahn
Date: Mon, 14 Apr 2025 12:50:33 -0600 Subject: [PATCH 03/12] fixing up the phoenix annotation request --- toolshed/routes/ai/llm/llm.handlers.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/toolshed/routes/ai/llm/llm.handlers.ts b/toolshed/routes/ai/llm/llm.handlers.ts index 3718be8510..4f0a4e119c 100644 --- a/toolshed/routes/ai/llm/llm.handlers.ts +++ b/toolshed/routes/ai/llm/llm.handlers.ts @@ -208,17 +208,19 @@ export const submitFeedback: AppRouteHandler
= async (c) => { ], }; + const phoenixAnnotationPayload = { + method: "POST", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": `Bearer ${env.CTTS_AI_LLM_PHOENIX_API_KEY}`, + }, + body: JSON.stringify(phoenixPayload), + }; + const response = await fetch( `${env.CTTS_AI_LLM_PHOENIX_API_URL}/span_annotations?sync=false`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - "Accept": "application/json", - "api_key": env.CTTS_AI_LLM_PHOENIX_API_KEY, - }, - body: JSON.stringify(phoenixPayload), - }, + phoenixAnnotationPayload, ); if (!response.ok) { From e15a1ba77f470362e7a3eff3f2c829d74982e8d5 Mon Sep 17 00:00:00 2001 From: jakedahn
Date: Mon, 14 Apr 2025 14:38:45 -0600 Subject: [PATCH 04/12] wiring up a feedback form --- jumble/src/components/ActionBar.tsx | 4 +- jumble/src/components/FeedbackActions.tsx | 139 +++++++++++++++++++ jumble/src/components/FeedbackDialog.tsx | 139 +++++++++++++++++++ jumble/src/contexts/ActionManagerContext.tsx | 1 + jumble/src/services/feedback.ts | 75 ++++++++++ jumble/src/views/Shell.tsx | 2 + jumble/vite.config.ts | 1 + 7 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 jumble/src/components/FeedbackActions.tsx create mode 100644 jumble/src/components/FeedbackDialog.tsx create mode 100644 jumble/src/services/feedback.ts diff --git a/jumble/src/components/ActionBar.tsx b/jumble/src/components/ActionBar.tsx index 3dc870bc3b..b8efb56e82 100644 --- a/jumble/src/components/ActionBar.tsx +++ b/jumble/src/components/ActionBar.tsx @@ -39,7 +39,7 @@ export function ActionBar() {
@@ -52,7 +52,7 @@ export function ActionBar() { return (
diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx new file mode 100644 index 0000000000..79905c96ba --- /dev/null +++ b/jumble/src/components/FeedbackActions.tsx @@ -0,0 +1,139 @@ +import { useEffect, useState } from "react"; +import { Action, useActionManager } from "@/contexts/ActionManagerContext.tsx"; +import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; +import { submitFeedback } from "@/services/feedback.ts"; +import { MdSend, MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; + +// For demo purposes - in a real app, this would come from the current response or context +const CURRENT_SPAN_ID = "48fc49c695cdc4f3"; + +export function FeedbackActions() { + // States for the feedback dialog + const [isFeedbackDialogOpen, setIsFeedbackDialogOpen] = useState(false); + const [isSubmitting, setIsSubmitting] = useState(false); + const [initialScore, setInitialScore] = useState
(1); + const [showFeedbackButtons, setShowFeedbackButtons] = useState(false); + const { registerAction } = useActionManager(); + + // Handler functions for feedback actions + const toggleFeedbackButtons = () => { + setShowFeedbackButtons((prev) => !prev); + }; + + const handleOpenFeedback = (score: number) => { + console.log( + `Opening ${score === 1 ? "positive" : "negative"} feedback dialog`, + ); + setInitialScore(score); + setIsFeedbackDialogOpen(true); + // Hide the feedback buttons when opening the modal + setShowFeedbackButtons(false); + }; + + const handleCloseFeedback = () => { + setIsFeedbackDialogOpen(false); + }; + + const handleSubmitFeedback = async ( + data: { score: number; explanation: string; userInfo?: any }, + ) => { + console.log("Submitting feedback:", data); + setIsSubmitting(true); + + try { + await submitFeedback( + { + score: data.score, + explanation: data.explanation, + spanId: CURRENT_SPAN_ID, + }, + data.userInfo, + ); + + setIsFeedbackDialogOpen(false); + alert("Feedback submitted successfully! Thank you for your input."); + } catch (error) { + alert( + `Error submitting feedback: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } finally { + setIsSubmitting(false); + } + }; + + // Register all actions + useEffect(() => { + // Create actions array for batch registration + const actions: Action[] = []; + + // Register the main feedback toggle button + const toggleAction: Action = { + id: "feedback-toggle", + label: "Feedback", + icon:
, + onClick: toggleFeedbackButtons, + priority: 30, + className: showFeedbackButtons ? "bg-blue-100" : "", + }; + + actions.push(toggleAction); + + // Register all actions and collect unregister functions + const unregisterFunctions = actions.map((action) => registerAction(action)); + + // Return combined cleanup function + return () => { + unregisterFunctions.forEach((unregister) => unregister()); + }; + }, [showFeedbackButtons, registerAction]); + + // Render feedback dialog and the popup buttons + return ( + <> + {/* Feedback dialog for submissions */} +
+ + {/* Popup buttons that appear when feedback button is clicked */} + {showFeedbackButtons && ( +
+ {/* This spacer accounts for the main button height */} +
+ + {/* Thumbs down button */} +
+
handleOpenFeedback(0)} + > +
+
+ Not Helpful +
+
+
+ + {/* Thumbs up button */} +
+
handleOpenFeedback(1)} + > +
+
+ Helpful +
+
+
+
+ )} + + ); +} diff --git a/jumble/src/components/FeedbackDialog.tsx b/jumble/src/components/FeedbackDialog.tsx new file mode 100644 index 0000000000..8762ed383f --- /dev/null +++ b/jumble/src/components/FeedbackDialog.tsx @@ -0,0 +1,139 @@ +import { useEffect, useState } from "react"; +import { fetchUserInfo } from "@/services/feedback.ts"; + +interface FeedbackDialogProps { + isOpen: boolean; + onClose: () => void; + onSubmit: ( + data: { score: number; explanation: string; userInfo?: any }, + ) => Promise
; + initialScore: number; + isSubmitting?: boolean; +} + +export function FeedbackDialog({ + isOpen, + onClose, + onSubmit, + initialScore, + isSubmitting = false, +}: FeedbackDialogProps) { + const [explanation, setExplanation] = useState(""); + + // Reset explanation when dialog opens with a new score + useEffect(() => { + if (isOpen) { + setExplanation(""); + } + }, [isOpen, initialScore]); + + // Log when dialog open state changes + useEffect(() => { + console.log( + "FeedbackDialog isOpen:", + isOpen, + "initialScore:", + initialScore, + ); + }, [isOpen, initialScore]); + + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (!isOpen) return; + + if (e.key === "Escape") { + onClose(); + } else if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { + e.preventDefault(); + if (!isSubmitting && explanation.trim()) { + submitFeedback(); + } + } + }; + + globalThis.addEventListener("keydown", handleKeyDown); + return () => globalThis.removeEventListener("keydown", handleKeyDown); + }, [isOpen, onClose, onSubmit, explanation, isSubmitting, initialScore]); + + if (!isOpen) return null; + + const submitFeedback = async () => { + if (!explanation.trim()) return; + + console.log("Submitting feedback form with explanation:", explanation); + + // Fetch user info only when submitting + try { + const userInfo = await fetchUserInfo(); + console.log("Fetched user info before submission:", userInfo); + + // Submit feedback with the user info + await onSubmit({ + score: initialScore, + explanation, + userInfo, + }); + } catch (error) { + console.error("Error fetching user info during submission:", error); + // Continue with submission even if user info fetch fails + await onSubmit({ score: initialScore, explanation }); + } + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + submitFeedback(); + }; + + return ( +
+
+
+ {initialScore === 1 ? "Positive Feedback" : "Improvement Feedback"} +
+
+
+
+ {initialScore === 1 + ? "What did you like about this response?" + : "How could this response be improved?"} +
+
setExplanation(e.target.value)} + className="w-full px-3 py-2 border-2 border-black" + rows={5} + disabled={isSubmitting} + placeholder={initialScore === 1 + ? "What aspects were helpful or well done?" + : "What would make this response more useful?"} + required + /> + </div> + + <div className="flex justify-end gap-2"> + <button + type="button" + onClick={onClose} + className="px-4 py-2 border-2 border-black hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed" + disabled={isSubmitting} + > + Cancel + </button> + <button + type="submit" + className="px-4 py-2 bg-black text-white border-2 border-black hover:bg-gray-800 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2" + disabled={isSubmitting || !explanation.trim()} + > + {isSubmitting ? "Submitting..." : ( + <span> + Submit <span className="text-xs">cmd+enter</span> + </span> + )} + </button> + </div> + </form> + </div> + </div> + ); +} diff --git a/jumble/src/contexts/ActionManagerContext.tsx b/jumble/src/contexts/ActionManagerContext.tsx index 182fc4bcd2..975ad09f05 100644 --- a/jumble/src/contexts/ActionManagerContext.tsx +++ b/jumble/src/contexts/ActionManagerContext.tsx @@ -24,6 +24,7 @@ export type Action = { priority?: number; to?: string; // For NavLink actions keyCombo?: KeyCombo; // Keyboard shortcut + className?: string; // Optional CSS class for styling the button }; type ActionManagerContextType = { diff --git a/jumble/src/services/feedback.ts b/jumble/src/services/feedback.ts new file mode 100644 index 0000000000..9fd9ae32a1 --- /dev/null +++ b/jumble/src/services/feedback.ts @@ -0,0 +1,75 @@ +interface UserInfo { + name: string; + email: string; + shortName: string; + avatar: string; +} + +interface FeedbackData { + score: number; + explanation: string; + spanId: string; +} + +export async function fetchUserInfo(): Promise<UserInfo | null> { + try { + console.log("Fetching user info from /api/whoami..."); + const response = await fetch("/api/whoami"); + if (!response.ok) { + throw new Error( + `Failed to fetch user info: ${response.status} ${response.statusText}`, + ); + } + const data = await response.json(); + console.log("User info fetched successfully:", data); + return data; + } catch (error) { + console.error("Error fetching user info:", error); + return null; + } +} + +export async function submitFeedback( + data: FeedbackData, + userInfo: UserInfo | null, +): Promise<boolean> { + try { + const payload = { + span_id: data.spanId, + name: userInfo?.email || "anonymous@user.com", + annotator_kind: "HUMAN", + result: { + label: "user_feedback", + score: data.score, + explanation: data.explanation, + }, + }; + + console.log("Sending feedback payload:", JSON.stringify(payload, null, 2)); + + const response = await fetch("/api/ai/llm/feedback", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + console.log("Feedback API response status:", response.status); + + if (!response.ok) { + const errorData = await response.json(); + console.error("Feedback API error response:", errorData); + throw new Error( + errorData.error || `Failed to submit feedback: ${response.status}`, + ); + } + + const responseData = await response.json(); + console.log("Feedback API success response:", responseData); + return true; + } catch (error) { + console.error("Error submitting feedback:", error); + throw error; + } +} diff --git a/jumble/src/views/Shell.tsx b/jumble/src/views/Shell.tsx index 8f259f2622..6d34826ee9 100644 --- a/jumble/src/views/Shell.tsx +++ b/jumble/src/views/Shell.tsx @@ -12,6 +12,7 @@ import { useGlobalActions } from "@/hooks/use-global-actions.tsx"; import { SyncStatusProvider } from "@/contexts/SyncStatusContext.tsx"; import { ToggleableNetworkInspector } from "@/components/NetworkInspector.tsx"; import { NetworkInspectorProvider } from "@/contexts/NetworkInspectorContext.tsx"; +import { FeedbackActions } from "@/components/FeedbackActions.tsx"; export default function Shell() { const { charmId } = useParams<CharmRouteParams>(); @@ -36,6 +37,7 @@ export default function Shell() { <ActionBar /> <CharmPublisher /> <CommandCenter /> + <FeedbackActions /> <ToggleableNetworkInspector visible={localStorage.getItem("networkInspectorVisible") === "true"} diff --git a/jumble/vite.config.ts b/jumble/vite.config.ts index 533717ff56..bbf6e4e719 100644 --- a/jumble/vite.config.ts +++ b/jumble/vite.config.ts @@ -18,6 +18,7 @@ export default defineConfig({ "localhost", "127.0.0.1", "bens-macbook-pro.saga-castor.ts.net", + "jake.saga-castor.ts.net", ], proxy: { "/api/integrations": { From a858c9d80d74c2a0c64181375a623a554ba82899 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 14:48:17 -0600 Subject: [PATCH 05/12] minor cleanup --- jumble/src/components/FeedbackActions.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index 79905c96ba..40bb0182d6 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -4,7 +4,8 @@ import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; import { submitFeedback } from "@/services/feedback.ts"; import { MdSend, MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; -// For demo purposes - in a real app, this would come from the current response or context +// FIXME(jake): This is for demo purposes... ideally we could just get the llm +// span from the persisted blobby blob of the charm recipe, but none of that is hooked up yet. const CURRENT_SPAN_ID = "48fc49c695cdc4f3"; export function FeedbackActions() { @@ -26,7 +27,6 @@ export function FeedbackActions() { ); setInitialScore(score); setIsFeedbackDialogOpen(true); - // Hide the feedback buttons when opening the modal setShowFeedbackButtons(false); }; @@ -104,12 +104,13 @@ export function FeedbackActions() { {/* Popup buttons that appear when feedback button is clicked */} {showFeedbackButtons && ( <div className="fixed z-[100] bottom-2 right-2 flex flex-col-reverse gap-2 pointer-events-none"> - {/* This spacer accounts for the main button height */} - <div className="h-16"></div> + {/* This spacer places the thumbs buttons just above the feedback button */} + <div className="h-12"></div> {/* Thumbs down button */} <div className="pointer-events-auto"> <button + type="button" className="w-12 h-12 cursor-pointer flex items-center justify-center border-2 border-black shadow-[2px_2px_0px_0px_rgba(0,0,0,0.5)] bg-red-50 hover:translate-y-[-2px] relative group" onClick={() => handleOpenFeedback(0)} > @@ -123,6 +124,7 @@ export function FeedbackActions() { {/* Thumbs up button */} <div className="pointer-events-auto"> <button + type="button" className="w-12 h-12 cursor-pointer flex items-center justify-center border-2 border-black shadow-[2px_2px_0px_0px_rgba(0,0,0,0.5)] bg-green-50 hover:translate-y-[-2px] relative group" onClick={() => handleOpenFeedback(1)} > From 6cb119442fe796e82a18001c4580208207da0669 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 14:49:54 -0600 Subject: [PATCH 06/12] re-enable cache --- toolshed/routes/ai/llm/llm.handlers.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/toolshed/routes/ai/llm/llm.handlers.ts b/toolshed/routes/ai/llm/llm.handlers.ts index 4f0a4e119c..f1f60bbe36 100644 --- a/toolshed/routes/ai/llm/llm.handlers.ts +++ b/toolshed/routes/ai/llm/llm.handlers.ts @@ -118,10 +118,10 @@ export const generateText: AppRouteHandler<GenerateTextRoute> = async (c) => { JSON.stringify(withoutMetadata(payload)), ); const cachedResult = await cache.loadItem(cacheKey); - // if (cachedResult) { - // const lastMessage = cachedResult.messages[cachedResult.messages.length - 1]; - // return c.json(lastMessage); - // } + if (cachedResult) { + const lastMessage = cachedResult.messages[cachedResult.messages.length - 1]; + return c.json(lastMessage); + } const persistCache = async ( messages: { role: string; content: string }[], From 15c5b03436edd18220d1115e086373aa73c785e6 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 15:34:09 -0600 Subject: [PATCH 07/12] wire up last span id to feedback buttons --- charm/src/iterate.ts | 6 +++--- jumble/src/components/FeedbackActions.tsx | 12 ++++++++++-- llm/src/client.ts | 6 +++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/charm/src/iterate.ts b/charm/src/iterate.ts index b93248ccff..f3c594f6ff 100644 --- a/charm/src/iterate.ts +++ b/charm/src/iterate.ts @@ -29,7 +29,7 @@ export const genSrc = async ({ schema, steps, model, - generationId + generationId, }: { src?: string; spec?: string; @@ -88,7 +88,7 @@ export async function iterate( schema: iframe.argumentSchema, steps: plan?.steps, model, - generationId + generationId, }); return generateNewRecipeVersion(charmManager, charm, newIFrameSrc, newSpec); @@ -337,7 +337,7 @@ async function twoPhaseCodeGeneration( newSpec, schema, steps: form.plan?.steps, - generationId: form.meta.generationId + generationId: form.meta.generationId, }); const name = extractTitle(newIFrameSrc, title); // Use the generated title as fallback const newRecipeSrc = buildFullRecipe({ diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index 40bb0182d6..07df99af15 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -6,7 +6,15 @@ import { MdSend, MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; // FIXME(jake): This is for demo purposes... ideally we could just get the llm // span from the persisted blobby blob of the charm recipe, but none of that is hooked up yet. -const CURRENT_SPAN_ID = "48fc49c695cdc4f3"; +// const CURRENT_SPAN_ID = "48fc49c695cdc4f3"; + +const getCurrentSpanID = (): string => { + const traceSpanID = localStorage.getItem("traceSpanID"); + if (!traceSpanID) { + return ""; + } + return traceSpanID as string; +}; export function FeedbackActions() { // States for the feedback dialog @@ -45,7 +53,7 @@ export function FeedbackActions() { { score: data.score, explanation: data.explanation, - spanId: CURRENT_SPAN_ID, + spanId: getCurrentSpanID(), }, data.userInfo, ); diff --git a/llm/src/client.ts b/llm/src/client.ts index 320c8c460c..d4128ecf58 100644 --- a/llm/src/client.ts +++ b/llm/src/client.ts @@ -76,13 +76,17 @@ export class LLMClient { throw new Error("No response body"); } + const traceSpanID = response.headers.get("x-ct-llm-trace-id") as string; + if (traceSpanID) { + localStorage.setItem("traceSpanID", traceSpanID); + } + // the server might return cached data instead of a stream if (response.headers.get("content-type") === "application/json") { const data = (await response.json()) as SimpleMessage; // FIXME(ja): can the LLM ever return anything other than a string? return data.content as string; } - // FIXME(ja): this doesn't handle falling back to other models // if we fail during streaming return await this.stream(response.body, partialCB); From 5894973751999e1b204a0aa0b3fd7e467c919586 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 15:55:23 -0600 Subject: [PATCH 08/12] fixing react-icons check and removing fixme comment --- jumble/src/components/FeedbackActions.tsx | 4 ---- jumble/src/global.d.ts | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index 07df99af15..70ad71e247 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -4,10 +4,6 @@ import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; import { submitFeedback } from "@/services/feedback.ts"; import { MdSend, MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; -// FIXME(jake): This is for demo purposes... ideally we could just get the llm -// span from the persisted blobby blob of the charm recipe, but none of that is hooked up yet. -// const CURRENT_SPAN_ID = "48fc49c695cdc4f3"; - const getCurrentSpanID = (): string => { const traceSpanID = localStorage.getItem("traceSpanID"); if (!traceSpanID) { diff --git a/jumble/src/global.d.ts b/jumble/src/global.d.ts index 076eb3c6e3..0cb52ac8b0 100644 --- a/jumble/src/global.d.ts +++ b/jumble/src/global.d.ts @@ -56,4 +56,7 @@ declare module "react-icons/md" { export const MdOutlineStar: React.FC<IconProps>; export const MdOutlineStarBorder: React.FC<IconProps>; export const MdShare: React.FC<IconProps>; + export const MdSend: React.FC<IconProps>; + export const MdThumbDownOffAlt: React.FC<IconProps>; + export const MdThumbUpOffAlt: React.FC<IconProps>; } From 6dbd2c6ad5128a90faaf3db14cff22561ee688e1 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 16:07:33 -0600 Subject: [PATCH 09/12] fixing feedback button registration --- jumble/src/components/FeedbackActions.tsx | 46 +++++++---------------- jumble/src/hooks/use-global-actions.tsx | 18 ++++++++- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index 70ad71e247..f74c659492 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -1,8 +1,7 @@ import { useEffect, useState } from "react"; -import { Action, useActionManager } from "@/contexts/ActionManagerContext.tsx"; import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; import { submitFeedback } from "@/services/feedback.ts"; -import { MdSend, MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; +import { MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; const getCurrentSpanID = (): string => { const traceSpanID = localStorage.getItem("traceSpanID"); @@ -18,13 +17,20 @@ export function FeedbackActions() { const [isSubmitting, setIsSubmitting] = useState(false); const [initialScore, setInitialScore] = useState<number>(1); const [showFeedbackButtons, setShowFeedbackButtons] = useState(false); - const { registerAction } = useActionManager(); - // Handler functions for feedback actions - const toggleFeedbackButtons = () => { - setShowFeedbackButtons((prev) => !prev); - }; + // Listen for toggle-feedback event + useEffect(() => { + const handleToggleFeedback = () => { + setShowFeedbackButtons((prev) => !prev); + }; + globalThis.addEventListener("toggle-feedback", handleToggleFeedback); + return () => { + globalThis.removeEventListener("toggle-feedback", handleToggleFeedback); + }; + }, []); + + // Handler functions for feedback actions const handleOpenFeedback = (score: number) => { console.log( `Opening ${score === 1 ? "positive" : "negative"} feedback dialog`, @@ -67,32 +73,6 @@ export function FeedbackActions() { } }; - // Register all actions - useEffect(() => { - // Create actions array for batch registration - const actions: Action[] = []; - - // Register the main feedback toggle button - const toggleAction: Action = { - id: "feedback-toggle", - label: "Feedback", - icon: <MdSend size={24} />, - onClick: toggleFeedbackButtons, - priority: 30, - className: showFeedbackButtons ? "bg-blue-100" : "", - }; - - actions.push(toggleAction); - - // Register all actions and collect unregister functions - const unregisterFunctions = actions.map((action) => registerAction(action)); - - // Return combined cleanup function - return () => { - unregisterFunctions.forEach((unregister) => unregister()); - }; - }, [showFeedbackButtons, registerAction]); - // Render feedback dialog and the popup buttons return ( <> diff --git a/jumble/src/hooks/use-global-actions.tsx b/jumble/src/hooks/use-global-actions.tsx index 74b7308fbc..e6e34b2bd2 100644 --- a/jumble/src/hooks/use-global-actions.tsx +++ b/jumble/src/hooks/use-global-actions.tsx @@ -1,7 +1,7 @@ import "@commontools/ui"; import { useLocation, useParams } from "react-router-dom"; import { type CharmRouteParams } from "@/routes.ts"; -import { MdEdit, MdOutlineStar, MdShare } from "react-icons/md"; +import { MdEdit, MdOutlineStar, MdSend, MdShare } from "react-icons/md"; import { useAction } from "@/contexts/ActionManagerContext.tsx"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -33,6 +33,22 @@ export function useGlobalActions() { ), ); + // Feedback action (always available) + useAction( + useMemo( + () => ({ + id: "feedback-toggle", + label: "Feedback", + icon: <MdSend size={24} />, + onClick: () => { + globalThis.dispatchEvent(new CustomEvent("toggle-feedback")); + }, + priority: 30, + }), + [], + ), + ); + const hasCharmId = useCallback(() => Boolean(charmId), [charmId]); const { charmManager } = useCharmManager(); From d71ad6267e61404376d90acacc904b3a90e41ca6 Mon Sep 17 00:00:00 2001 From: jakedahn <jake@common.tools> Date: Mon, 14 Apr 2025 16:13:34 -0600 Subject: [PATCH 10/12] only show feedback button on charm pages --- jumble/src/hooks/use-global-actions.tsx | 33 +++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/jumble/src/hooks/use-global-actions.tsx b/jumble/src/hooks/use-global-actions.tsx index e6e34b2bd2..dcb1efae1e 100644 --- a/jumble/src/hooks/use-global-actions.tsx +++ b/jumble/src/hooks/use-global-actions.tsx @@ -33,22 +33,6 @@ export function useGlobalActions() { ), ); - // Feedback action (always available) - useAction( - useMemo( - () => ({ - id: "feedback-toggle", - label: "Feedback", - icon: <MdSend size={24} />, - onClick: () => { - globalThis.dispatchEvent(new CustomEvent("toggle-feedback")); - }, - priority: 30, - }), - [], - ), - ); - const hasCharmId = useCallback(() => Boolean(charmId), [charmId]); const { charmManager } = useCharmManager(); @@ -108,4 +92,21 @@ export function useGlobalActions() { [hasCharmId, togglePath], ), ); + + // Feedback action (available only when a charm is open) + useAction( + useMemo( + () => ({ + id: "feedback-toggle", + label: "Feedback", + icon: <MdSend size={24} />, + onClick: () => { + globalThis.dispatchEvent(new CustomEvent("toggle-feedback")); + }, + priority: 30, + predicate: hasCharmId, + }), + [hasCharmId], + ), + ); } From 1f4939ccfcc6c2e2c72d320ba6f8650e0eb2f21b Mon Sep 17 00:00:00 2001 From: Jesse Andrews <anotherjesse@gmail.com> Date: Mon, 14 Apr 2025 18:21:31 -0400 Subject: [PATCH 11/12] use globalThis instead of localStorage to support multiple tabs --- jumble/src/components/FeedbackActions.tsx | 3 ++- llm/src/client.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index f74c659492..0ca05d80fc 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -4,7 +4,8 @@ import { submitFeedback } from "@/services/feedback.ts"; import { MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; const getCurrentSpanID = (): string => { - const traceSpanID = localStorage.getItem("traceSpanID"); + // @ts-ignore: we set the lastTraceSpanID in llm client + const traceSpanID = globalThis.lastTraceSpanID; if (!traceSpanID) { return ""; } diff --git a/llm/src/client.ts b/llm/src/client.ts index d4128ecf58..96d375282c 100644 --- a/llm/src/client.ts +++ b/llm/src/client.ts @@ -78,7 +78,8 @@ export class LLMClient { const traceSpanID = response.headers.get("x-ct-llm-trace-id") as string; if (traceSpanID) { - localStorage.setItem("traceSpanID", traceSpanID); + // @ts-ignore: this is a hack until we send this through workflow + globalThis.lastTraceSpanID = traceSpanID; } // the server might return cached data instead of a stream From 74e3cc4bfac018a1ffa5586ce14f850397d1052c Mon Sep 17 00:00:00 2001 From: Jesse Andrews <anotherjesse@gmail.com> Date: Mon, 14 Apr 2025 18:43:58 -0400 Subject: [PATCH 12/12] use builder env to store last trace --- builder/src/env.ts | 12 ++++++++++++ builder/src/index.ts | 2 ++ jumble/src/components/FeedbackActions.tsx | 12 ++---------- llm/src/client.ts | 4 ++-- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/builder/src/env.ts b/builder/src/env.ts index 10734159bd..db15a26e3b 100644 --- a/builder/src/env.ts +++ b/builder/src/env.ts @@ -30,3 +30,15 @@ export function setRecipeEnvironment(env: RecipeEnvironment) { export function getRecipeEnvironment(): RecipeEnvironment { return globalEnv; } + +// until we thread the trace IDs through the entire workflow/recipe/... + +let lastTraceSpanID: string | undefined; + +export function setLastTraceSpanID(spanID: string) { + lastTraceSpanID = spanID; +} + +export function getLastTraceSpanID(): string | undefined { + return lastTraceSpanID; +} diff --git a/builder/src/index.ts b/builder/src/index.ts index 0604ce71a0..df14301d6e 100644 --- a/builder/src/index.ts +++ b/builder/src/index.ts @@ -10,8 +10,10 @@ export { render, } from "./module.ts"; export { + getLastTraceSpanID, getRecipeEnvironment, type RecipeEnvironment, + setLastTraceSpanID, setRecipeEnvironment, } from "./env.ts"; export { diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx index 0ca05d80fc..20645ecebf 100644 --- a/jumble/src/components/FeedbackActions.tsx +++ b/jumble/src/components/FeedbackActions.tsx @@ -1,17 +1,9 @@ import { useEffect, useState } from "react"; import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; import { submitFeedback } from "@/services/feedback.ts"; +import { getLastTraceSpanID } from "@commontools/builder"; import { MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; -const getCurrentSpanID = (): string => { - // @ts-ignore: we set the lastTraceSpanID in llm client - const traceSpanID = globalThis.lastTraceSpanID; - if (!traceSpanID) { - return ""; - } - return traceSpanID as string; -}; - export function FeedbackActions() { // States for the feedback dialog const [isFeedbackDialogOpen, setIsFeedbackDialogOpen] = useState(false); @@ -56,7 +48,7 @@ export function FeedbackActions() { { score: data.score, explanation: data.explanation, - spanId: getCurrentSpanID(), + spanId: getLastTraceSpanID() as string, }, data.userInfo, ); diff --git a/llm/src/client.ts b/llm/src/client.ts index 96d375282c..e98b708546 100644 --- a/llm/src/client.ts +++ b/llm/src/client.ts @@ -1,4 +1,5 @@ import { LlmPrompt } from "./prompts/prompting.ts"; +import { setLastTraceSpanID } from "@commontools/builder"; export type SimpleMessage = { role: "user" | "assistant"; @@ -78,8 +79,7 @@ export class LLMClient { const traceSpanID = response.headers.get("x-ct-llm-trace-id") as string; if (traceSpanID) { - // @ts-ignore: this is a hack until we send this through workflow - globalThis.lastTraceSpanID = traceSpanID; + setLastTraceSpanID(traceSpanID); } // the server might return cached data instead of a stream