diff --git a/builder/src/env.ts b/builder/src/env.ts index 10734159b..db15a26e3 100644 --- a/builder/src/env.ts +++ b/builder/src/env.ts @@ -30,3 +30,15 @@ export function setRecipeEnvironment(env: RecipeEnvironment) { export function getRecipeEnvironment(): RecipeEnvironment { return globalEnv; } + +// until we thread the trace IDs through the entire workflow/recipe/... + +let lastTraceSpanID: string | undefined; + +export function setLastTraceSpanID(spanID: string) { + lastTraceSpanID = spanID; +} + +export function getLastTraceSpanID(): string | undefined { + return lastTraceSpanID; +} diff --git a/builder/src/index.ts b/builder/src/index.ts index 0604ce71a..df14301d6 100644 --- a/builder/src/index.ts +++ b/builder/src/index.ts @@ -10,8 +10,10 @@ export { render, } from "./module.ts"; export { + getLastTraceSpanID, getRecipeEnvironment, type RecipeEnvironment, + setLastTraceSpanID, setRecipeEnvironment, } from "./env.ts"; export { diff --git a/charm/src/iterate.ts b/charm/src/iterate.ts index b93248ccf..f3c594f6f 100644 --- a/charm/src/iterate.ts +++ b/charm/src/iterate.ts @@ -29,7 +29,7 @@ export const genSrc = async ({ schema, steps, model, - generationId + generationId, }: { src?: string; spec?: string; @@ -88,7 +88,7 @@ export async function iterate( schema: iframe.argumentSchema, steps: plan?.steps, model, - generationId + generationId, }); return generateNewRecipeVersion(charmManager, charm, newIFrameSrc, newSpec); @@ -337,7 +337,7 @@ async function twoPhaseCodeGeneration( newSpec, schema, steps: form.plan?.steps, - generationId: form.meta.generationId + generationId: form.meta.generationId, }); const name = extractTitle(newIFrameSrc, title); // Use the generated title as fallback const newRecipeSrc = buildFullRecipe({ diff --git a/jumble/src/components/ActionBar.tsx b/jumble/src/components/ActionBar.tsx index 3dc870bc3..b8efb56e8 100644 --- a/jumble/src/components/ActionBar.tsx +++ b/jumble/src/components/ActionBar.tsx @@ -39,7 +39,7 @@ export function ActionBar() {
@@ -52,7 +52,7 @@ export function ActionBar() { return (
diff --git a/jumble/src/components/FeedbackActions.tsx b/jumble/src/components/FeedbackActions.tsx new file mode 100644 index 000000000..20645eceb --- /dev/null +++ b/jumble/src/components/FeedbackActions.tsx @@ -0,0 +1,118 @@ +import { useEffect, useState } from "react"; +import { FeedbackDialog } from "@/components/FeedbackDialog.tsx"; +import { submitFeedback } from "@/services/feedback.ts"; +import { getLastTraceSpanID } from "@commontools/builder"; +import { MdThumbDownOffAlt, MdThumbUpOffAlt } from "react-icons/md"; + +export function FeedbackActions() { + // States for the feedback dialog + const [isFeedbackDialogOpen, setIsFeedbackDialogOpen] = useState(false); + const [isSubmitting, setIsSubmitting] = useState(false); + const [initialScore, setInitialScore] = useState
(1); + const [showFeedbackButtons, setShowFeedbackButtons] = useState(false); + + // Listen for toggle-feedback event + useEffect(() => { + const handleToggleFeedback = () => { + setShowFeedbackButtons((prev) => !prev); + }; + + globalThis.addEventListener("toggle-feedback", handleToggleFeedback); + return () => { + globalThis.removeEventListener("toggle-feedback", handleToggleFeedback); + }; + }, []); + + // Handler functions for feedback actions + const handleOpenFeedback = (score: number) => { + console.log( + `Opening ${score === 1 ? "positive" : "negative"} feedback dialog`, + ); + setInitialScore(score); + setIsFeedbackDialogOpen(true); + setShowFeedbackButtons(false); + }; + + const handleCloseFeedback = () => { + setIsFeedbackDialogOpen(false); + }; + + const handleSubmitFeedback = async ( + data: { score: number; explanation: string; userInfo?: any }, + ) => { + console.log("Submitting feedback:", data); + setIsSubmitting(true); + + try { + await submitFeedback( + { + score: data.score, + explanation: data.explanation, + spanId: getLastTraceSpanID() as string, + }, + data.userInfo, + ); + + setIsFeedbackDialogOpen(false); + alert("Feedback submitted successfully! Thank you for your input."); + } catch (error) { + alert( + `Error submitting feedback: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } finally { + setIsSubmitting(false); + } + }; + + // Render feedback dialog and the popup buttons + return ( + <> + {/* Feedback dialog for submissions */} +
+ + {/* Popup buttons that appear when feedback button is clicked */} + {showFeedbackButtons && ( +
+ {/* This spacer places the thumbs buttons just above the feedback button */} +
+ + {/* Thumbs down button */} +
+
handleOpenFeedback(0)} + > +
+
+ Not Helpful +
+
+
+ + {/* Thumbs up button */} +
+
handleOpenFeedback(1)} + > +
+
+ Helpful +
+
+
+
+ )} + + ); +} diff --git a/jumble/src/components/FeedbackDialog.tsx b/jumble/src/components/FeedbackDialog.tsx new file mode 100644 index 000000000..8762ed383 --- /dev/null +++ b/jumble/src/components/FeedbackDialog.tsx @@ -0,0 +1,139 @@ +import { useEffect, useState } from "react"; +import { fetchUserInfo } from "@/services/feedback.ts"; + +interface FeedbackDialogProps { + isOpen: boolean; + onClose: () => void; + onSubmit: ( + data: { score: number; explanation: string; userInfo?: any }, + ) => Promise
; + initialScore: number; + isSubmitting?: boolean; +} + +export function FeedbackDialog({ + isOpen, + onClose, + onSubmit, + initialScore, + isSubmitting = false, +}: FeedbackDialogProps) { + const [explanation, setExplanation] = useState(""); + + // Reset explanation when dialog opens with a new score + useEffect(() => { + if (isOpen) { + setExplanation(""); + } + }, [isOpen, initialScore]); + + // Log when dialog open state changes + useEffect(() => { + console.log( + "FeedbackDialog isOpen:", + isOpen, + "initialScore:", + initialScore, + ); + }, [isOpen, initialScore]); + + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (!isOpen) return; + + if (e.key === "Escape") { + onClose(); + } else if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { + e.preventDefault(); + if (!isSubmitting && explanation.trim()) { + submitFeedback(); + } + } + }; + + globalThis.addEventListener("keydown", handleKeyDown); + return () => globalThis.removeEventListener("keydown", handleKeyDown); + }, [isOpen, onClose, onSubmit, explanation, isSubmitting, initialScore]); + + if (!isOpen) return null; + + const submitFeedback = async () => { + if (!explanation.trim()) return; + + console.log("Submitting feedback form with explanation:", explanation); + + // Fetch user info only when submitting + try { + const userInfo = await fetchUserInfo(); + console.log("Fetched user info before submission:", userInfo); + + // Submit feedback with the user info + await onSubmit({ + score: initialScore, + explanation, + userInfo, + }); + } catch (error) { + console.error("Error fetching user info during submission:", error); + // Continue with submission even if user info fetch fails + await onSubmit({ score: initialScore, explanation }); + } + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + submitFeedback(); + }; + + return ( +
+
+
+ {initialScore === 1 ? "Positive Feedback" : "Improvement Feedback"} +
+
+
+
+ {initialScore === 1 + ? "What did you like about this response?" + : "How could this response be improved?"} +
+
setExplanation(e.target.value)} + className="w-full px-3 py-2 border-2 border-black" + rows={5} + disabled={isSubmitting} + placeholder={initialScore === 1 + ? "What aspects were helpful or well done?" + : "What would make this response more useful?"} + required + /> + </div> + + <div className="flex justify-end gap-2"> + <button + type="button" + onClick={onClose} + className="px-4 py-2 border-2 border-black hover:bg-gray-100 disabled:opacity-50 disabled:cursor-not-allowed" + disabled={isSubmitting} + > + Cancel + </button> + <button + type="submit" + className="px-4 py-2 bg-black text-white border-2 border-black hover:bg-gray-800 disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-2" + disabled={isSubmitting || !explanation.trim()} + > + {isSubmitting ? "Submitting..." : ( + <span> + Submit <span className="text-xs">cmd+enter</span> + </span> + )} + </button> + </div> + </form> + </div> + </div> + ); +} diff --git a/jumble/src/contexts/ActionManagerContext.tsx b/jumble/src/contexts/ActionManagerContext.tsx index 182fc4bcd..975ad09f0 100644 --- a/jumble/src/contexts/ActionManagerContext.tsx +++ b/jumble/src/contexts/ActionManagerContext.tsx @@ -24,6 +24,7 @@ export type Action = { priority?: number; to?: string; // For NavLink actions keyCombo?: KeyCombo; // Keyboard shortcut + className?: string; // Optional CSS class for styling the button }; type ActionManagerContextType = { diff --git a/jumble/src/global.d.ts b/jumble/src/global.d.ts index 076eb3c6e..0cb52ac8b 100644 --- a/jumble/src/global.d.ts +++ b/jumble/src/global.d.ts @@ -56,4 +56,7 @@ declare module "react-icons/md" { export const MdOutlineStar: React.FC<IconProps>; export const MdOutlineStarBorder: React.FC<IconProps>; export const MdShare: React.FC<IconProps>; + export const MdSend: React.FC<IconProps>; + export const MdThumbDownOffAlt: React.FC<IconProps>; + export const MdThumbUpOffAlt: React.FC<IconProps>; } diff --git a/jumble/src/hooks/use-global-actions.tsx b/jumble/src/hooks/use-global-actions.tsx index 74b7308fb..dcb1efae1 100644 --- a/jumble/src/hooks/use-global-actions.tsx +++ b/jumble/src/hooks/use-global-actions.tsx @@ -1,7 +1,7 @@ import "@commontools/ui"; import { useLocation, useParams } from "react-router-dom"; import { type CharmRouteParams } from "@/routes.ts"; -import { MdEdit, MdOutlineStar, MdShare } from "react-icons/md"; +import { MdEdit, MdOutlineStar, MdSend, MdShare } from "react-icons/md"; import { useAction } from "@/contexts/ActionManagerContext.tsx"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -92,4 +92,21 @@ export function useGlobalActions() { [hasCharmId, togglePath], ), ); + + // Feedback action (available only when a charm is open) + useAction( + useMemo( + () => ({ + id: "feedback-toggle", + label: "Feedback", + icon: <MdSend size={24} />, + onClick: () => { + globalThis.dispatchEvent(new CustomEvent("toggle-feedback")); + }, + priority: 30, + predicate: hasCharmId, + }), + [hasCharmId], + ), + ); } diff --git a/jumble/src/services/feedback.ts b/jumble/src/services/feedback.ts new file mode 100644 index 000000000..9fd9ae32a --- /dev/null +++ b/jumble/src/services/feedback.ts @@ -0,0 +1,75 @@ +interface UserInfo { + name: string; + email: string; + shortName: string; + avatar: string; +} + +interface FeedbackData { + score: number; + explanation: string; + spanId: string; +} + +export async function fetchUserInfo(): Promise<UserInfo | null> { + try { + console.log("Fetching user info from /api/whoami..."); + const response = await fetch("/api/whoami"); + if (!response.ok) { + throw new Error( + `Failed to fetch user info: ${response.status} ${response.statusText}`, + ); + } + const data = await response.json(); + console.log("User info fetched successfully:", data); + return data; + } catch (error) { + console.error("Error fetching user info:", error); + return null; + } +} + +export async function submitFeedback( + data: FeedbackData, + userInfo: UserInfo | null, +): Promise<boolean> { + try { + const payload = { + span_id: data.spanId, + name: userInfo?.email || "anonymous@user.com", + annotator_kind: "HUMAN", + result: { + label: "user_feedback", + score: data.score, + explanation: data.explanation, + }, + }; + + console.log("Sending feedback payload:", JSON.stringify(payload, null, 2)); + + const response = await fetch("/api/ai/llm/feedback", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + console.log("Feedback API response status:", response.status); + + if (!response.ok) { + const errorData = await response.json(); + console.error("Feedback API error response:", errorData); + throw new Error( + errorData.error || `Failed to submit feedback: ${response.status}`, + ); + } + + const responseData = await response.json(); + console.log("Feedback API success response:", responseData); + return true; + } catch (error) { + console.error("Error submitting feedback:", error); + throw error; + } +} diff --git a/jumble/src/views/Shell.tsx b/jumble/src/views/Shell.tsx index 8f259f262..6d34826ee 100644 --- a/jumble/src/views/Shell.tsx +++ b/jumble/src/views/Shell.tsx @@ -12,6 +12,7 @@ import { useGlobalActions } from "@/hooks/use-global-actions.tsx"; import { SyncStatusProvider } from "@/contexts/SyncStatusContext.tsx"; import { ToggleableNetworkInspector } from "@/components/NetworkInspector.tsx"; import { NetworkInspectorProvider } from "@/contexts/NetworkInspectorContext.tsx"; +import { FeedbackActions } from "@/components/FeedbackActions.tsx"; export default function Shell() { const { charmId } = useParams<CharmRouteParams>(); @@ -36,6 +37,7 @@ export default function Shell() { <ActionBar /> <CharmPublisher /> <CommandCenter /> + <FeedbackActions /> <ToggleableNetworkInspector visible={localStorage.getItem("networkInspectorVisible") === "true"} diff --git a/jumble/vite.config.ts b/jumble/vite.config.ts index 533717ff5..bbf6e4e71 100644 --- a/jumble/vite.config.ts +++ b/jumble/vite.config.ts @@ -18,6 +18,7 @@ export default defineConfig({ "localhost", "127.0.0.1", "bens-macbook-pro.saga-castor.ts.net", + "jake.saga-castor.ts.net", ], proxy: { "/api/integrations": { diff --git a/llm/src/client.ts b/llm/src/client.ts index 320c8c460..e98b70854 100644 --- a/llm/src/client.ts +++ b/llm/src/client.ts @@ -1,4 +1,5 @@ import { LlmPrompt } from "./prompts/prompting.ts"; +import { setLastTraceSpanID } from "@commontools/builder"; export type SimpleMessage = { role: "user" | "assistant"; @@ -76,13 +77,17 @@ export class LLMClient { throw new Error("No response body"); } + const traceSpanID = response.headers.get("x-ct-llm-trace-id") as string; + if (traceSpanID) { + setLastTraceSpanID(traceSpanID); + } + // the server might return cached data instead of a stream if (response.headers.get("content-type") === "application/json") { const data = (await response.json()) as SimpleMessage; // FIXME(ja): can the LLM ever return anything other than a string? return data.content as string; } - // FIXME(ja): this doesn't handle falling back to other models // if we fail during streaming return await this.stream(response.body, partialCB); diff --git a/toolshed/env.ts b/toolshed/env.ts index f3c49a985..71d762383 100644 --- a/toolshed/env.ts +++ b/toolshed/env.ts @@ -44,6 +44,7 @@ const EnvSchema = z.object({ // LLM Observability Tool CTTS_AI_LLM_PHOENIX_PROJECT: z.string().default(""), CTTS_AI_LLM_PHOENIX_URL: z.string().default(""), + CTTS_AI_LLM_PHOENIX_API_URL: z.string().default(""), CTTS_AI_LLM_PHOENIX_API_KEY: z.string().default(""), // =========================================================================== diff --git a/toolshed/routes/ai/llm/generateText.ts b/toolshed/routes/ai/llm/generateText.ts index ec6b90f10..90cd2cf8d 100644 --- a/toolshed/routes/ai/llm/generateText.ts +++ b/toolshed/routes/ai/llm/generateText.ts @@ -1,4 +1,5 @@ import { streamText } from "ai"; +import { trace } from "@opentelemetry/api"; import { findModel, TASK_MODELS } from "./models.ts"; @@ -36,6 +37,7 @@ export interface GenerateTextResult { message: { role: "user" | "assistant"; content: string }; messages: { role: "user" | "assistant"; content: string }[]; stream?: ReadableStream; + spanId?: string; } // Configure the model parameters for JSON mode based on provider @@ -202,8 +204,12 @@ export async function generateText( metadata: params.metadata, }; + // This is where the LLM API call is made const llmStream = await streamText(streamParams as any); + // Get the active span ID from OpenTelemetry + const spanId = trace.getActiveSpan()?.spanContext().spanId; + // If not streaming, handle regular response if (!params.stream) { let result = ""; @@ -237,6 +243,7 @@ export async function generateText( return { message: messages[messages.length - 1], messages: [...messages], + spanId, }; } @@ -300,5 +307,6 @@ export async function generateText( message: messages[messages.length - 1], messages: [...messages], stream, + spanId, }; } diff --git a/toolshed/routes/ai/llm/instrumentation.ts b/toolshed/routes/ai/llm/instrumentation.ts index b4e786611..61d98c888 100644 --- a/toolshed/routes/ai/llm/instrumentation.ts +++ b/toolshed/routes/ai/llm/instrumentation.ts @@ -27,7 +27,13 @@ export function register() { }, }), spanFilter: (span) => { - return isOpenInferenceSpan(span); + // console.log("SPAN", span); + const includeSpanCriteria = [ + isOpenInferenceSpan(span), + span.attributes["http.route"] == "/api/ai/llm", // Include the root span, which is in the hono app + span.instrumentationLibrary.name.includes("@vercel/otel"), // Include the actual LLM API fetch span + ]; + return includeSpanCriteria.some((c) => c); }, }), ], diff --git a/toolshed/routes/ai/llm/llm.handlers.ts b/toolshed/routes/ai/llm/llm.handlers.ts index b55e7df90..f1f60bbe3 100644 --- a/toolshed/routes/ai/llm/llm.handlers.ts +++ b/toolshed/routes/ai/llm/llm.handlers.ts @@ -1,11 +1,16 @@ import * as HttpStatusCodes from "stoker/http-status-codes"; import type { AppRouteHandler } from "@/lib/types.ts"; -import type { GenerateTextRoute, GetModelsRoute } from "./llm.routes.ts"; +import type { + FeedbackRoute, + GenerateTextRoute, + GetModelsRoute, +} from "./llm.routes.ts"; import { ALIAS_NAMES, ModelList, MODELS, TASK_MODELS } from "./models.ts"; import * as cache from "./cache.ts"; import type { Context } from "@hono/hono"; import { generateText as generateTextCore } from "./generateText.ts"; import { findModel } from "./models.ts"; +import env from "@/env.ts"; const withoutMetadata = (obj: any) => { const { metadata, ...rest } = obj; @@ -160,7 +165,11 @@ export const generateText: AppRouteHandler<GenerateTextRoute> = async (c) => { // If response is not streaming, save to cache and return the message if (!payload.stream) { await persistCache(result.messages); - return c.json(result.message); + const response = c.json(result.message); + if (result.spanId) { + response.headers.set("x-ct-llm-trace-id", result.spanId); + } + return response; } return new Response(result.stream, { @@ -169,6 +178,7 @@ export const generateText: AppRouteHandler<GenerateTextRoute> = async (c) => { "Transfer-Encoding": "chunked", "Cache-Control": "no-cache", Connection: "keep-alive", + ...(result.spanId ? { "x-ct-llm-trace-id": result.spanId } : {}), }, }); } catch (error) { @@ -177,3 +187,51 @@ export const generateText: AppRouteHandler<GenerateTextRoute> = async (c) => { return c.json({ error: message }, HttpStatusCodes.BAD_REQUEST); } }; + +/** + * Handler for POST /feedback endpoint + * Submits user feedback on an LLM response to Phoenix + */ +export const submitFeedback: AppRouteHandler<FeedbackRoute> = async (c) => { + const payload = await c.req.json(); + + try { + const phoenixPayload = { + data: [ + { + span_id: payload.span_id, + name: payload.name || "user feedback", + annotator_kind: payload.annotator_kind || "HUMAN", + result: payload.result, + metadata: payload.metadata || {}, + }, + ], + }; + + const phoenixAnnotationPayload = { + method: "POST", + headers: { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": `Bearer ${env.CTTS_AI_LLM_PHOENIX_API_KEY}`, + }, + body: JSON.stringify(phoenixPayload), + }; + + const response = await fetch( + `${env.CTTS_AI_LLM_PHOENIX_API_URL}/span_annotations?sync=false`, + phoenixAnnotationPayload, + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Phoenix API error: ${response.status} ${errorText}`); + } + + return c.json({ success: true }, HttpStatusCodes.OK); + } catch (error) { + console.error("Error submitting feedback:", error); + const message = error instanceof Error ? error.message : "Unknown error"; + return c.json({ error: message }, HttpStatusCodes.BAD_REQUEST); + } +}; diff --git a/toolshed/routes/ai/llm/llm.index.ts b/toolshed/routes/ai/llm/llm.index.ts index dddcf21ba..43579f367 100644 --- a/toolshed/routes/ai/llm/llm.index.ts +++ b/toolshed/routes/ai/llm/llm.index.ts @@ -11,7 +11,8 @@ if (env.CTTS_AI_LLM_PHOENIX_PROJECT) { const router = createRouter() .openapi(routes.getModels, handlers.getModels) - .openapi(routes.generateText, handlers.generateText); + .openapi(routes.generateText, handlers.generateText) + .openapi(routes.feedback, handlers.submitFeedback); router.use( "/api/ai/llm/*", diff --git a/toolshed/routes/ai/llm/llm.routes.ts b/toolshed/routes/ai/llm/llm.routes.ts index 2ff1b2345..7c1d556c1 100644 --- a/toolshed/routes/ai/llm/llm.routes.ts +++ b/toolshed/routes/ai/llm/llm.routes.ts @@ -72,6 +72,18 @@ export type GetModelsRouteQueryParams = z.infer< typeof GetModelsRouteQueryParams >; +export const FeedbackSchema = z.object({ + span_id: z.string(), + name: z.string().default("user feedback"), + annotator_kind: z.enum(["HUMAN", "LLM"]).default("HUMAN"), + result: z.object({ + label: z.string().optional(), + score: z.number().optional(), + explanation: z.string().optional(), + }), + metadata: z.record(z.unknown()).optional(), +}); + // Route definitions export const getModels = createRoute({ path: "/api/ai/llm/models", @@ -157,5 +169,54 @@ export const generateText = createRoute({ }, }); +export const feedback = createRoute({ + path: "/api/ai/llm/feedback", + method: "post", + tags, + request: { + body: { + content: { + "application/json": { + schema: FeedbackSchema.openapi({ + example: { + span_id: "67f6740bbe1ddc3f", + name: "correctness", + annotator_kind: "HUMAN", + result: { + label: "correct", + score: 1, + explanation: "The response answered the question I asked", + }, + }, + }), + }, + }, + }, + }, + responses: { + [HttpStatusCodes.OK]: jsonContent( + z.object({ + success: z.boolean(), + }).openapi({ + example: { + success: true, + }, + }), + "Feedback submitted successfully", + ), + [HttpStatusCodes.BAD_REQUEST]: { + content: { + "application/json": { + schema: z.object({ + error: z.string(), + }), + }, + }, + description: "Invalid request parameters", + }, + }, +}); + export type GetModelsRoute = typeof getModels; export type GenerateTextRoute = typeof generateText; +export type FeedbackRoute = typeof feedback;