adding a shared llm client wrapper that can be imported and used (#280)

Author: jakedahn

typescript/packages/toolshed/lib/llm.ts

@@ -0,0 +1,40 @@
+import { AppType } from "@/app.ts";
+import { hc } from "hono/client";
+
+// NOTE(jake): Ideally this would be exposed via the hono client, but I wasn't
+// able to get it all wired up. Importing the route definition is fine for now.
+import type { GetModelsRouteQueryParams } from "@/routes/ai/llm/llm.routes.ts";
+
+const client = hc<AppType>("http://localhost:8000/");
+
+export async function listAvailableModels({
+  capability,
+  task,
+  search,
+}: GetModelsRouteQueryParams) {
+  const res = await client.api.ai.llm.models.$get({
+    query: {
+      search,
+      capability,
+      task,
+    },
+  });
+  return res.json();
+}
+
+export async function generateText(
+  query: Parameters<typeof client.api.ai.llm.$post>[0]["json"],
+): Promise<string> {
+  const res = await client.api.ai.llm.$post({ json: query });
+  const data = await res.json();
+
+  if ("error" in data) {
+    throw new Error(data.error);
+  }
+
+  if ("type" in data && data.type === "json") {
+    return data.body.content;
+  }
+
+  throw new Error("Unexpected response format");
+}

typescript/packages/toolshed/routes/ai/llm/llm.routes.ts

@@ -6,10 +6,12 @@ import { z } from "zod";
 const tags = ["AI Language Models"];
 
 export const MessageSchema = z.object({
-  role: z.enum(["user", "assistant"]),
+  role: z.string(),
   content: z.string(),
 });
 
+export type LLMResponseMessage = z.infer<typeof MessageSchema>;
+
 export const LLMRequestSchema = z.object({
   messages: z.array(MessageSchema),
   system: z.string().optional(),
@@ -55,16 +57,24 @@ const JsonResponse = z.object({
   }),
 });
 
+export type LLMJSONResponse = z.infer<typeof JsonResponse>;
+
+const GetModelsRouteQueryParams = z.object({
+  search: z.string().optional(),
+  capability: z.string().optional(),
+  task: z.string().optional(),
+});
+
+export type GetModelsRouteQueryParams = z.infer<
+  typeof GetModelsRouteQueryParams
+>;
+
 // Route definitions
 export const getModels = createRoute({
   path: "/api/ai/llm/models",
   method: "get",
   tags,
-  query: z.object({
-    search: z.string().optional(),
-    capability: z.string().optional(),
-    task: z.string().optional(),
-  }),
+  query: GetModelsRouteQueryParams,
   responses: {
     [HttpStatusCodes.OK]: jsonContent(
       ModelsResponseSchema.openapi({

typescript/packages/toolshed/routes/ai/spell/behavior/effects.ts

@@ -4,20 +4,6 @@ import { handleResponse } from "@/lib/response.ts";
 
 const client = hc<AppType>("http://localhost:8000/");
 
-export async function generateText(
-  query: Parameters<typeof client.api.ai.llm.$post>[0]["json"],
-) {
-  const res = await client.api.ai.llm.$post({ json: query });
-  // cheating
-  // deno-lint-ignore no-explicit-any
-  const data = (await res.json() as any) as {
-    role: "assistant";
-    content: string;
-  };
-
-  return data.content;
-}
-
 export async function getAllBlobs(): Promise<string[]> {
   const res = await client.api.storage.blobby.$get({ query: { all: "true" } });
   const data = await res.json();

typescript/packages/toolshed/routes/ai/spell/behavior/strategies/scanByCollections.ts

@@ -1,6 +1,7 @@
 import { SearchResult } from "../search.ts";
 import { Logger, PrefixedLogger } from "@/lib/prefixed-logger.ts";
-import { generateText, getBlob } from "../effects.ts";
+import { getBlob } from "../effects.ts";
+import { generateText } from "@/lib/llm.ts";
 
 async function generateKeywords(
   query: string,
@@ -11,7 +12,7 @@ async function generateKeywords(
     model: "claude-3-5-sonnet",
     messages: [
       {
-        role: "user" as const,
+        role: "user",
         content: query,
       },
     ],

typescript/packages/toolshed/routes/ai/spell/behavior/strategies/scanBySchema.ts

@@ -2,7 +2,8 @@ import { checkSchemaMatch } from "@/lib/schema-match.ts";
 import { SearchResult } from "../search.ts";
 import { Logger, PrefixedLogger } from "@/lib/prefixed-logger.ts";
 import type { RedisClientType } from "redis";
-import { generateText, getAllBlobs, getBlob } from "../effects.ts";
+import { generateText } from "@/lib/llm.ts";
+import { getAllBlobs, getBlob } from "../effects.ts";
 import { Schema } from "jsonschema";
 
 export async function generateSchema(

typescript/packages/toolshed/routes/ai/spell/spell.handlers.ts

@@ -1,6 +1,7 @@
 import * as HttpStatusCodes from "stoker/http-status-codes";
 import { z } from "zod";
-import { generateText, getAllBlobs, getBlob } from "./behavior/effects.ts";
+import { getAllBlobs, getBlob } from "./behavior/effects.ts";
+import { generateText } from "@/lib/llm.ts";
 
 import type { AppRouteHandler } from "@/lib/types.ts";
 import type { ProcessSchemaRoute, SearchSchemaRoute } from "./spell.routes.ts";