/ai/spell/search (toolshed) (#277)

* Stub search functionality

* Stubbed search implementation

* Implement three basic strategy ideas for search

* Extract strategies and refactor logging

* Extract LLM library functions

* Factor out all redis code into lib

* Format pass

* Introduce Logger interface

* Improve docstrings

* Add types for redis

* One more format pass

* Default cors() settings clipper can reach blobby

* Fix lint errors

* Reverse refactor, use hono/client

Mostly taming errors unsure if anything works

* Restore state of world

* Fix exported router types

* Get all effects working again with hono/client

* Format pass

* Fix lint

* adding a shared llm client wrapper that can be imported and used (#280)

* Fix llm call

* Fixing errors so I can run the tests

* Basic smoke tests for spell endpoints

* Feels like this shouldn't work but it does

* Comment out tests

Need to work out how to handle LLM in CI

* Add explaination

---------

Co-authored-by: Jake Dahn <jake@common.tools>

Author: bfollington

typescript/packages/toolshed/deno.json

@@ -1,6 +1,6 @@
 {
   "tasks": {
-    "dev": "deno run -A --watch index.ts",
+    "dev": "deno run -A --watch --env-file=.env index.ts",
     "test": "deno test -A --env-file=.env.test",
     "build-lookslike": "deno run -A scripts/build-lookslike.ts"
   },
@@ -18,7 +18,7 @@
     "rules": {
       "tags": ["recommended"],
       "include": ["ban-untagged-todo"],
-      "exclude": ["no-unused-vars"]
+      "exclude": ["no-unused-vars", "no-explicit-any"]
     }
   },
   "nodeModulesDir": "auto",
@@ -55,6 +55,7 @@
     "pino-pretty": "npm:pino-pretty@^13.0.0",
     "redis": "npm:redis@^4.7.0",
     "stoker": "npm:stoker@^1.4.2",
-    "zod": "npm:zod@^3.24.1"
+    "zod": "npm:zod@^3.24.1",
+    "mistreevous": "npm:mistreevous@4.2.0"
   }
 }

typescript/packages/toolshed/deno.lock

@@ -0,0 +1,39 @@
+import { State } from "mistreevous";
+import { Logger, PrefixedLogger } from "./prefixed-logger.ts";
+
+// Handles logging and instrumentation
+export abstract class BaseAgent {
+  protected logger: PrefixedLogger;
+  protected agentName: string;
+  protected stepDurations: Record<string, number> = {};
+  protected logs: string[] = [];
+  protected startTime: number = 0;
+
+  constructor(logger: Logger, agentName: string) {
+    this.agentName = agentName;
+    this.logger = new PrefixedLogger(logger, agentName);
+    this.startTime = Date.now();
+  }
+
+  protected async measureStep(
+    stepName: string,
+    fn: () => Promise<State>,
+  ): Promise<State> {
+    const start = Date.now();
+    const result = await fn();
+    this.stepDurations[stepName] = Date.now() - start;
+    this.logger.info(
+      `${this.agentName}: ${stepName} took ${this.stepDurations[stepName]}ms`,
+    );
+    return result;
+  }
+
+  protected getMetadata() {
+    const totalDuration = Date.now() - this.startTime;
+    return {
+      totalDuration,
+      stepDurations: this.stepDurations,
+      logs: this.logs,
+    };
+  }
+}

typescript/packages/toolshed/lib/agent.ts

@@ -0,0 +1,46 @@
+import { AppType } from "@/app.ts";
+import { hc } from "hono/client";
+
+// NOTE(jake): Ideally this would be exposed via the hono client, but I wasn't
+// able to get it all wired up. Importing the route definition is fine for now.
+import type { GetModelsRouteQueryParams } from "@/routes/ai/llm/llm.routes.ts";
+
+const client = hc<AppType>("http://localhost:8000/");
+
+export async function listAvailableModels({
+  capability,
+  task,
+  search,
+}: GetModelsRouteQueryParams) {
+  const res = await client.api.ai.llm.models.$get({
+    query: {
+      search,
+      capability,
+      task,
+    },
+  });
+  return res.json();
+}
+
+export async function generateText(
+  query: Parameters<typeof client.api.ai.llm.$post>[0]["json"],
+): Promise<string> {
+  const res = await client.api.ai.llm.$post({ json: query });
+  const data = await res.json();
+
+  if ("error" in data) {
+    throw new Error(data.error);
+  }
+
+  if ("type" in data && data.type === "json") {
+    return data.body.content;
+  }
+
+  if ("content" in data) {
+    // bf: this is actually the case that runs, even if the types disagree
+    // no idea why
+    return (data as any).content;
+  }
+
+  throw new Error("Unexpected response from LLM server");
+}

typescript/packages/toolshed/lib/llm.ts

@@ -0,0 +1,57 @@
+export interface Logger {
+  // deno-lint-ignore no-explicit-any
+  info(...args: any[]): void;
+  // deno-lint-ignore no-explicit-any
+  error(...args: any[]): void;
+  // deno-lint-ignore no-explicit-any
+  warn(...args: any[]): void;
+  // deno-lint-ignore no-explicit-any
+  debug(...args: any[]): void;
+}
+
+export class PrefixedLogger implements Logger {
+  private logger: Logger;
+  private prefix: string;
+  private logMessages: string[] = [];
+
+  constructor(logger: Logger = console, prefix: string) {
+    this.logger = logger;
+    this.prefix = prefix;
+    this.info = this.info.bind(this);
+    this.error = this.error.bind(this);
+    this.warn = this.warn.bind(this);
+    this.debug = this.debug.bind(this);
+  }
+
+  // deno-lint-ignore no-explicit-any
+  info(...args: any[]) {
+    const message = [`[${this.prefix}]`, ...args].join(" ");
+    this.logMessages.push(message);
+    this.logger.info(message);
+  }
+
+  // deno-lint-ignore no-explicit-any
+  error(...args: any[]) {
+    const message = [`[${this.prefix}]`, ...args].join(" ");
+    this.logMessages.push(message);
+    this.logger.error(message);
+  }
+
+  // deno-lint-ignore no-explicit-any
+  warn(...args: any[]) {
+    const message = [`[${this.prefix}]`, ...args].join(" ");
+    this.logMessages.push(message);
+    this.logger.warn(message);
+  }
+
+  // deno-lint-ignore no-explicit-any
+  debug(...args: any[]) {
+    const message = [`[${this.prefix}]`, ...args].join(" ");
+    this.logMessages.push(message);
+    this.logger.debug(message);
+  }
+
+  getLogs(): string[] {
+    return this.logMessages;
+  }
+}

typescript/packages/toolshed/lib/prefixed-logger.ts

@@ -0,0 +1,17 @@
+export async function handleResponse<T>(response: Response): Promise<T> {
+  const data = await response.json();
+
+  if ("error" in data) {
+    throw new Error(data.error);
+  }
+
+  if (data.type === "json") {
+    return data.body;
+  }
+
+  if (data instanceof Response) {
+    throw new Error(data.statusText);
+  }
+
+  return null as never;
+}

typescript/packages/toolshed/lib/response.ts

@@ -0,0 +1,46 @@
+import { Schema, Validator } from "jsonschema";
+
+export function checkSchemaMatch(
+  data: Record<string, unknown>,
+  schema: Schema,
+): boolean {
+  const validator = new Validator();
+
+  const jsonSchema: unknown = {
+    type: "object",
+    properties: Object.keys(schema).reduce(
+      (acc: Record<string, unknown>, key) => {
+        const schemaValue = schema[key as keyof Schema];
+        acc[key] = { type: (schemaValue as any)?.type || typeof schemaValue };
+        return acc;
+      },
+      {},
+    ),
+    required: Object.keys(schema),
+    additionalProperties: true,
+  };
+
+  const rootResult = validator.validate(data, jsonSchema as Schema);
+  if (rootResult.valid) {
+    return true;
+  }
+
+  function checkSubtrees(obj: unknown): boolean {
+    if (typeof obj !== "object" || obj === null) {
+      return false;
+    }
+
+    if (Array.isArray(obj)) {
+      return obj.some((item) => checkSubtrees(item));
+    }
+
+    const result = validator.validate(obj, jsonSchema as Schema);
+    if (result.valid) {
+      return true;
+    }
+
+    return Object.values(obj).some((value) => checkSubtrees(value));
+  }
+
+  return checkSubtrees(data);
+}

typescript/packages/toolshed/lib/schema-match.ts

@@ -1,10 +1,10 @@
 import type { OpenAPIHono, RouteConfig, RouteHandler } from "@hono/zod-openapi";
-import type { PinoLogger } from "hono-pino";
+import type { Logger } from "pino";
 import type { RedisClientType } from "redis";
 
 export interface AppBindings {
   Variables: {
-    logger: PinoLogger;
+    logger: Logger;
     blobbyRedis: RedisClientType;
   };
 }

typescript/packages/toolshed/lib/types.ts

@@ -25,7 +25,9 @@ export async function loadItem(key: string): Promise<CacheItem | null> {
     const cacheData = await Deno.readTextFile(filePath);
     console.log(
       `${timestamp()} ${colors.green}📦 Cache loaded:${colors.reset} ${
-        filePath.slice(-12)
+        filePath.slice(
+          -12,
+        )
       }`,
     );
     return JSON.parse(cacheData);