diff --git a/typescript/packages/toolshed/lib/types.ts b/typescript/packages/toolshed/lib/types.ts index 24cf9ab26..2702cb2fd 100644 --- a/typescript/packages/toolshed/lib/types.ts +++ b/typescript/packages/toolshed/lib/types.ts @@ -5,7 +5,6 @@ import type { RedisClientType } from "redis"; export interface AppBindings { Variables: { logger: Logger; - blobbyRedis: RedisClientType; }; } diff --git a/typescript/packages/toolshed/routes/storage/blobby/blobby.handlers.ts b/typescript/packages/toolshed/routes/storage/blobby/blobby.handlers.ts index 92020cff4..e11fb8f08 100644 --- a/typescript/packages/toolshed/routes/storage/blobby/blobby.handlers.ts +++ b/typescript/packages/toolshed/routes/storage/blobby/blobby.handlers.ts @@ -12,20 +12,12 @@ import { getUserBlobs, removeBlobFromUser, } from "./lib/redis.ts"; -import type { RedisClientType } from "redis"; -import { DiskStorage } from "@/routes/storage/blobby/lib/storage.ts"; -import env from "@/env.ts"; - -const DATA_DIR = `${env.CACHE_DIR}/blobby`; - -export const storage = new DiskStorage(DATA_DIR); -await storage.init(); +import { getRedisClient, storage } from "./utils.ts"; export const uploadBlobHandler: AppRouteHandler< typeof uploadBlob > = async (c) => { - const redis = c.get("blobbyRedis"); - if (!redis) throw new Error("Redis client not found in context"); + const redis = await getRedisClient(); const logger = c.get("logger"); const key = c.req.param("key"); const content = await c.req.json(); @@ -90,8 +82,7 @@ export const getBlobPathHandler: AppRouteHandler< export const listBlobsHandler: AppRouteHandler = async ( c, ) => { - const redis: RedisClientType = c.get("blobbyRedis"); - if (!redis) throw new Error("Redis client not found in context"); + const redis = await getRedisClient(); const logger = c.get("logger"); const showAll = c.req.query("all") === "true"; const showAllWithData = c.req.query("allWithData") !== undefined; @@ -174,8 +165,7 @@ export const listBlobsHandler: AppRouteHandler = async ( export const deleteBlobHandler: AppRouteHandler = async ( c, ) => { - const redis = c.get("blobbyRedis"); - if (!redis) throw new Error("Redis client not found in context"); + const redis = await getRedisClient(); const logger = c.get("logger"); const key = c.req.param("key"); diff --git a/typescript/packages/toolshed/routes/storage/blobby/blobby.index.ts b/typescript/packages/toolshed/routes/storage/blobby/blobby.index.ts index 0bfdfda20..9193a77d9 100644 --- a/typescript/packages/toolshed/routes/storage/blobby/blobby.index.ts +++ b/typescript/packages/toolshed/routes/storage/blobby/blobby.index.ts @@ -5,40 +5,10 @@ import { createRouter } from "@/lib/create-app.ts"; import * as handlers from "./blobby.handlers.ts"; import * as routes from "./blobby.routes.ts"; -import env from "@/env.ts"; -import { createClient } from "redis"; -import type { RedisClientType } from "redis"; import { cors } from "@hono/hono/cors"; const router = createRouter(); -router.use("/api/storage/blobby/*", async (c, next) => { - const logger = c.get("logger"); - try { - const redis = createClient({ - url: env.BLOBBY_REDIS_URL, - }); - - redis.on("error", (err) => { - logger.error({ err }, "Redis client error"); - }); - - logger.info("Connecting to Redis..."); - if (!redis.isOpen) { - await redis.connect(); - } - logger.info("Redis connected successfully"); - - c.set("blobbyRedis", redis as RedisClientType); - await next(); - logger.info("Closing Redis connection"); - await redis.quit(); - } catch (error) { - logger.error(error, "Error in Redis middleware"); - throw error; - } -}); - router.use( "/api/storage/blobby/*", cors({ diff --git a/typescript/packages/toolshed/routes/storage/blobby/blobby.test.ts b/typescript/packages/toolshed/routes/storage/blobby/blobby.test.ts index c324fc495..9baf1c276 100644 --- a/typescript/packages/toolshed/routes/storage/blobby/blobby.test.ts +++ b/typescript/packages/toolshed/routes/storage/blobby/blobby.test.ts @@ -1,334 +1,383 @@ import { assertEquals } from "@std/assert"; import { sha256 } from "@/lib/sha2.ts"; +import { ensureDir } from "@std/fs"; import env from "@/env.ts"; import createApp from "@/lib/create-app.ts"; import router from "./blobby.index.ts"; +import { closeRedisClient, storage } from "./utils.ts"; if (env.ENV !== "test") { throw new Error("ENV must be 'test'"); } const app = createApp().route("/", router); +const TEST_DATA_DIR = `${env.CACHE_DIR}/blobby-test`; + +// Setup function to run before all tests +async function setup() { + // Create a test-specific data directory + await ensureDir(TEST_DATA_DIR); + // Override the storage base directory for tests + storage.baseDir = TEST_DATA_DIR; + await storage.init(); +} -Deno.test("blobby storage routes", async (t) => { - const testContent = { - message: `This is a test blob created at ${new Date().toISOString()}`, - }; - const key = await sha256(JSON.stringify(testContent)); - - await t.step("POST /api/storage/blobby/{key} uploads blob", async () => { - const response = await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(testContent), - }), - ); - - assertEquals(response.status, 200); - const json = await response.json(); - assertEquals(json.key, key); - }); - - await t.step("GET /api/storage/blobby/{key} retrieves blob", async () => { - const response = await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}`), - ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(json.message, testContent.message); - assertEquals(typeof json.blobCreatedAt, "string"); - assertEquals(json.blobAuthor, "system"); - }); - - await t.step("GET /api/storage/blobby lists blobs", async () => { - const response = await app.fetch( - new Request("http://localhost/api/storage/blobby"), - ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(Array.isArray(json.blobs), true); - assertEquals(json.blobs.includes(key), true); - }); - - await t.step( - "GET /api/storage/blobby?allWithData=true lists blobs with data", - async () => { - const response = await app.fetch( - new Request("http://localhost/api/storage/blobby?allWithData=true"), - ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(typeof json, "object"); - assertEquals(json[key].message, testContent.message); - assertEquals(typeof json[key].blobCreatedAt, "string"); - assertEquals(json[key].blobAuthor, "system"); - }, - ); - - await t.step( - "GET /api/storage/blobby/{key}/message gets nested path", - async () => { - const response = await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}/message`), - ); - assertEquals(response.status, 200); - - const text = await response.text(); - assertEquals(text, testContent.message); - }, - ); - - await t.step( - "GET /api/storage/blobby/{key}/invalid returns 404", - async () => { - const response = await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}/invalid`), - ); - assertEquals(response.status, 404); - - const json = await response.json(); - assertEquals(json.error, "Path not found"); - }, - ); - - await t.step( - "GET /api/storage/blobby?prefix=test- lists blobs with prefix", - async () => { - const testPrefixContent = { - message: "This is a test-prefixed blob", - }; - const testKey = `test-${await sha256(JSON.stringify(testPrefixContent))}`; +// Teardown function to run after all tests +async function teardown() { + // Clean up test data directory + try { + Deno.removeSync(TEST_DATA_DIR, { recursive: true }); + } catch (error) { + // Ignore errors if directory doesn't exist + } + + // Close Redis client + await closeRedisClient(); +} - const otherContent = { - message: "This is another blob", +Deno.test({ + name: "blobby storage routes", + async fn(t) { + // Run setup before all tests + await setup(); + + try { + const testContent = { + message: `This is a test blob created at ${new Date().toISOString()}`, }; - const otherKey = `other-${await sha256(JSON.stringify(otherContent))}`; - - await app.fetch( - new Request(`http://localhost/api/storage/blobby/${testKey}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(testPrefixContent), - }), - ); + const key = await sha256(JSON.stringify(testContent)); - await app.fetch( - new Request(`http://localhost/api/storage/blobby/${otherKey}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(otherContent), - }), - ); + await t.step("POST /api/storage/blobby/{key} uploads blob", async () => { + const response = await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(testContent), + }), + ); - const response = await app.fetch( - new Request("http://localhost/api/storage/blobby?prefix=test-"), - ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(Array.isArray(json.blobs), true); - assertEquals(json.blobs.includes(testKey), true); - assertEquals(json.blobs.includes(otherKey), false); - }, - ); - - await t.step( - "GET /api/storage/blobby?prefix=test-&allWithData=true lists prefixed blobs with data", - async () => { - const response = await app.fetch( - new Request( - "http://localhost/api/storage/blobby?prefix=test-&allWithData=true", - ), - ); - assertEquals(response.status, 200); + assertEquals(response.status, 200); + const json = await response.json(); + assertEquals(json.key, key); + }); - const json = await response.json(); - assertEquals(typeof json, "object"); + await t.step("GET /api/storage/blobby/{key} retrieves blob", async () => { + const response = await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}`), + ); + assertEquals(response.status, 200); - Object.keys(json).forEach((key) => { - assertEquals(key.startsWith("test-"), true); + const json = await response.json(); + assertEquals(json.message, testContent.message); + assertEquals(typeof json.blobCreatedAt, "string"); + assertEquals(json.blobAuthor, "system"); }); - const testKeys = Object.keys(json).filter((k) => k.startsWith("test-")); - for (const key of testKeys) { - assertEquals(typeof json[key].message, "string"); - assertEquals(typeof json[key].blobCreatedAt, "string"); - assertEquals(json[key].blobAuthor, "system"); - } - }, - ); - - await t.step( - "GET /api/storage/blobby?search=test lists blobs containing text", - async () => { - // Create blobs with different content - const matchingContent = { - message: "This contains test in the content", - other: "field", - }; - const matchingKey = await sha256(JSON.stringify(matchingContent)); + await t.step("GET /api/storage/blobby lists blobs", async () => { + const response = await app.fetch( + new Request("http://localhost/api/storage/blobby"), + ); + assertEquals(response.status, 200); - const nonMatchingContent = { - message: "This has different content", - other: "nothing here", - }; - const nonMatchingKey = await sha256(JSON.stringify(nonMatchingContent)); - - // Upload both blobs - await app.fetch( - new Request(`http://localhost/api/storage/blobby/${matchingKey}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(matchingContent), - }), - ); + const json = await response.json(); + assertEquals(Array.isArray(json.blobs), true); + assertEquals(json.blobs.includes(key), true); + }); - await app.fetch( - new Request(`http://localhost/api/storage/blobby/${nonMatchingKey}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(nonMatchingContent), - }), + await t.step( + "GET /api/storage/blobby?allWithData=true lists blobs with data", + async () => { + const response = await app.fetch( + new Request("http://localhost/api/storage/blobby?allWithData=true"), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(typeof json, "object"); + assertEquals(json[key].message, testContent.message); + assertEquals(typeof json[key].blobCreatedAt, "string"); + assertEquals(json[key].blobAuthor, "system"); + }, ); - // Test fulltext search - const response = await app.fetch( - new Request("http://localhost/api/storage/blobby?search=test"), - ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(Array.isArray(json.blobs), true); - assertEquals(json.blobs.includes(matchingKey), true); - assertEquals(json.blobs.includes(nonMatchingKey), false); - }, - ); - - await t.step( - "GET /api/storage/blobby?search=test&allWithData=true lists matching blobs with data", - async () => { - const response = await app.fetch( - new Request( - "http://localhost/api/storage/blobby?search=test&allWithData=true", - ), + await t.step( + "GET /api/storage/blobby/{key}/message gets nested path", + async () => { + const response = await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}/message`), + ); + assertEquals(response.status, 200); + + const text = await response.text(); + assertEquals(text, testContent.message); + }, ); - assertEquals(response.status, 200); - const json = await response.json(); - assertEquals(typeof json, "object"); + await t.step( + "GET /api/storage/blobby/{key}/invalid returns 404", + async () => { + const response = await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}/invalid`), + ); + assertEquals(response.status, 404); - // Verify all returned objects contain the search term - Object.entries(json).forEach(([key, value]) => { - const stringified = JSON.stringify(value).toLowerCase(); - assertEquals(stringified.includes("test"), true); - }); - }, - ); - - await t.step( - "GET /api/storage/blobby?prefix=test-&search=blob combines prefix and search", - async () => { - const response = await app.fetch( - new Request( - "http://localhost/api/storage/blobby?prefix=test-&search=blob", - ), + const json = await response.json(); + assertEquals(json.error, "Path not found"); + }, ); - assertEquals(response.status, 200); - const json = await response.json(); - assertEquals(Array.isArray(json.blobs), true); + await t.step( + "GET /api/storage/blobby?prefix=test- lists blobs with prefix", + async () => { + const testPrefixContent = { + message: "This is a test-prefixed blob", + }; + const testKey = `test-${await sha256( + JSON.stringify(testPrefixContent), + )}`; + + const otherContent = { + message: "This is another blob", + }; + const otherKey = `other-${await sha256( + JSON.stringify(otherContent), + )}`; + + await app.fetch( + new Request(`http://localhost/api/storage/blobby/${testKey}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(testPrefixContent), + }), + ); + + await app.fetch( + new Request(`http://localhost/api/storage/blobby/${otherKey}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(otherContent), + }), + ); + + const response = await app.fetch( + new Request("http://localhost/api/storage/blobby?prefix=test-"), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(Array.isArray(json.blobs), true); + assertEquals(json.blobs.includes(testKey), true); + assertEquals(json.blobs.includes(otherKey), false); + }, + ); - // Verify all returned keys start with prefix and content contains search term - for (const key of json.blobs) { - assertEquals(key.startsWith("test-"), true); - const blobResponse = await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}`), - ); - const blobContent = await blobResponse.json(); - const stringified = JSON.stringify(blobContent).toLowerCase(); - assertEquals(stringified.includes("blob"), true); - } - }, - ); - - await t.step( - "GET /api/storage/blobby?keys=key1,key2 fetches specific blobs", - async () => { - // Create three test blobs - const blob1 = { - message: "First test blob", - id: 1, - }; - const blob2 = { - message: "Second test blob", - id: 2, - }; - const blob3 = { - message: "Third test blob", - id: 3, - }; + await t.step( + "GET /api/storage/blobby?prefix=test-&allWithData=true lists prefixed blobs with data", + async () => { + const response = await app.fetch( + new Request( + "http://localhost/api/storage/blobby?prefix=test-&allWithData=true", + ), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(typeof json, "object"); + + Object.keys(json).forEach((key) => { + assertEquals(key.startsWith("test-"), true); + }); + + const testKeys = Object.keys(json).filter((k) => + k.startsWith("test-") + ); + for (const key of testKeys) { + assertEquals(typeof json[key].message, "string"); + assertEquals(typeof json[key].blobCreatedAt, "string"); + assertEquals(json[key].blobAuthor, "system"); + } + }, + ); - const key1 = await sha256(JSON.stringify(blob1)); - const key2 = await sha256(JSON.stringify(blob2)); - const key3 = await sha256(JSON.stringify(blob3)); + await t.step( + "GET /api/storage/blobby?search=test lists blobs containing text", + async () => { + // Create blobs with different content + const matchingContent = { + message: "This contains test in the content", + other: "field", + }; + const matchingKey = await sha256(JSON.stringify(matchingContent)); + + const nonMatchingContent = { + message: "This has different content", + other: "nothing here", + }; + const nonMatchingKey = await sha256( + JSON.stringify(nonMatchingContent), + ); + + // Upload both blobs + await app.fetch( + new Request(`http://localhost/api/storage/blobby/${matchingKey}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(matchingContent), + }), + ); + + await app.fetch( + new Request( + `http://localhost/api/storage/blobby/${nonMatchingKey}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(nonMatchingContent), + }, + ), + ); + + // Test fulltext search + const response = await app.fetch( + new Request("http://localhost/api/storage/blobby?search=test"), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(Array.isArray(json.blobs), true); + assertEquals(json.blobs.includes(matchingKey), true); + assertEquals(json.blobs.includes(nonMatchingKey), false); + }, + ); - // Upload all three blobs - const blobs = [[blob1, key1], [blob2, key2], [blob3, key3]]; - for ( - const [content, key] of blobs - ) { - await app.fetch( - new Request(`http://localhost/api/storage/blobby/${key}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(content), - }), - ); - } + await t.step( + "GET /api/storage/blobby?search=test&allWithData=true lists matching blobs with data", + async () => { + const response = await app.fetch( + new Request( + "http://localhost/api/storage/blobby?search=test&allWithData=true", + ), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(typeof json, "object"); + + // Verify all returned objects contain the search term + Object.entries(json).forEach(([key, value]) => { + const stringified = JSON.stringify(value).toLowerCase(); + assertEquals(stringified.includes("test"), true); + }); + }, + ); - // Fetch only two of the blobs - const response = await app.fetch( - new Request(`http://localhost/api/storage/blobby?keys=${key1},${key2}`), + await t.step( + "GET /api/storage/blobby?prefix=test-&search=blob combines prefix and search", + async () => { + const response = await app.fetch( + new Request( + "http://localhost/api/storage/blobby?prefix=test-&search=blob", + ), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(Array.isArray(json.blobs), true); + + // Verify all returned keys start with prefix and content contains search term + for (const key of json.blobs) { + assertEquals(key.startsWith("test-"), true); + const blobResponse = await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}`), + ); + const blobContent = await blobResponse.json(); + const stringified = JSON.stringify(blobContent).toLowerCase(); + assertEquals(stringified.includes("blob"), true); + } + }, ); - assertEquals(response.status, 200); - - const json = await response.json(); - assertEquals(typeof json, "object"); - - // Should only contain the two requested blobs - assertEquals(Object.keys(json).length, 2); - assertEquals(json[key1].message, blob1.message); - assertEquals(json[key2].message, blob2.message); - assertEquals(json[key3], undefined); - - // Verify blob metadata - assertEquals(typeof json[key1].blobCreatedAt, "string"); - assertEquals(json[key1].blobAuthor, "system"); - assertEquals(typeof json[key2].blobCreatedAt, "string"); - assertEquals(json[key2].blobAuthor, "system"); - }, - ); - - await t.step( - "GET /api/storage/blobby?keys=invalid returns empty result", - async () => { - const response = await app.fetch( - new Request( - "http://localhost/api/storage/blobby?keys=invalid1,invalid2", - ), + + await t.step( + "GET /api/storage/blobby?keys=key1,key2 fetches specific blobs", + async () => { + // Create three test blobs + const blob1 = { + message: "First test blob", + id: 1, + }; + const blob2 = { + message: "Second test blob", + id: 2, + }; + const blob3 = { + message: "Third test blob", + id: 3, + }; + + const key1 = await sha256(JSON.stringify(blob1)); + const key2 = await sha256(JSON.stringify(blob2)); + const key3 = await sha256(JSON.stringify(blob3)); + + // Upload all three blobs + const blobs = [[blob1, key1], [blob2, key2], [blob3, key3]]; + for ( + const [content, key] of blobs + ) { + await app.fetch( + new Request(`http://localhost/api/storage/blobby/${key}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(content), + }), + ); + } + + // Fetch only two of the blobs + const response = await app.fetch( + new Request( + `http://localhost/api/storage/blobby?keys=${key1},${key2}`, + ), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(typeof json, "object"); + + // Should only contain the two requested blobs + assertEquals(Object.keys(json).length, 2); + assertEquals(json[key1].message, blob1.message); + assertEquals(json[key2].message, blob2.message); + assertEquals(json[key3], undefined); + + // Verify blob metadata + assertEquals(typeof json[key1].blobCreatedAt, "string"); + assertEquals(json[key1].blobAuthor, "system"); + assertEquals(typeof json[key2].blobCreatedAt, "string"); + assertEquals(json[key2].blobAuthor, "system"); + }, ); - assertEquals(response.status, 200); - const json = await response.json(); - assertEquals(typeof json, "object"); - assertEquals(Object.keys(json).length, 0); - }, - ); + await t.step( + "GET /api/storage/blobby?keys=invalid returns empty result", + async () => { + const response = await app.fetch( + new Request( + "http://localhost/api/storage/blobby?keys=invalid1,invalid2", + ), + ); + assertEquals(response.status, 200); + + const json = await response.json(); + assertEquals(typeof json, "object"); + assertEquals(Object.keys(json).length, 0); + }, + ); + } finally { + // Run teardown after all tests, even if they fail + await teardown(); + } + }, }); diff --git a/typescript/packages/toolshed/routes/storage/blobby/lib/storage.ts b/typescript/packages/toolshed/routes/storage/blobby/lib/storage.ts index 2da4a6727..559ae12c0 100644 --- a/typescript/packages/toolshed/routes/storage/blobby/lib/storage.ts +++ b/typescript/packages/toolshed/routes/storage/blobby/lib/storage.ts @@ -2,7 +2,7 @@ import { join } from "@std/path"; import { ensureDir } from "@std/fs"; export class DiskStorage { - constructor(private baseDir: string) {} + constructor(public baseDir: string) {} async init() { await ensureDir(this.baseDir); diff --git a/typescript/packages/toolshed/routes/storage/blobby/utils.ts b/typescript/packages/toolshed/routes/storage/blobby/utils.ts new file mode 100644 index 000000000..d972c8995 --- /dev/null +++ b/typescript/packages/toolshed/routes/storage/blobby/utils.ts @@ -0,0 +1,30 @@ +import { createClient, type RedisClientType } from "redis"; +import env from "@/env.ts"; +import { DiskStorage } from "@/routes/storage/blobby/lib/storage.ts"; + +const DATA_DIR = `${env.CACHE_DIR}/blobby`; + +export const storage = new DiskStorage(DATA_DIR); +await storage.init(); + +let redisClient: RedisClientType | null = null; + +export const getRedisClient = async () => { + if (!redisClient) { + redisClient = createClient({ + url: env.BLOBBY_REDIS_URL, + }); + } + if (!redisClient.isOpen) { + await redisClient.connect(); + } + + return redisClient; +}; + +export const closeRedisClient = async () => { + if (redisClient?.isOpen) { + await redisClient.quit(); + redisClient = null; + } +};