From bc968c95d2cca6e2f9a23367570e82772aeb5e22 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Tue, 4 Mar 2025 15:46:52 -0500 Subject: [PATCH] cleanup --- typescript/packages/collectathon/GMAIL.md | 21 - typescript/packages/collectathon/README.md | 23 - typescript/packages/collectathon/action.ts | 109 --- typescript/packages/collectathon/calendar.ts | 65 -- typescript/packages/collectathon/chat.ts | 108 --- .../chrome-extension/background.js | 38 - .../chrome-extension/manifest.json | 13 - .../collectathon/chrome-extension/popup.html | 40 - .../collectathon/chrome-extension/popup.js | 222 ----- .../packages/collectathon/collections.ts | 153 ---- typescript/packages/collectathon/db.ts | 51 -- typescript/packages/collectathon/deno.lock | 818 ------------------ typescript/packages/collectathon/deps.ts | 17 - typescript/packages/collectathon/dream.ts | 98 --- typescript/packages/collectathon/github.ts | 173 ---- typescript/packages/collectathon/gmail.ts | 119 --- typescript/packages/collectathon/import.ts | 167 ---- typescript/packages/collectathon/items.ts | 249 ------ typescript/packages/collectathon/llm.ts | 112 --- typescript/packages/collectathon/mail.ts | 83 -- typescript/packages/collectathon/main.ts | 257 ------ typescript/packages/collectathon/package.json | 49 -- typescript/packages/collectathon/rss.ts | 64 -- typescript/packages/collectathon/rules.ts | 136 --- typescript/packages/collectathon/schema.ts | 16 - typescript/packages/collectathon/search.ts | 57 -- typescript/packages/collectathon/server.ts | 166 ---- .../packages/collectathon/spec/dreams.md | 9 - typescript/packages/collectathon/start.sh | 4 - typescript/packages/collectathon/synopsys.ts | 211 ----- typescript/packages/collectathon/tests.sh | 121 --- .../packages/collectathon/tsconfig.json | 9 - typescript/packages/collectathon/view.ts | 157 ---- typescript/packages/collectathon/webpage.ts | 94 -- 34 files changed, 4029 deletions(-) delete mode 100644 typescript/packages/collectathon/GMAIL.md delete mode 100644 typescript/packages/collectathon/README.md delete mode 100644 typescript/packages/collectathon/action.ts delete mode 100644 typescript/packages/collectathon/calendar.ts delete mode 100644 typescript/packages/collectathon/chat.ts delete mode 100644 typescript/packages/collectathon/chrome-extension/background.js delete mode 100644 typescript/packages/collectathon/chrome-extension/manifest.json delete mode 100644 typescript/packages/collectathon/chrome-extension/popup.html delete mode 100644 typescript/packages/collectathon/chrome-extension/popup.js delete mode 100644 typescript/packages/collectathon/collections.ts delete mode 100644 typescript/packages/collectathon/db.ts delete mode 100644 typescript/packages/collectathon/deno.lock delete mode 100644 typescript/packages/collectathon/deps.ts delete mode 100644 typescript/packages/collectathon/dream.ts delete mode 100644 typescript/packages/collectathon/github.ts delete mode 100644 typescript/packages/collectathon/gmail.ts delete mode 100644 typescript/packages/collectathon/import.ts delete mode 100644 typescript/packages/collectathon/items.ts delete mode 100644 typescript/packages/collectathon/llm.ts delete mode 100644 typescript/packages/collectathon/mail.ts delete mode 100644 typescript/packages/collectathon/main.ts delete mode 100644 typescript/packages/collectathon/package.json delete mode 100644 typescript/packages/collectathon/rss.ts delete mode 100644 typescript/packages/collectathon/rules.ts delete mode 100644 typescript/packages/collectathon/schema.ts delete mode 100644 typescript/packages/collectathon/search.ts delete mode 100644 typescript/packages/collectathon/server.ts delete mode 100644 typescript/packages/collectathon/spec/dreams.md delete mode 100755 typescript/packages/collectathon/start.sh delete mode 100644 typescript/packages/collectathon/synopsys.ts delete mode 100755 typescript/packages/collectathon/tests.sh delete mode 100644 typescript/packages/collectathon/tsconfig.json delete mode 100644 typescript/packages/collectathon/view.ts delete mode 100644 typescript/packages/collectathon/webpage.ts diff --git a/typescript/packages/collectathon/GMAIL.md b/typescript/packages/collectathon/GMAIL.md deleted file mode 100644 index c70027c01..000000000 --- a/typescript/packages/collectathon/GMAIL.md +++ /dev/null @@ -1,21 +0,0 @@ -script for gmail to synopsys - -# Setup Credentials - -You'll need to get an [oauth crednetials.json](https://developers.google.com/gmail/api/quickstart/nodejs#authorize_credentials_for_a_desktop_application) file from google cloud console and put it in this directory. - -1. visit [google cloud console](https://console.cloud.google.com/apis/credentials) -2. create a new project (or select an existing one) -3. enable the gmail api (you may need to search for it) -4. create credentials and select desktop app -- Create Credentials > OAuth client ID -- Application type > Desktop app -- After the credentials are created, click `Download JSON` and put it in this directory as `credentials.json` - -# Running - -The first time you run the script it will open a browser window prompting you to login and authorize access. Afterwards credentials will be stored in a `token.json` file and you won't have to do this again (hopefully) - -``` -deno run --allow-net --allow-read --allow-write --allow-run gmail.ts [number of emails to import] -``` diff --git a/typescript/packages/collectathon/README.md b/typescript/packages/collectathon/README.md deleted file mode 100644 index 97c735064..000000000 --- a/typescript/packages/collectathon/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Collectathon - -Deno + sqlite. - -`npm run start` will run both the CLI and the server. - -`npm run server` will run the server. - -`npm run cli` will run the CLI. - -## Env - -For the server, create a `.env` file with the following: - -``` -ANTHROPIC_API_KEY=... -OPENAI_API_KEY=... -SYNOPSYS_API_URL=http://localhost:8080 -``` - -## Tests - -See `test.sh` for web API tests. diff --git a/typescript/packages/collectathon/action.ts b/typescript/packages/collectathon/action.ts deleted file mode 100644 index f1ff178cb..000000000 --- a/typescript/packages/collectathon/action.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { db } from "./db.ts"; -import { chat, fastCompletion } from "./llm.ts"; -import { CoreMessage } from "npm:ai@3.3.21"; - -export async function handleActionCommand( - collectionName: string, - userPrompt: string, -) { - // Fetch the collection items - const items = db.query<[number, string]>( - `SELECT i.id, i.content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collectionName], - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collectionName}`); - return; - } - - // Parse the JSON content of each item - const jsonItems = items.map(([id, content]) => ({ - id, - ...JSON.parse(content), - })); - - // Extract the shape of the JSON items - const itemShape = await extractJsonShape(jsonItems); - - console.log("Items shape:", itemShape); - - // Generate the transformation function using the LLM - const transformationFunction = await generateTransformationFunction( - userPrompt, - itemShape, - ); - - // Print the function and ask for user confirmation - console.log("Generated transformation function:"); - console.log(transformationFunction); - const confirmation = prompt("Do you want to execute this function? (y/n): "); - - if (confirmation.toLowerCase() !== "y") { - console.log("Function execution cancelled."); - return; - } - - // Execute the transformation function - try { - const result = eval( - `(${transformationFunction})(${JSON.stringify(jsonItems)})`, - ); - console.log("Transformation result:"); - console.log(JSON.stringify(result, null, 2)); - } catch (error) { - console.error("Error executing transformation function:", error); - } -} - -async function extractJsonShape(items: any[]): Promise { - const systemPrompt = - "Output a json schema that covers all the keys and values of the JSON objects."; - const userMessage = `${JSON.stringify(items, null, 2)}`; - - const messages: CoreMessage[] = [ - { role: "system", content: systemPrompt }, - { role: "user", content: userMessage }, - ]; - - const response = await fastCompletion(systemPrompt, messages); - return response; -} - -async function generateTransformationFunction( - userPrompt: string, - itemShape: string, -): Promise { - const systemPrompt = - "You are an expert JavaScript programmer. Generate a JavaScript function that takes an array of JSON objects as input and transforms it based on the user's prompt. The function should return the transformed data."; - const userMessage = `Generate a JavaScript function to perform the following transformation on an array of JSON objects: ${userPrompt}\n\nThe function should take a single parameter 'items' which is the array of JSON objects. The shape of each item is:\n${itemShape}\n\nReturn only the function, without any explanation or additional text. - - e.g. - - function(items) { - ... - }`; - - const messages: CoreMessage[] = [ - { role: "system", content: systemPrompt }, - { role: "user", content: userMessage }, - ]; - - const response = await chat(systemPrompt, messages, true); - return `${response}`; -} - -export function addActionCommand(args: string[]) { - if (args.length < 2) { - console.log("Usage: action "); - return; - } - - const collectionName = args.shift()!; - const prompt = args.join(" "); - handleActionCommand(collectionName, prompt); -} diff --git a/typescript/packages/collectathon/calendar.ts b/typescript/packages/collectathon/calendar.ts deleted file mode 100644 index 756441a36..000000000 --- a/typescript/packages/collectathon/calendar.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { parseICS } from "./deps.ts"; -import { db } from "./db.ts"; -import { addItemToCollection, getOrCreateCollection } from "./collections.ts"; - -export async function clipCalendar(calendarUrl: string, collectionName: string) { - console.log(`Importing calendar from ${calendarUrl} to collection ${collectionName}`); - - db.query("BEGIN TRANSACTION"); - if (!calendarUrl.endsWith('.ics')) { - console.error('Invalid calendar URL. Must be an ICS file.'); - return; - } - - try { - const response = await fetch(calendarUrl); - const icsData = await response.text(); - const events = parseICS(icsData); - - const collectionId = await getOrCreateCollection(collectionName); - let importedCount = 0; - - for (const event of events) { - if (event.type === 'VEVENT') { - const title = event.name || 'Untitled Event'; - const startDate = event.startDate?.toISOString() || ''; - const endDate = event.endDate?.toISOString() || ''; - - const contentJson = { - title: title, - description: event.description || '', - startDate: startDate, - endDate: endDate, - location: event.location || '', - organizer: event.organizer || '', - }; - - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - calendarUrl, - title, - JSON.stringify(contentJson), - event.description || '', - "Calendar", - ] - ); - const itemId = result[0][0] as number; - - await db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId] - ); - importedCount++; - } - } - - console.log(`Imported ${importedCount} events to collection "${collectionName}"`); - db.query("COMMIT"); - } catch (error) { - console.error(`Error importing Calendar: ${error.message}`); - db.query("ROLLBACK"); - } -} - -// Remove the addItem helper function as it's no longer needed diff --git a/typescript/packages/collectathon/chat.ts b/typescript/packages/collectathon/chat.ts deleted file mode 100644 index 126f7a493..000000000 --- a/typescript/packages/collectathon/chat.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { CoreMessage } from "npm:ai@3.3.21"; -import { chat } from "./llm.ts"; -import { db } from "./db.ts"; - -function formatCollectionItems(collections: string[]): string { - let formattedContent = ""; - - for (const collection of collections) { - formattedContent += `\n`; - - const items = db.query<[number, string, string, string]>( - `SELECT i.id, i.title, i.content, i.raw_content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collection], - ); - - for (const [id, title, content, rawContent] of items) { - formattedContent += `\n`; - formattedContent += `${content}\n`; - formattedContent += `\n\n`; - } - - formattedContent += `\n\n`; - } - - return formattedContent; -} - -export async function startChat(initialCollections: string[]) { - let collections = initialCollections; - let contextContent = formatCollectionItems(collections); - - let systemPrompt = `You are an AI assistant engaging in a conversation about the following collections of items. Use this context to inform your responses: - -${contextContent} - -Respond conversationally and draw upon the context provided when relevant.`; - - console.log( - "Starting chat. Type '/exit' to leave, '/drop [collection]' to remove a collection, or '/add [collection]' to add a collection.", - ); - - const messages: CoreMessage[] = []; - - function refreshContext() { - contextContent = formatCollectionItems(collections); - systemPrompt = `You are an AI assistant engaging in a conversation about the following collections of items. Use this context to inform your responses: - - ${contextContent} - - Respond conversationally and draw upon the context provided when relevant.`; - } - - while (true) { - const userInput = prompt("User: "); - if (!userInput) continue; - - if (userInput.startsWith("/")) { - const [command, ...args] = userInput.slice(1).split(" "); - switch (command) { - case "exit": - console.log("Exiting chat mode."); - return; - case "drop": - if (args.length === 0) { - console.log("Usage: /drop [collection]"); - } else { - const collectionToDrop = args.join(" "); - collections = collections.filter((c) => c !== collectionToDrop); - console.log(`Dropped collection: ${collectionToDrop}`); - refreshContext(); - } - break; - case "add": - if (args.length === 0) { - console.log("Usage: /add [collection]"); - } else { - const collectionToAdd = args.join(" "); - if (!collections.includes(collectionToAdd)) { - collections.push(collectionToAdd); - console.log(`Added collection: ${collectionToAdd}`); - refreshContext(); - } else { - console.log(`Collection ${collectionToAdd} is already included.`); - } - } - break; - case "refresh": - console.log("Refreshing context."); - refreshContext(); - break; - default: - console.log( - "Unknown command. Available commands: /exit, /drop [collection], /add [collection], /refresh", - ); - } - } else { - messages.push({ role: "user", content: userInput }); - - const response = await chat(systemPrompt, messages); - - messages.push({ role: "assistant", content: response }); - } - } -} diff --git a/typescript/packages/collectathon/chrome-extension/background.js b/typescript/packages/collectathon/chrome-extension/background.js deleted file mode 100644 index 7f97290de..000000000 --- a/typescript/packages/collectathon/chrome-extension/background.js +++ /dev/null @@ -1,38 +0,0 @@ -chrome.runtime.onInstalled.addListener(() => { - chrome.contextMenus.create({ - id: "clipToCollectathon", - title: "Clip to Collectathon", - contexts: ["selection", "link", "image", "video", "audio"] - }); -}); - -chrome.contextMenus.onClicked.addListener((info, tab) => { - if (info.menuItemId === "clipToCollectathon") { - let content = {}; - - if (info.selectionText) { - content.type = "text"; - content.text = info.selectionText; - } else if (info.linkUrl) { - content.type = "link"; - content.url = info.linkUrl; - } else if (info.srcUrl) { - content.type = info.mediaType || "image"; - content.url = info.srcUrl; - } - - content.pageUrl = info.pageUrl; - - chrome.storage.local.set({ clipContent: content }, () => { - chrome.action.openPopup(); - }); - } -}); - -chrome.runtime.onConnect.addListener(function(port) { - if (port.name === "popup") { - port.onDisconnect.addListener(function() { - chrome.storage.local.remove('clipContent'); - }); - } -}); diff --git a/typescript/packages/collectathon/chrome-extension/manifest.json b/typescript/packages/collectathon/chrome-extension/manifest.json deleted file mode 100644 index 4ad6db7da..000000000 --- a/typescript/packages/collectathon/chrome-extension/manifest.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "manifest_version": 3, - "name": "Common Clipper", - "version": "1.0", - "description": "Clip URLs and selected content to Collectathon", - "permissions": ["activeTab", "contextMenus", "storage", "scripting"], - "background": { - "service_worker": "background.js" - }, - "action": { - "default_popup": "popup.html" - } -} diff --git a/typescript/packages/collectathon/chrome-extension/popup.html b/typescript/packages/collectathon/chrome-extension/popup.html deleted file mode 100644 index b8bdd3fc0..000000000 --- a/typescript/packages/collectathon/chrome-extension/popup.html +++ /dev/null @@ -1,40 +0,0 @@ - - - - Collectathon Clipper - - - - -

Common Clipper

- -
-

Suggested Collections

-
-
-

Search Results

-
- -
- - - - - diff --git a/typescript/packages/collectathon/chrome-extension/popup.js b/typescript/packages/collectathon/chrome-extension/popup.js deleted file mode 100644 index cfe186644..000000000 --- a/typescript/packages/collectathon/chrome-extension/popup.js +++ /dev/null @@ -1,222 +0,0 @@ -const API_URL = "http://localhost:8001"; -let clipContent = null; - -chrome.runtime.connect({ name: "popup" }); - -function displayClippedContent() { - const contentDiv = document.getElementById("clippedContent"); - if (clipContent) { - let contentHtml = `

Clipped Content:

`; - if (clipContent.type === "text") { - contentHtml += `

${clipContent.text}

`; - } else if (clipContent.type === "link") { - contentHtml += `

Link: ${clipContent.url}

`; - } else if (clipContent.type === "image") { - contentHtml += ``; - } else { - contentHtml += `

${clipContent.type}: ${clipContent.url}

`; - } - contentDiv.innerHTML = contentHtml; - } else { - contentDiv.innerHTML = ""; - } -} - -function clearClipContent() { - chrome.storage.local.remove("clipContent", () => { - clipContent = null; - displayClippedContent(); - }); -} - -function slugify(text) { - return text - .toString() - .toLowerCase() - .replace(/\s+/g, "-") // Replace spaces with - - .replace(/[^\w\-]+/g, "") // Remove all non-word chars - .replace(/\-\-+/g, "-") // Replace multiple - with single - - .replace(/^-+/, "") // Trim - from start of text - .replace(/-+$/, ""); // Trim - from end of text -} - -function getPageSource() { - return document.documentElement.outerHTML; -} - -async function clipContentToCollection(collections, prompt = "") { - return new Promise((resolve, reject) => { - chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => { - const url = clipContent ? clipContent.pageUrl : tabs[0].url; - const content = clipContent || { type: "webpage", url: url }; - await new Promise((resolveSource) => { - let activeTab = tabs[0]; - chrome.scripting.executeScript( - { - target: { tabId: activeTab.id }, - function: getPageSource, - }, - (injectionResults) => { - for (const frameResult of injectionResults) { - console.log("Frame HTML:", frameResult.result); - content.html = frameResult.result; - resolveSource(); - } - }, - ); - }); - - fetch(`${API_URL}/clip`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - url: url, - collections: collections, - prompt: prompt, - content: content, - }), - }) - .then((response) => response.json()) - .then((data) => { - clearClipContent(); - resolve(data); - }) - .catch((error) => { - console.error("Error:", error); - reject(error); - }); - }); - }); -} - -function openCommonOS() { - const COMMON_OS_URL = "http://localhost:5173"; - const newTab = window.open(COMMON_OS_URL, "_blank"); - newTab.focus(); - window.close(); -} - -document.addEventListener("DOMContentLoaded", () => { - const searchInput = document.getElementById("searchInput"); - const recentCollections = document.getElementById("recentCollections"); - const searchResults = document.getElementById("searchResults"); - - document - .getElementById("openCommonOSButton") - .addEventListener("click", openCommonOS); - - document - .getElementById("openInCommonOSButton") - .addEventListener("click", function () { - clipContentToCollection(["inbox"]) - .then(() => { - openCommonOS(); - }) - .catch((error) => { - alert("An error occurred while clipping the content to inbox."); - }); - }); - - chrome.tabs.query( - { active: true, currentWindow: true }, - async function (tabs) { - const currentUrl = tabs[0].url; - // Load recent collections - fetch( - `${API_URL}/suggested-collections?url=${encodeURIComponent(currentUrl)}`, - ) - .then((response) => response.json()) - .then((collections) => { - recentCollections.innerHTML = - "

Suggested Collections

" + - collections - .map( - (collection) => ` -
- - -
- `, - ) - .join(""); - }); - }, - ); - - // Search collections - searchInput.addEventListener("input", () => { - const query = searchInput.value; - if (query) { - fetch(`${API_URL}/search-collections?q=${encodeURIComponent(query)}`) - .then((response) => response.json()) - .then((collections) => { - let resultsHtml = "

Search Results

"; - collections.forEach((collection) => { - resultsHtml += ` -
- - -
- `; - }); - - // Check if the exact query is not in the results - if (!collections.includes(query)) { - const slugifiedQuery = slugify(query); - resultsHtml += ` -
- - -
- `; - } - - searchResults.innerHTML = resultsHtml; - }); - } else { - searchResults.innerHTML = "

Search Results

"; - } - }); - - chrome.storage.local.get(["clipContent"], (result) => { - clipContent = result.clipContent; - displayClippedContent(); - }); -}); - -function getSelectedCollections() { - const checkboxes = document.querySelectorAll( - 'input[type="checkbox"]:checked', - ); - return Array.from(checkboxes).map((checkbox) => checkbox.name); -} - -document.getElementById("clipButton").addEventListener("click", () => { - const selectedCollections = getSelectedCollections(); - const prompt = document.getElementById("prompt").value; - - if (selectedCollections.length === 0) { - alert("Please select at least one collection."); - return; - } - - clipContentToCollection(selectedCollections, prompt) - .then((data) => { - alert("Content clipped successfully!"); - - // Create a pre element to display the response - const responseElement = document.createElement("pre"); - responseElement.style.height = "200px"; - responseElement.style.overflowY = "scroll"; - responseElement.textContent = JSON.stringify(data, null, 2); - - // Append the response element to the body - document.body.appendChild(responseElement); - }) - .catch((error) => { - console.error("Error:", error); - alert("An error occurred while clipping the content."); - }); -}); diff --git a/typescript/packages/collectathon/collections.ts b/typescript/packages/collectathon/collections.ts deleted file mode 100644 index 9b40f28d8..000000000 --- a/typescript/packages/collectathon/collections.ts +++ /dev/null @@ -1,153 +0,0 @@ -// collections.ts -import { db } from "./db.ts"; -import { Table } from "./deps.ts"; - -export async function getOrCreateCollection(name: string): Promise { - const existing = await db.query("SELECT id FROM collections WHERE name = ?", [ - name, - ]); - if (existing.length > 0) { - return existing[0][0] as number; - } - const result = db.query( - "INSERT INTO collections (name) VALUES (?) RETURNING id", - [name], - ); - return result[0][0] as number; -} - -export async function listCollections() { - const collections = db.query<[number, string, number]>(` - SELECT c.id, c.name, COUNT(ic.item_id) as item_count - FROM collections c - LEFT JOIN item_collections ic ON c.id = ic.collection_id - GROUP BY c.id - ORDER BY c.name - `); - const t = new Table(); - - collections.forEach((collection) => { - t.cell("ID", collection[0]); - t.cell("Name", collection[1]); - t.cell("Item Count", collection[2], Table.number(0)); - t.newRow(); - }); - - console.log("Collections:"); - console.log(t.toString()); - - const formattedCollections = collections.map((collection) => ({ - id: collection[0], - name: collection[1], - itemCount: collection[2] - })); - - return formattedCollections; -} - -export async function listItems(collectionName: string) { - const items = db.query<[number, string, string]>( - ` - SELECT i.id, i.title, REPLACE(SUBSTR(i.raw_content, 1, 32), CHAR(10), ' ') as snippet - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ? - ORDER BY i.id - `, - [collectionName], - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collectionName}`); - return; - } - - console.log(`Items in collection "${collectionName}":`); - const t = new Table(); - - items.forEach((item) => { - t.cell("ID", item[0]); - t.cell("Title", item[1]); - t.cell("Snippet", item[2]); - t.newRow(); - }); - console.log(t.toString()); - - const formattedItems = items.map((item) => ({ - id: item[0], - title: item[1], - snippet: item[2] - })); - - return formattedItems; -} - -export async function addItemToCollection( - itemId: number, - collectionName: string, - silent = false, -) { - try { - const collectionId = await getOrCreateCollection(collectionName); - await db.query( - "INSERT OR IGNORE INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - if (!silent) { - console.log(`Added item ${itemId} to collection "${collectionName}"`); - } - } catch (error) { - console.error(`Error adding item to collection: ${error.message}`); - } -} - -export async function removeItemFromCollection( - itemId: number, - collectionName: string, -) { - try { - db.query( - ` - DELETE FROM item_collections - WHERE item_id = ? AND collection_id = (SELECT id FROM collections WHERE name = ?) - `, - [itemId, collectionName], - ); - console.log(`Removed item ${itemId} from collection "${collectionName}"`); - } catch (error) { - console.error(`Error removing item from collection: ${error.message}`); - } -} - -export async function deleteCollection(collectionName: string) { - try { - db.query("BEGIN TRANSACTION"); - db.query( - "DELETE FROM item_collections WHERE collection_id = (SELECT id FROM collections WHERE name = ?)", - [collectionName], - ); - db.query("DELETE FROM collections WHERE name = ?", [collectionName]); - db.query("COMMIT"); - console.log(`Deleted collection "${collectionName}"`); - } catch (error) { - db.query("ROLLBACK"); - console.error(`Error deleting collection: ${error.message}`); - } -} - -export async function moveCollection(oldName: string, newName: string) { - try { - const result = db.query("UPDATE collections SET name = ? WHERE name = ?", [ - newName, - oldName, - ]); - if (result.affectedRows > 0) { - console.log(`Renamed collection "${oldName}" to "${newName}"`); - } else { - console.log(`Collection "${oldName}" not found`); - } - } catch (error) { - console.error(`Error renaming collection: ${error.message}`); - } -} diff --git a/typescript/packages/collectathon/db.ts b/typescript/packages/collectathon/db.ts deleted file mode 100644 index 28c959f61..000000000 --- a/typescript/packages/collectathon/db.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { DB } from "./deps.ts"; - -export const db = new DB("collections.db"); - -// Create tables -db.execute(` - CREATE TABLE IF NOT EXISTS items ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - url TEXT, - title TEXT, - content JSON, - raw_content TEXT, - source TEXT, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ) -`); - -db.execute(` - CREATE TABLE IF NOT EXISTS collections ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE - ) -`); - -db.execute(` - CREATE TABLE IF NOT EXISTS item_collections ( - item_id INTEGER, - collection_id INTEGER, - PRIMARY KEY (item_id, collection_id), - FOREIGN KEY (item_id) REFERENCES items(id), - FOREIGN KEY (collection_id) REFERENCES collections(id) - ) -`); - -db.execute(` - CREATE TABLE IF NOT EXISTS rules ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - collection_name TEXT, - rule TEXT, - target_collection TEXT - ) -`); - -db.execute(` - CREATE TABLE IF NOT EXISTS views ( - id TEXT PRIMARY KEY, - collection TEXT, - html TEXT, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ) -`); diff --git a/typescript/packages/collectathon/deno.lock b/typescript/packages/collectathon/deno.lock deleted file mode 100644 index 187a3df4c..000000000 --- a/typescript/packages/collectathon/deno.lock +++ /dev/null @@ -1,818 +0,0 @@ -{ - "version": "3", - "packages": { - "specifiers": { - "npm:@ai-sdk/anthropic@0.0.50": "npm:@ai-sdk/anthropic@0.0.50_zod@3.23.8", - "npm:@ai-sdk/openai@0.0.60": "npm:@ai-sdk/openai@0.0.60_zod@3.23.8", - "npm:ai@3.3.21": "npm:ai@3.3.21", - "npm:ai@3.3.39": "npm:ai@3.3.39", - "npm:easy-table": "npm:easy-table@1.2.0", - "npm:multiformats@13.3.0": "npm:multiformats@13.3.0" - }, - "npm": { - "@ai-sdk/anthropic@0.0.50_zod@3.23.8": { - "integrity": "sha512-++mqmFcUoQgjoCchAU6eVG3QfKdwkeJVNdMZ+jUiNdawn8diA6BlARlu7xFT4F7W3bcStfYv4hK1jwRyzAQtCg==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19_zod@3.23.8", - "zod": "zod@3.23.8" - } - }, - "@ai-sdk/openai@0.0.60_zod@3.23.8": { - "integrity": "sha512-NEdDdv3o76jT6UeWHxP6I/lMYcjFQhQGQi/U2gVqW1PEU4Pjaud7tAVSy27IPbiRakg6GOzWrltI2JhZgAI1wg==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19_zod@3.23.8", - "zod": "zod@3.23.8" - } - }, - "@ai-sdk/provider-utils@1.0.17": { - "integrity": "sha512-2VyeTH5DQ6AxqvwdyytKIeiZyYTyJffpufWjE67zM2sXMIHgYl7fivo8m5wVl6Cbf1dFPSGKq//C9s+lz+NHrQ==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.22", - "eventsource-parser": "eventsource-parser@1.1.2", - "nanoid": "nanoid@3.3.6", - "secure-json-parse": "secure-json-parse@2.7.0" - } - }, - "@ai-sdk/provider-utils@1.0.19": { - "integrity": "sha512-p02Fq5Mnc8T6nwRBN1Iaou8YXvN1sDS6hbmJaD5UaRbXjizbh+8rpFS/o7jqAHTwf3uHCDitP3pnODyHdc/CDQ==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "eventsource-parser": "eventsource-parser@1.1.2", - "nanoid": "nanoid@3.3.6", - "secure-json-parse": "secure-json-parse@2.7.0" - } - }, - "@ai-sdk/provider-utils@1.0.19_zod@3.23.8": { - "integrity": "sha512-p02Fq5Mnc8T6nwRBN1Iaou8YXvN1sDS6hbmJaD5UaRbXjizbh+8rpFS/o7jqAHTwf3uHCDitP3pnODyHdc/CDQ==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "eventsource-parser": "eventsource-parser@1.1.2", - "nanoid": "nanoid@3.3.6", - "secure-json-parse": "secure-json-parse@2.7.0", - "zod": "zod@3.23.8" - } - }, - "@ai-sdk/provider@0.0.22": { - "integrity": "sha512-smZ1/2jL/JSKnbhC6ama/PxI2D/psj+YAe0c0qpd5ComQCNFltg72VFf0rpUSFMmFuj1pCCNoBOCrvyl8HTZHQ==", - "dependencies": { - "json-schema": "json-schema@0.4.0" - } - }, - "@ai-sdk/provider@0.0.23": { - "integrity": "sha512-oAc49O5+xypVrKM7EUU5P/Y4DUL4JZUWVxhejoAVOTOl3WZUEWsMbP3QZR+TrimQIsS0WR/n9UuF6U0jPdp0tQ==", - "dependencies": { - "json-schema": "json-schema@0.4.0" - } - }, - "@ai-sdk/react@0.0.53": { - "integrity": "sha512-sIsmTFoR/QHvUUkltmHwP4bPjwy2vko6j/Nj8ayxLhEHs04Ug+dwXQyfA7MwgimEE3BcDQpWL8ikVj0m3ZILWQ==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.40", - "swr": "swr@2.2.5_react@18.3.1" - } - }, - "@ai-sdk/react@0.0.59": { - "integrity": "sha512-1WbgO3J2/OoheMuNMxy5itJ3NVqOpqpAQxFNp7AoXgnDv4wDF4kTif61rTlKh7dCPvBHj2HXLmob+TrVFaWhYw==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.44", - "swr": "swr@2.2.5_react@18.3.1" - } - }, - "@ai-sdk/solid@0.0.43": { - "integrity": "sha512-7PlPLaeMAu97oOY2gjywvKZMYHF+GDfUxYNcuJ4AZ3/MRBatzs/U2r4ClT1iH8uMOcMg02RX6UKzP5SgnUBjVw==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.40" - } - }, - "@ai-sdk/solid@0.0.47": { - "integrity": "sha512-lVMxIxtuNqoo/TObSFGflEP2dUeJv7bfPQbS4jHTZGBNlyhgBRY2Xc19yNjA3QKRfvQNDVoQusqxn+18MiHJJQ==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.44" - } - }, - "@ai-sdk/svelte@0.0.45": { - "integrity": "sha512-w5Sdl0ArFIM3Fp8BbH4TUvlrS84WP/jN/wC1+fghMOXd7ceVO3Yhs9r71wTqndhgkLC7LAEX9Ll7ZEPfW9WBDA==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.40", - "sswr": "sswr@2.1.0_svelte@4.2.19" - } - }, - "@ai-sdk/svelte@0.0.49": { - "integrity": "sha512-gV0MhaWxkatjf7uJrCAHO3bWrihokNUwGhuMCgyG+y53lwJKAYhR0zCoDRM2HnTJ89fdnx/PVe3R9fOWEVY5qA==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.44", - "sswr": "sswr@2.1.0_svelte@4.2.19" - } - }, - "@ai-sdk/ui-utils@0.0.40": { - "integrity": "sha512-f0eonPUBO13pIO8jA9IGux7IKMeqpvWK22GBr3tOoSRnO5Wg5GEpXZU1V0Po+unpeZHyEPahrWbj5JfXcyWCqw==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.22", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "json-schema": "json-schema@0.4.0", - "secure-json-parse": "secure-json-parse@2.7.0", - "zod-to-json-schema": "zod-to-json-schema@3.23.2_zod@3.23.8" - } - }, - "@ai-sdk/ui-utils@0.0.44": { - "integrity": "sha512-0qiyun/n5zqJzQs/WfQT86dZE5DiDhSHJc7b7ZGLYvNMztHkRQmak2zUCZP4IyGVZEicyEPQK6NEEpBgkmd3Dg==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "json-schema": "json-schema@0.4.0", - "secure-json-parse": "secure-json-parse@2.7.0", - "zod-to-json-schema": "zod-to-json-schema@3.23.2_zod@3.23.8" - } - }, - "@ai-sdk/vue@0.0.45": { - "integrity": "sha512-bqeoWZqk88TQmfoPgnFUKkrvhOIcOcSH5LMPgzZ8XwDqz5tHHrMHzpPfHCj7XyYn4ROTFK/2kKdC/ta6Ko0fMw==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.40", - "swrv": "swrv@1.0.4_vue@3.5.11" - } - }, - "@ai-sdk/vue@0.0.49": { - "integrity": "sha512-GLjk5uhn0dA8iXpqdF91NyOw+VCgDIo22zrdkRtDg+nLaqkFSjgdDLAp7CL+ihW4F0/IkpZym3j0lFi9LiCjZA==", - "dependencies": { - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.44", - "swrv": "swrv@1.0.4_vue@3.5.11" - } - }, - "@ampproject/remapping@2.3.0": { - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "dependencies": { - "@jridgewell/gen-mapping": "@jridgewell/gen-mapping@0.3.5", - "@jridgewell/trace-mapping": "@jridgewell/trace-mapping@0.3.25" - } - }, - "@babel/helper-string-parser@7.25.7": { - "integrity": "sha512-CbkjYdsJNHFk8uqpEkpCvRs3YRp9tY6FmFY7wLMSYuGYkrdUi7r2lc4/wqsvlHoMznX3WJ9IP8giGPq68T/Y6g==", - "dependencies": {} - }, - "@babel/helper-validator-identifier@7.25.7": { - "integrity": "sha512-AM6TzwYqGChO45oiuPqwL2t20/HdMC1rTPAesnBCgPCSF1x3oN9MVUwQV2iyz4xqWrctwK5RNC8LV22kaQCNYg==", - "dependencies": {} - }, - "@babel/parser@7.25.7": { - "integrity": "sha512-aZn7ETtQsjjGG5HruveUK06cU3Hljuhd9Iojm4M8WWv3wLE6OkE5PWbDUkItmMgegmccaITudyuW5RPYrYlgWw==", - "dependencies": { - "@babel/types": "@babel/types@7.25.7" - } - }, - "@babel/types@7.25.7": { - "integrity": "sha512-vwIVdXG+j+FOpkwqHRcBgHLYNL7XMkufrlaFvL9o6Ai9sJn9+PdyIL5qa0XzTZw084c+u9LOls53eoZWP/W5WQ==", - "dependencies": { - "@babel/helper-string-parser": "@babel/helper-string-parser@7.25.7", - "@babel/helper-validator-identifier": "@babel/helper-validator-identifier@7.25.7", - "to-fast-properties": "to-fast-properties@2.0.0" - } - }, - "@jridgewell/gen-mapping@0.3.5": { - "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", - "dependencies": { - "@jridgewell/set-array": "@jridgewell/set-array@1.2.1", - "@jridgewell/sourcemap-codec": "@jridgewell/sourcemap-codec@1.5.0", - "@jridgewell/trace-mapping": "@jridgewell/trace-mapping@0.3.25" - } - }, - "@jridgewell/resolve-uri@3.1.2": { - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dependencies": {} - }, - "@jridgewell/set-array@1.2.1": { - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dependencies": {} - }, - "@jridgewell/sourcemap-codec@1.5.0": { - "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", - "dependencies": {} - }, - "@jridgewell/trace-mapping@0.3.25": { - "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", - "dependencies": { - "@jridgewell/resolve-uri": "@jridgewell/resolve-uri@3.1.2", - "@jridgewell/sourcemap-codec": "@jridgewell/sourcemap-codec@1.5.0" - } - }, - "@opentelemetry/api@1.9.0": { - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "dependencies": {} - }, - "@types/diff-match-patch@1.0.36": { - "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", - "dependencies": {} - }, - "@types/estree@1.0.6": { - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "dependencies": {} - }, - "@vue/compiler-core@3.5.11": { - "integrity": "sha512-PwAdxs7/9Hc3ieBO12tXzmTD+Ln4qhT/56S+8DvrrZ4kLDn4Z/AMUr8tXJD0axiJBS0RKIoNaR0yMuQB9v9Udg==", - "dependencies": { - "@babel/parser": "@babel/parser@7.25.7", - "@vue/shared": "@vue/shared@3.5.11", - "entities": "entities@4.5.0", - "estree-walker": "estree-walker@2.0.2", - "source-map-js": "source-map-js@1.2.1" - } - }, - "@vue/compiler-dom@3.5.11": { - "integrity": "sha512-pyGf8zdbDDRkBrEzf8p7BQlMKNNF5Fk/Cf/fQ6PiUz9at4OaUfyXW0dGJTo2Vl1f5U9jSLCNf0EZJEogLXoeew==", - "dependencies": { - "@vue/compiler-core": "@vue/compiler-core@3.5.11", - "@vue/shared": "@vue/shared@3.5.11" - } - }, - "@vue/compiler-sfc@3.5.11": { - "integrity": "sha512-gsbBtT4N9ANXXepprle+X9YLg2htQk1sqH/qGJ/EApl+dgpUBdTv3yP7YlR535uHZY3n6XaR0/bKo0BgwwDniw==", - "dependencies": { - "@babel/parser": "@babel/parser@7.25.7", - "@vue/compiler-core": "@vue/compiler-core@3.5.11", - "@vue/compiler-dom": "@vue/compiler-dom@3.5.11", - "@vue/compiler-ssr": "@vue/compiler-ssr@3.5.11", - "@vue/shared": "@vue/shared@3.5.11", - "estree-walker": "estree-walker@2.0.2", - "magic-string": "magic-string@0.30.11", - "postcss": "postcss@8.4.47", - "source-map-js": "source-map-js@1.2.1" - } - }, - "@vue/compiler-ssr@3.5.11": { - "integrity": "sha512-P4+GPjOuC2aFTk1Z4WANvEhyOykcvEd5bIj2KVNGKGfM745LaXGr++5njpdBTzVz5pZifdlR1kpYSJJpIlSePA==", - "dependencies": { - "@vue/compiler-dom": "@vue/compiler-dom@3.5.11", - "@vue/shared": "@vue/shared@3.5.11" - } - }, - "@vue/reactivity@3.5.11": { - "integrity": "sha512-Nqo5VZEn8MJWlCce8XoyVqHZbd5P2NH+yuAaFzuNSR96I+y1cnuUiq7xfSG+kyvLSiWmaHTKP1r3OZY4mMD50w==", - "dependencies": { - "@vue/shared": "@vue/shared@3.5.11" - } - }, - "@vue/runtime-core@3.5.11": { - "integrity": "sha512-7PsxFGqwfDhfhh0OcDWBG1DaIQIVOLgkwA5q6MtkPiDFjp5gohVnJEahSktwSFLq7R5PtxDKy6WKURVN1UDbzA==", - "dependencies": { - "@vue/reactivity": "@vue/reactivity@3.5.11", - "@vue/shared": "@vue/shared@3.5.11" - } - }, - "@vue/runtime-dom@3.5.11": { - "integrity": "sha512-GNghjecT6IrGf0UhuYmpgaOlN7kxzQBhxWEn08c/SQDxv1yy4IXI1bn81JgEpQ4IXjRxWtPyI8x0/7TF5rPfYQ==", - "dependencies": { - "@vue/reactivity": "@vue/reactivity@3.5.11", - "@vue/runtime-core": "@vue/runtime-core@3.5.11", - "@vue/shared": "@vue/shared@3.5.11", - "csstype": "csstype@3.1.3" - } - }, - "@vue/server-renderer@3.5.11_vue@3.5.11": { - "integrity": "sha512-cVOwYBxR7Wb1B1FoxYvtjJD8X/9E5nlH4VSkJy2uMA1MzYNdzAAB//l8nrmN9py/4aP+3NjWukf9PZ3TeWULaA==", - "dependencies": { - "@vue/compiler-ssr": "@vue/compiler-ssr@3.5.11", - "@vue/shared": "@vue/shared@3.5.11", - "vue": "vue@3.5.11" - } - }, - "@vue/shared@3.5.11": { - "integrity": "sha512-W8GgysJVnFo81FthhzurdRAWP/byq3q2qIw70e0JWblzVhjgOMiC2GyovXrZTFQJnFVryYaKGP3Tc9vYzYm6PQ==", - "dependencies": {} - }, - "acorn@8.12.1": { - "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", - "dependencies": {} - }, - "ai@3.3.21": { - "integrity": "sha512-gUkYEpghc2/+/2ZO82GSf3R2nMXiC0ZZBvhubetVTK93RW1ts/wuuLig7SH3K3vPgj3us0XK67/9FC1XOFwupg==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.22", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.17", - "@ai-sdk/react": "@ai-sdk/react@0.0.53", - "@ai-sdk/solid": "@ai-sdk/solid@0.0.43", - "@ai-sdk/svelte": "@ai-sdk/svelte@0.0.45", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.40", - "@ai-sdk/vue": "@ai-sdk/vue@0.0.45", - "@opentelemetry/api": "@opentelemetry/api@1.9.0", - "eventsource-parser": "eventsource-parser@1.1.2", - "json-schema": "json-schema@0.4.0", - "jsondiffpatch": "jsondiffpatch@0.6.0", - "nanoid": "nanoid@3.3.6", - "secure-json-parse": "secure-json-parse@2.7.0", - "zod-to-json-schema": "zod-to-json-schema@3.23.2_zod@3.23.8" - } - }, - "ai@3.3.39": { - "integrity": "sha512-6/URulDBjhRE0r/gircxjiRd62tUl37rrGL3NLAGen5V9w9m43dBhUING906tMEJmr6xPvDUlBSmh0bdpWzHMQ==", - "dependencies": { - "@ai-sdk/provider": "@ai-sdk/provider@0.0.23", - "@ai-sdk/provider-utils": "@ai-sdk/provider-utils@1.0.19", - "@ai-sdk/react": "@ai-sdk/react@0.0.59", - "@ai-sdk/solid": "@ai-sdk/solid@0.0.47", - "@ai-sdk/svelte": "@ai-sdk/svelte@0.0.49", - "@ai-sdk/ui-utils": "@ai-sdk/ui-utils@0.0.44", - "@ai-sdk/vue": "@ai-sdk/vue@0.0.49", - "@opentelemetry/api": "@opentelemetry/api@1.9.0", - "eventsource-parser": "eventsource-parser@1.1.2", - "json-schema": "json-schema@0.4.0", - "jsondiffpatch": "jsondiffpatch@0.6.0", - "nanoid": "nanoid@3.3.6", - "secure-json-parse": "secure-json-parse@2.7.0", - "zod-to-json-schema": "zod-to-json-schema@3.23.2_zod@3.23.8" - } - }, - "ansi-regex@5.0.1": { - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dependencies": {} - }, - "aria-query@5.3.2": { - "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", - "dependencies": {} - }, - "axobject-query@4.1.0": { - "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", - "dependencies": {} - }, - "chalk@5.3.0": { - "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", - "dependencies": {} - }, - "client-only@0.0.1": { - "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", - "dependencies": {} - }, - "clone@1.0.4": { - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "dependencies": {} - }, - "code-red@1.0.4": { - "integrity": "sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==", - "dependencies": { - "@jridgewell/sourcemap-codec": "@jridgewell/sourcemap-codec@1.5.0", - "@types/estree": "@types/estree@1.0.6", - "acorn": "acorn@8.12.1", - "estree-walker": "estree-walker@3.0.3", - "periscopic": "periscopic@3.1.0" - } - }, - "css-tree@2.3.1": { - "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==", - "dependencies": { - "mdn-data": "mdn-data@2.0.30", - "source-map-js": "source-map-js@1.2.1" - } - }, - "csstype@3.1.3": { - "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "dependencies": {} - }, - "defaults@1.0.4": { - "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "dependencies": { - "clone": "clone@1.0.4" - } - }, - "diff-match-patch@1.0.5": { - "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", - "dependencies": {} - }, - "easy-table@1.2.0": { - "integrity": "sha512-OFzVOv03YpvtcWGe5AayU5G2hgybsg3iqA6drU8UaoZyB9jLGMTrz9+asnLp/E+6qPh88yEI1gvyZFZ41dmgww==", - "dependencies": { - "ansi-regex": "ansi-regex@5.0.1", - "wcwidth": "wcwidth@1.0.1" - } - }, - "entities@4.5.0": { - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dependencies": {} - }, - "estree-walker@2.0.2": { - "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", - "dependencies": {} - }, - "estree-walker@3.0.3": { - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dependencies": { - "@types/estree": "@types/estree@1.0.6" - } - }, - "eventsource-parser@1.1.2": { - "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", - "dependencies": {} - }, - "is-reference@3.0.2": { - "integrity": "sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==", - "dependencies": { - "@types/estree": "@types/estree@1.0.6" - } - }, - "js-tokens@4.0.0": { - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dependencies": {} - }, - "json-schema@0.4.0": { - "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", - "dependencies": {} - }, - "jsondiffpatch@0.6.0": { - "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", - "dependencies": { - "@types/diff-match-patch": "@types/diff-match-patch@1.0.36", - "chalk": "chalk@5.3.0", - "diff-match-patch": "diff-match-patch@1.0.5" - } - }, - "locate-character@3.0.0": { - "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", - "dependencies": {} - }, - "loose-envify@1.4.0": { - "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", - "dependencies": { - "js-tokens": "js-tokens@4.0.0" - } - }, - "magic-string@0.30.11": { - "integrity": "sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==", - "dependencies": { - "@jridgewell/sourcemap-codec": "@jridgewell/sourcemap-codec@1.5.0" - } - }, - "mdn-data@2.0.30": { - "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", - "dependencies": {} - }, - "multiformats@13.3.0": { - "integrity": "sha512-CBiqvsufgmpo01VT5ze94O+uc+Pbf6f/sThlvWss0sBZmAOu6GQn5usrYV2sf2mr17FWYc0rO8c/CNe2T90QAA==", - "dependencies": {} - }, - "nanoid@3.3.6": { - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dependencies": {} - }, - "nanoid@3.3.7": { - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", - "dependencies": {} - }, - "periscopic@3.1.0": { - "integrity": "sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==", - "dependencies": { - "@types/estree": "@types/estree@1.0.6", - "estree-walker": "estree-walker@3.0.3", - "is-reference": "is-reference@3.0.2" - } - }, - "picocolors@1.1.0": { - "integrity": "sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw==", - "dependencies": {} - }, - "postcss@8.4.47": { - "integrity": "sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==", - "dependencies": { - "nanoid": "nanoid@3.3.7", - "picocolors": "picocolors@1.1.0", - "source-map-js": "source-map-js@1.2.1" - } - }, - "react@18.3.1": { - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", - "dependencies": { - "loose-envify": "loose-envify@1.4.0" - } - }, - "secure-json-parse@2.7.0": { - "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", - "dependencies": {} - }, - "source-map-js@1.2.1": { - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dependencies": {} - }, - "sswr@2.1.0_svelte@4.2.19": { - "integrity": "sha512-Cqc355SYlTAaUt8iDPaC/4DPPXK925PePLMxyBKuWd5kKc5mwsG3nT9+Mq2tyguL5s7b4Jg+IRMpTRsNTAfpSQ==", - "dependencies": { - "svelte": "svelte@4.2.19", - "swrev": "swrev@4.0.0" - } - }, - "svelte@4.2.19": { - "integrity": "sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==", - "dependencies": { - "@ampproject/remapping": "@ampproject/remapping@2.3.0", - "@jridgewell/sourcemap-codec": "@jridgewell/sourcemap-codec@1.5.0", - "@jridgewell/trace-mapping": "@jridgewell/trace-mapping@0.3.25", - "@types/estree": "@types/estree@1.0.6", - "acorn": "acorn@8.12.1", - "aria-query": "aria-query@5.3.2", - "axobject-query": "axobject-query@4.1.0", - "code-red": "code-red@1.0.4", - "css-tree": "css-tree@2.3.1", - "estree-walker": "estree-walker@3.0.3", - "is-reference": "is-reference@3.0.2", - "locate-character": "locate-character@3.0.0", - "magic-string": "magic-string@0.30.11", - "periscopic": "periscopic@3.1.0" - } - }, - "swr@2.2.5_react@18.3.1": { - "integrity": "sha512-QtxqyclFeAsxEUeZIYmsaQ0UjimSq1RZ9Un7I68/0ClKK/U3LoyQunwkQfJZr2fc22DfIXLNDc2wFyTEikCUpg==", - "dependencies": { - "client-only": "client-only@0.0.1", - "react": "react@18.3.1", - "use-sync-external-store": "use-sync-external-store@1.2.2_react@18.3.1" - } - }, - "swrev@4.0.0": { - "integrity": "sha512-LqVcOHSB4cPGgitD1riJ1Hh4vdmITOp+BkmfmXRh4hSF/t7EnS4iD+SOTmq7w5pPm/SiPeto4ADbKS6dHUDWFA==", - "dependencies": {} - }, - "swrv@1.0.4_vue@3.5.11": { - "integrity": "sha512-zjEkcP8Ywmj+xOJW3lIT65ciY/4AL4e/Or7Gj0MzU3zBJNMdJiT8geVZhINavnlHRMMCcJLHhraLTAiDOTmQ9g==", - "dependencies": { - "vue": "vue@3.5.11" - } - }, - "to-fast-properties@2.0.0": { - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dependencies": {} - }, - "use-sync-external-store@1.2.2_react@18.3.1": { - "integrity": "sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw==", - "dependencies": { - "react": "react@18.3.1" - } - }, - "vue@3.5.11": { - "integrity": "sha512-/8Wurrd9J3lb72FTQS7gRMNQD4nztTtKPmuDuPuhqXmmpD6+skVjAeahNpVzsuky6Sy9gy7wn8UadqPtt9SQIg==", - "dependencies": { - "@vue/compiler-dom": "@vue/compiler-dom@3.5.11", - "@vue/compiler-sfc": "@vue/compiler-sfc@3.5.11", - "@vue/runtime-dom": "@vue/runtime-dom@3.5.11", - "@vue/server-renderer": "@vue/server-renderer@3.5.11_vue@3.5.11", - "@vue/shared": "@vue/shared@3.5.11" - } - }, - "wcwidth@1.0.1": { - "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "dependencies": { - "defaults": "defaults@1.0.4" - } - }, - "zod-to-json-schema@3.23.2_zod@3.23.8": { - "integrity": "sha512-uSt90Gzc/tUfyNqxnjlfBs8W6WSGpNBv0rVsNxP/BVSMHMKGdthPYff4xtCHYloJGM0CFxFsb3NbC0eqPhfImw==", - "dependencies": { - "zod": "zod@3.23.8" - } - }, - "zod@3.23.8": { - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", - "dependencies": {} - } - } - }, - "remote": { - "https://cdn.jsdelivr.net/npm/rrule@2/+esm": "00fe5b10eabe4bde57264d53c0532acedd67d5f92fd0b336755e5d2510679869", - "https://cdn.jsdelivr.net/npm/tslib@2.6.2/+esm": "ec46345e43c69ce85450e2b1e0acc64bd971ff5a8e05f9e524e824ff61c2f73f", - "https://deno.land/std@0.106.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", - "https://deno.land/std@0.106.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", - "https://deno.land/std@0.106.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", - "https://deno.land/std@0.178.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", - "https://deno.land/std@0.178.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", - "https://deno.land/std@0.178.0/async/deferred.ts": "42790112f36a75a57db4a96d33974a936deb7b04d25c6084a9fa8a49f135def8", - "https://deno.land/std@0.178.0/bytes/bytes_list.ts": "b4cbdfd2c263a13e8a904b12d082f6177ea97d9297274a4be134e989450dfa6a", - "https://deno.land/std@0.178.0/bytes/concat.ts": "d26d6f3d7922e6d663dacfcd357563b7bf4a380ce5b9c2bbe0c8586662f25ce2", - "https://deno.land/std@0.178.0/bytes/copy.ts": "939d89e302a9761dcf1d9c937c7711174ed74c59eef40a1e4569a05c9de88219", - "https://deno.land/std@0.178.0/bytes/ends_with.ts": "4228811ebc71615d27f065c54b5e815ec1972538772b0f413c0efe05245b472e", - "https://deno.land/std@0.178.0/bytes/equals.ts": "b87494ce5442dc786db46f91378100028c402f83a14a2f7bbff6bda7810aefe3", - "https://deno.land/std@0.178.0/bytes/includes_needle.ts": "76a8163126fb2f8bf86fd7f22192c3bb04bf6a20b987a095127c2ca08adf3ba6", - "https://deno.land/std@0.178.0/bytes/index_of_needle.ts": "65c939607df609374c4415598fa4dad04a2f14c4d98cd15775216f0aaf597f24", - "https://deno.land/std@0.178.0/bytes/last_index_of_needle.ts": "7181072883cb4908c6ce8f7a5bb1d96787eef2c2ab3aa94fe4268ab326a53cbf", - "https://deno.land/std@0.178.0/bytes/mod.ts": "e869bba1e7a2e3a9cc6c2d55471888429a544e70a840c087672e656e7ba21815", - "https://deno.land/std@0.178.0/bytes/repeat.ts": "6f5e490d8d72bcbf8d84a6bb04690b9b3eb5822c5a11687bca73a2318a842294", - "https://deno.land/std@0.178.0/bytes/starts_with.ts": "3e607a70c9c09f5140b7a7f17a695221abcc7244d20af3eb47ccbb63f5885135", - "https://deno.land/std@0.178.0/crypto/keystack.ts": "877ab0f19eb7d37ad6495190d3c3e39f58e9c52e0b6a966f82fd6df67ca55f90", - "https://deno.land/std@0.178.0/crypto/timing_safe_equal.ts": "29a3e05afa48277ab4d9588c0b61f4afe542529302af180c866a4f2a09524169", - "https://deno.land/std@0.178.0/encoding/base64.ts": "7de04c2f8aeeb41453b09b186480be90f2ff357613b988e99fabb91d2eeceba1", - "https://deno.land/std@0.178.0/encoding/base64url.ts": "3f1178f6446834457b16bfde8b559c1cd3481727fe384d3385e4a9995dc2d851", - "https://deno.land/std@0.178.0/http/_negotiation/common.ts": "14d1a52427ab258a4b7161cd80e1d8a207b7cc64b46e911780f57ead5f4323c6", - "https://deno.land/std@0.178.0/http/_negotiation/encoding.ts": "ff747d107277c88cb7a6a62a08eeb8d56dad91564cbcccb30694d5dc126dcc53", - "https://deno.land/std@0.178.0/http/_negotiation/language.ts": "7bcddd8db3330bdb7ce4fc00a213c5547c1968139864201efd67ef2d0d51887d", - "https://deno.land/std@0.178.0/http/_negotiation/media_type.ts": "58847517cd549384ad677c0fe89e0a4815be36fe7a303ea63cee5f6a1d7e1692", - "https://deno.land/std@0.178.0/http/cookie_map.ts": "8cf428c03ef17c197196a5c382495cd4adecbfb597f6f8f3cd7e1775c3e79ffd", - "https://deno.land/std@0.178.0/http/http_errors.ts": "57169d9bdf4cda1982a3742693c146ab1bf2cbc88df003b40ac905a30013d4cb", - "https://deno.land/std@0.178.0/http/http_status.ts": "8a7bcfe3ac025199ad804075385e57f63d055b2aed539d943ccc277616d6f932", - "https://deno.land/std@0.178.0/http/negotiation.ts": "32761c921afa7847cf767fe81c81785721abccc0db0fc51c7ec2a45868b4ee4a", - "https://deno.land/std@0.178.0/io/buf_reader.ts": "90a7adcb3638d8e1361695cdf844d58bcd97c41711dc6f9f8acc0626ebe097f5", - "https://deno.land/std@0.178.0/io/buf_writer.ts": "2fcaadd9f157970fede6e79c8ea9a58556d8cf3c8a686c3fcaaf3875460092cc", - "https://deno.land/std@0.178.0/io/buffer.ts": "e2b7564f684dad625cab08f5106f33572d325705d19a36822b3272fbdfb8f726", - "https://deno.land/std@0.178.0/io/copy_n.ts": "c498021ce291576a68b5bae9f9d3a27f97644f4af6c1047fb1cff054af19e436", - "https://deno.land/std@0.178.0/io/limited_reader.ts": "d709b5b3113d4cbf934415ba242596e0ecb130e8868fb47197217e09dbb59558", - "https://deno.land/std@0.178.0/io/mod.ts": "2665bcccc1fd6e8627cca167c3e92aaecbd9897556b6f69e6d258070ef63fd9b", - "https://deno.land/std@0.178.0/io/multi_reader.ts": "5f7ef6e987486322b38c72e206b8fbc8916d55a87fbcdc97a8b2596386c28d44", - "https://deno.land/std@0.178.0/io/read_delim.ts": "7e102c66f00a118fa1e1ccd4abb080496f43766686907fd8b9522fdf85443586", - "https://deno.land/std@0.178.0/io/read_int.ts": "7cb8bcdfaf1107586c3bacc583d11c64c060196cb070bb13ae8c2061404f911f", - "https://deno.land/std@0.178.0/io/read_lines.ts": "baee9e35034f2fdfccf63bc24b7e3cb45aa1c1c5de26d178f7bcbc572e87772f", - "https://deno.land/std@0.178.0/io/read_long.ts": "f0aaa420e3da1261c5d33c5e729f09922f3d9fa49f046258d4ff7a00d800c71e", - "https://deno.land/std@0.178.0/io/read_range.ts": "28152daf32e43dd9f7d41d8466852b0d18ad766cd5c4334c91fef6e1b3a74eb5", - "https://deno.land/std@0.178.0/io/read_short.ts": "805cb329574b850b84bf14a92c052c59b5977a492cd780c41df8ad40826c1a20", - "https://deno.land/std@0.178.0/io/read_string_delim.ts": "46eb0c9db3547caf8c759631effa200bbe48924f9b34f41edc627bde36cee52d", - "https://deno.land/std@0.178.0/io/slice_long_to_bytes.ts": "b096472afa3a0dd90fa84584dde7706ed29fc16d48009a581c49368f09fe70f4", - "https://deno.land/std@0.178.0/io/string_reader.ts": "ad9cbecb8509732afcf3d73bb72fa551ec0ccc34f9b8127826247f0190753a65", - "https://deno.land/std@0.178.0/io/string_writer.ts": "8a03c5858c24965a54c6538bed15f32a7c72f5704a12bda56f83a40e28e5433e", - "https://deno.land/std@0.178.0/media_types/_db.ts": "7606d83e31f23ce1a7968cbaee852810c2cf477903a095696cdc62eaab7ce570", - "https://deno.land/std@0.178.0/media_types/_util.ts": "916efbd30b6148a716f110e67a4db29d6949bf4048997b754415dd7e42c52378", - "https://deno.land/std@0.178.0/media_types/content_type.ts": "c682589a0aeb016bfed355cc1ed6fbb3ead2ea48fc0000ac5de6a5730613ad1c", - "https://deno.land/std@0.178.0/media_types/extension.ts": "7a4ef2813d7182f724a941f38161525996e4a67abc3cf6a0f9bc2168d73a0f0e", - "https://deno.land/std@0.178.0/media_types/extensions_by_type.ts": "4358023feac696e6e9d49c0f1e76a859f03ca254df57812f31f8536890c3a443", - "https://deno.land/std@0.178.0/media_types/format_media_type.ts": "1e35e16562e5c417401ffc388a9f8f421f97f0ee06259cbe990c51bae4e6c7a8", - "https://deno.land/std@0.178.0/media_types/get_charset.ts": "8be15a1fd31a545736b91ace56d0e4c66ea0d7b3fdc5c90760e8202e7b4b1fad", - "https://deno.land/std@0.178.0/media_types/mod.ts": "d3f0b99f85053bc0b98ecc24eaa3546dfa09b856dc0bbaf60d8956d2cdd710c8", - "https://deno.land/std@0.178.0/media_types/parse_media_type.ts": "bed260d868ea271445ae41d748e7afed9b5a7f407d2777ead08cecf73e9278de", - "https://deno.land/std@0.178.0/media_types/type_by_extension.ts": "6076a7fc63181d70f92ec582fdea2c927eb2cfc7f9c9bee9d6add2aca86f2355", - "https://deno.land/std@0.178.0/media_types/vendor/mime-db.v1.52.0.ts": "6925bbcae81ca37241e3f55908d0505724358cda3384eaea707773b2c7e99586", - "https://deno.land/std@0.178.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", - "https://deno.land/std@0.178.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", - "https://deno.land/std@0.178.0/path/_util.ts": "d7abb1e0dea065f427b89156e28cdeb32b045870acdf865833ba808a73b576d0", - "https://deno.land/std@0.178.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", - "https://deno.land/std@0.178.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", - "https://deno.land/std@0.178.0/path/mod.ts": "4b83694ac500d7d31b0cdafc927080a53dc0c3027eb2895790fb155082b0d232", - "https://deno.land/std@0.178.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", - "https://deno.land/std@0.178.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", - "https://deno.land/std@0.178.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", - "https://deno.land/std@0.178.0/streams/_common.ts": "f45cba84f0d813de3326466095539602364a9ba521f804cc758f7a475cda692d", - "https://deno.land/std@0.178.0/streams/buffer.ts": "7e7676c29e0e72f6821c3b5fede2540886a216bb91c849bb5db20bb82a01d8a1", - "https://deno.land/std@0.178.0/streams/byte_slice_stream.ts": "cf5785b0d9223ebb51fcf6679d881dfaf614c3b288fb4577b511b6f7801a01aa", - "https://deno.land/std@0.178.0/streams/copy.ts": "de0de21701d8cceba84ca01d9731c77f4b3597bb9de6a1b08f32250353feeae8", - "https://deno.land/std@0.178.0/streams/delimiter_stream.ts": "de8f822a1c394cdb805a97e364400955cd1757cc282d932b4040a0f78fba3b5f", - "https://deno.land/std@0.178.0/streams/early_zip_readable_streams.ts": "64828085be5812ec5f4896c10b59f14e5a322b2c0439be9331dde332ae9c91de", - "https://deno.land/std@0.178.0/streams/iterate_reader.ts": "06491ed8f1bb1c619abbfa04c10b173ff95a93e51fe5037b7c1ad0b5cc01fc7d", - "https://deno.land/std@0.178.0/streams/limited_bytes_transform_stream.ts": "3bc04143b8b91a923f5ee81a3c618b6606ac7da66ccbcde62a67aaa0375cbc71", - "https://deno.land/std@0.178.0/streams/limited_transform_stream.ts": "b336f5d649a06e35e2692849e3682a673bb32531738443eb2ce9f57538722f75", - "https://deno.land/std@0.178.0/streams/merge_readable_streams.ts": "5d6302888f4bb0616dafb5768771be0aec9bedc05fbae6b3d726d05ffbec5b15", - "https://deno.land/std@0.178.0/streams/mod.ts": "c07ec010e700b9ea887dc36ca08729828bc7912f711e4054e24d33fd46282252", - "https://deno.land/std@0.178.0/streams/read_all.ts": "bfa220b0e1d06fa4d0cb5178baba8f8b466019a7411511982bfa2320ca292815", - "https://deno.land/std@0.178.0/streams/readable_stream_from_iterable.ts": "cae337ddafd2abc5e3df699ef2af888ac04091f12732ae658448fba2c7b187e8", - "https://deno.land/std@0.178.0/streams/readable_stream_from_reader.ts": "9aceaeefa9e04b08f56b2d07272baedc3b6432840b66198d72fa2ada3e6608ea", - "https://deno.land/std@0.178.0/streams/reader_from_iterable.ts": "05f7759b9336fd4c233d9daadf92aec9a7d2c07a05986da466a935cec2dd79d9", - "https://deno.land/std@0.178.0/streams/reader_from_stream_reader.ts": "3fda9390ec8520c8a9ea2aba2579208b18880a7663d7a9feec8f193b7af14e41", - "https://deno.land/std@0.178.0/streams/text_delimiter_stream.ts": "ee216316360366c3744197f5665a066a25e6baa8cfe836fbe9a0033e079e089e", - "https://deno.land/std@0.178.0/streams/text_line_stream.ts": "a9dd2636c6b90e626e19df26c97034c5f638bdd957cbd5c531d6278fe1d08e90", - "https://deno.land/std@0.178.0/streams/to_transform_stream.ts": "31c8cce967a2f602be5f164973a5c6cedd4c76e1d2fbc22ae0081b084f73fb0e", - "https://deno.land/std@0.178.0/streams/writable_stream_from_writer.ts": "0320b759aa343f9f4f58b014fe301d9a7abcbfb8413d260502a885995e6a0776", - "https://deno.land/std@0.178.0/streams/write_all.ts": "3b2e1ce44913f966348ce353d02fa5369e94115181037cd8b602510853ec3033", - "https://deno.land/std@0.178.0/streams/writer_from_stream_writer.ts": "31126a6bf2e678c5a718011d4831dbe75dbdbd885965d3dbd5dd105e6f20f976", - "https://deno.land/std@0.178.0/streams/zip_readable_streams.ts": "9eb82070d83055fe6f077192fb204dc7612695a4b330148e9aa376df1a65e708", - "https://deno.land/std@0.181.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", - "https://deno.land/std@0.181.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", - "https://deno.land/std@0.181.0/bytes/bytes_list.ts": "b4cbdfd2c263a13e8a904b12d082f6177ea97d9297274a4be134e989450dfa6a", - "https://deno.land/std@0.181.0/bytes/concat.ts": "d26d6f3d7922e6d663dacfcd357563b7bf4a380ce5b9c2bbe0c8586662f25ce2", - "https://deno.land/std@0.181.0/bytes/copy.ts": "939d89e302a9761dcf1d9c937c7711174ed74c59eef40a1e4569a05c9de88219", - "https://deno.land/std@0.181.0/collections/filter_values.ts": "5b9feaf17b9a6e5ffccdd36cf6f38fa4ffa94cff2602d381c2ad0c2a97929652", - "https://deno.land/std@0.181.0/collections/without_all.ts": "a89f5da0b5830defed4f59666e188df411d8fece35a5f6ca69be6ca71a95c185", - "https://deno.land/std@0.181.0/dotenv/mod.ts": "8dcbc8a40b896a0bf094582aaeadbfc76d3528872faf2efc0302beb1d2f6afd0", - "https://deno.land/std@0.181.0/fs/_util.ts": "65381f341af1ff7f40198cee15c20f59951ac26e51ddc651c5293e24f9ce6f32", - "https://deno.land/std@0.181.0/fs/copy.ts": "14214efd94fc3aa6db1e4af2b4b9578e50f7362b7f3725d5a14ad259a5df26c8", - "https://deno.land/std@0.181.0/fs/empty_dir.ts": "c3d2da4c7352fab1cf144a1ecfef58090769e8af633678e0f3fabaef98594688", - "https://deno.land/std@0.181.0/fs/ensure_dir.ts": "dc64c4c75c64721d4e3fb681f1382f803ff3d2868f08563ff923fdd20d071c40", - "https://deno.land/std@0.181.0/fs/ensure_file.ts": "c38602670bfaf259d86ca824a94e6cb9e5eb73757fefa4ebf43a90dd017d53d9", - "https://deno.land/std@0.181.0/fs/ensure_link.ts": "c0f5b2f0ec094ed52b9128eccb1ee23362a617457aa0f699b145d4883f5b2fb4", - "https://deno.land/std@0.181.0/fs/ensure_symlink.ts": "5006ab2f458159c56d689b53b1e48d57e05eeb1eaf64e677f7f76a30bc4fdba1", - "https://deno.land/std@0.181.0/fs/eol.ts": "f1f2eb348a750c34500741987b21d65607f352cf7205f48f4319d417fff42842", - "https://deno.land/std@0.181.0/fs/exists.ts": "b8c8a457b71e9d7f29b9d2f87aad8dba2739cbe637e8926d6ba6e92567875f8e", - "https://deno.land/std@0.181.0/fs/expand_glob.ts": "e4f56259a0a70fe23f05215b00de3ac5e6ba46646ab2a06ebbe9b010f81c972a", - "https://deno.land/std@0.181.0/fs/mod.ts": "bc3d0acd488cc7b42627044caf47d72019846d459279544e1934418955ba4898", - "https://deno.land/std@0.181.0/fs/move.ts": "4cb47f880e3f0582c55e71c9f8b1e5e8cfaacb5e84f7390781dd563b7298ec19", - "https://deno.land/std@0.181.0/fs/walk.ts": "ea95ffa6500c1eda6b365be488c056edc7c883a1db41ef46ec3bf057b1c0fe32", - "https://deno.land/std@0.181.0/io/buf_reader.ts": "abeb92b18426f11d72b112518293a96aef2e6e55f80b84235e8971ac910affb5", - "https://deno.land/std@0.181.0/io/buf_writer.ts": "48c33c8f00b61dcbc7958706741cec8e59810bd307bc6a326cbd474fe8346dfd", - "https://deno.land/std@0.181.0/io/buffer.ts": "17f4410eaaa60a8a85733e8891349a619eadfbbe42e2f319283ce2b8f29723ab", - "https://deno.land/std@0.181.0/io/copy_n.ts": "0cc7ce07c75130f6fc18621ec1911c36e147eb9570664fee0ea12b1988167590", - "https://deno.land/std@0.181.0/io/limited_reader.ts": "6c9a216f8eef39c1ee2a6b37a29372c8fc63455b2eeb91f06d9646f8f759fc8b", - "https://deno.land/std@0.181.0/io/mod.ts": "2665bcccc1fd6e8627cca167c3e92aaecbd9897556b6f69e6d258070ef63fd9b", - "https://deno.land/std@0.181.0/io/multi_reader.ts": "9c2a0a31686c44b277e16da1d97b4686a986edcee48409b84be25eedbc39b271", - "https://deno.land/std@0.181.0/io/read_delim.ts": "c02b93cc546ae8caad8682ae270863e7ace6daec24c1eddd6faabc95a9d876a3", - "https://deno.land/std@0.181.0/io/read_int.ts": "7cb8bcdfaf1107586c3bacc583d11c64c060196cb070bb13ae8c2061404f911f", - "https://deno.land/std@0.181.0/io/read_lines.ts": "c526c12a20a9386dc910d500f9cdea43cba974e853397790bd146817a7eef8cc", - "https://deno.land/std@0.181.0/io/read_long.ts": "f0aaa420e3da1261c5d33c5e729f09922f3d9fa49f046258d4ff7a00d800c71e", - "https://deno.land/std@0.181.0/io/read_range.ts": "28152daf32e43dd9f7d41d8466852b0d18ad766cd5c4334c91fef6e1b3a74eb5", - "https://deno.land/std@0.181.0/io/read_short.ts": "805cb329574b850b84bf14a92c052c59b5977a492cd780c41df8ad40826c1a20", - "https://deno.land/std@0.181.0/io/read_string_delim.ts": "5dc9f53bdf78e7d4ee1e56b9b60352238ab236a71c3e3b2a713c3d78472a53ce", - "https://deno.land/std@0.181.0/io/slice_long_to_bytes.ts": "48d9bace92684e880e46aa4a2520fc3867f9d7ce212055f76ecc11b22f9644b7", - "https://deno.land/std@0.181.0/io/string_reader.ts": "da0f68251b3d5b5112485dfd4d1b1936135c9b4d921182a7edaf47f74c25cc8f", - "https://deno.land/std@0.181.0/io/string_writer.ts": "8a03c5858c24965a54c6538bed15f32a7c72f5704a12bda56f83a40e28e5433e", - "https://deno.land/std@0.181.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", - "https://deno.land/std@0.181.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", - "https://deno.land/std@0.181.0/path/_util.ts": "d7abb1e0dea065f427b89156e28cdeb32b045870acdf865833ba808a73b576d0", - "https://deno.land/std@0.181.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", - "https://deno.land/std@0.181.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", - "https://deno.land/std@0.181.0/path/mod.ts": "bf718f19a4fdd545aee1b06409ca0805bd1b68ecf876605ce632e932fe54510c", - "https://deno.land/std@0.181.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", - "https://deno.land/std@0.181.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", - "https://deno.land/std@0.181.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", - "https://deno.land/std@0.184.0/async/abortable.ts": "fd682fa46f3b7b16b4606a5ab52a7ce309434b76f820d3221bdfb862719a15d7", - "https://deno.land/std@0.184.0/async/deadline.ts": "c5facb0b404eede83e38bd2717ea8ab34faa2ffb20ef87fd261fcba32ba307aa", - "https://deno.land/std@0.184.0/async/debounce.ts": "adab11d04ca38d699444ac8a9d9856b4155e8dda2afd07ce78276c01ea5a4332", - "https://deno.land/std@0.184.0/async/deferred.ts": "42790112f36a75a57db4a96d33974a936deb7b04d25c6084a9fa8a49f135def8", - "https://deno.land/std@0.184.0/async/delay.ts": "73aa04cec034c84fc748c7be49bb15cac3dd43a57174bfdb7a4aec22c248f0dd", - "https://deno.land/std@0.184.0/async/mod.ts": "f04344fa21738e5ad6bea37a6bfffd57c617c2d372bb9f9dcfd118a1b622e576", - "https://deno.land/std@0.184.0/async/mux_async_iterator.ts": "70c7f2ee4e9466161350473ad61cac0b9f115cff4c552eaa7ef9d50c4cbb4cc9", - "https://deno.land/std@0.184.0/async/pool.ts": "fd082bd4aaf26445909889435a5c74334c017847842ec035739b4ae637ae8260", - "https://deno.land/std@0.184.0/async/retry.ts": "dd19d93033d8eaddbfcb7654c0366e9d3b0a21448bdb06eba4a7d8a8cf936a92", - "https://deno.land/std@0.184.0/async/tee.ts": "47e42d35f622650b02234d43803d0383a89eb4387e1b83b5a40106d18ae36757", - "https://deno.land/std@0.184.0/http/server.ts": "cbb17b594651215ba95c01a395700684e569c165a567e4e04bba327f41197433", - "https://deno.land/x/cors@v1.2.2/abcCors.ts": "cdf83a7eaa69a1bf3ab910d18b9422217902fac47601adcaf0afac5a61845d48", - "https://deno.land/x/cors@v1.2.2/attainCors.ts": "7d6aba0f942495cc31119604e0895c9bb8edd8f8baa7fe78e6c655bd0b4cbf59", - "https://deno.land/x/cors@v1.2.2/cors.ts": "0e2d9167e3685f9bcf48f565e312b6e1883fa458f7337e5ce7bc2e3b29767980", - "https://deno.land/x/cors@v1.2.2/mithCors.ts": "3a359d6e716e0410ede278ab54d875b293a2d66d838aaa7cfbf9ddc1e9e990d3", - "https://deno.land/x/cors@v1.2.2/mod.ts": "2b351913f56d77ad80cb3b8633d4539c9eeddb426dae79437ada0e6a9cb4f1a6", - "https://deno.land/x/cors@v1.2.2/oakCors.ts": "1348dc7673c61b85d2e80559a7b44f8e0246eaa6bcc6ec744fafe5d9b13b5c71", - "https://deno.land/x/cors@v1.2.2/opineCors.ts": "fb5790115c26b7061d84b8d6c17d258a1e241bcab75b0bc3ca1fdb2e57bc5072", - "https://deno.land/x/cors@v1.2.2/types.ts": "97546633ccc7f0df7a29bacba5d91dc6f61decdd1b65258300244dba905d34b8", - "https://deno.land/x/oak@v12.1.0/application.ts": "641c071bf14b476d603be18a984573e8a7c0da9c1bfc141ae44dad8012f2c5d6", - "https://deno.land/x/oak@v12.1.0/body.ts": "c7392f1dae04a360838f43b9cdd2f83d29c1eff4e6071d5f0cf1f3999b1602bc", - "https://deno.land/x/oak@v12.1.0/buf_reader.ts": "7cf96aa0ac670b75098113cf88a291a68332cc45efa8a9698f064ac5b8098a0f", - "https://deno.land/x/oak@v12.1.0/content_disposition.ts": "8b8c3cb2fba7138cd5b7f82fc3b5ea39b33db924a824b28261659db7e164621e", - "https://deno.land/x/oak@v12.1.0/context.ts": "81b97dab6b35ab872d5b2ff5221b4bc71b425f7069ce1a1dcc1d20c50a09865f", - "https://deno.land/x/oak@v12.1.0/deps.ts": "29134b39303d85527915650d6f11909c1e29c4cd208dd2ad9887ce2537167b42", - "https://deno.land/x/oak@v12.1.0/etag.ts": "19918f5e1964e3fe6c9fe524a88ffbf9900ce1dfe4146b187b2a86256bb6b663", - "https://deno.land/x/oak@v12.1.0/headers.ts": "f50fb05614432bda971021633129aa2e8737e0844e0f01c27a937997b4d8dd4f", - "https://deno.land/x/oak@v12.1.0/helpers.ts": "42212afa07a560b2958359cc19577417e89d9574d6579551a0af36ff7f00cc6e", - "https://deno.land/x/oak@v12.1.0/http_request.ts": "9f2cc5f4b9e72c5156dd4710875d9f54f992d69ca1a8ad5e0915a773f0e26e8b", - "https://deno.land/x/oak@v12.1.0/http_server_flash.ts": "6afeb6bfe77a08d99b205ebf09d18c98b4050ce9cbc66e24be1fba6e46bc7a4f", - "https://deno.land/x/oak@v12.1.0/http_server_native.ts": "0141e1339ed9a33bc26ce537ddab5adbb3542b35916d92de286aed4937e4a6d6", - "https://deno.land/x/oak@v12.1.0/http_server_native_request.ts": "be315d476550e149c58d7ccd2812be30f373ceedc9c323c300eef03b7c071aa9", - "https://deno.land/x/oak@v12.1.0/isMediaType.ts": "62d638abcf837ece3a8f07a4b7ca59794135cb0d4b73194c7d5837efd4161005", - "https://deno.land/x/oak@v12.1.0/mediaTyper.ts": "042b853fc8e9c3f6c628dd389e03ef481552bf07242efc3f8a1af042102a6105", - "https://deno.land/x/oak@v12.1.0/middleware.ts": "de14f045a2ddfe845d89b5d3140ff52cbcc6f3b3965391106ce04480f9786737", - "https://deno.land/x/oak@v12.1.0/middleware/proxy.ts": "b927232f97ec18af4185d7912e45b1191e3ffe24a9c875262ad524211b1274c9", - "https://deno.land/x/oak@v12.1.0/mod.ts": "210619d431e41c763486467b0498ef50de79119d750abc57331f76e36f642fc1", - "https://deno.land/x/oak@v12.1.0/multipart.ts": "98fe9f226de8c26a16d067027b69fb1e34ad8c4055767dd157907d06cea36f9a", - "https://deno.land/x/oak@v12.1.0/range.ts": "68a6df7ab3b868843e33f52deb94c3d4cab25cb9ef369691990c2ac15b04fafb", - "https://deno.land/x/oak@v12.1.0/request.ts": "5852ad36389b48e0428a6f3c90854d01f10d1b15949b56001e1e75c2a00ef0f9", - "https://deno.land/x/oak@v12.1.0/response.ts": "867d81f7eb0c65c7b8e0e0e9e145ededd5b6daa9ad922e6adc6a36a525f439a6", - "https://deno.land/x/oak@v12.1.0/router.ts": "5b266091e55f634c9130e6de5dd331ddfc4c190ee7916a25e0a0f75502edbc32", - "https://deno.land/x/oak@v12.1.0/send.ts": "c592a6782c82442d011923297a724c2cba8de14c4a9384c6eb9cebb766817e92", - "https://deno.land/x/oak@v12.1.0/server_sent_event.ts": "948b0fe4cb3fe38c7db15e476eb3b7671ef20e566d130e9f701d7c0146aa47dd", - "https://deno.land/x/oak@v12.1.0/structured_clone.ts": "9c2d21c62f616400305a60cbd29eb06764ee97edc423223424b6cf55df0e8be2", - "https://deno.land/x/oak@v12.1.0/testing.ts": "a0be5c84981afde666de29630f34b09d944ca1a2fe6a5185644b60ad95e16d18", - "https://deno.land/x/oak@v12.1.0/util.ts": "3af8c4ed04c6cc2bedbe66e562a77fc59c72df31c55a902a63885861ca1639d6", - "https://deno.land/x/open@v0.0.5/index.ts": "387293f5063d620137d9ba87fa4a9aece5ac435ca9f5bf5e3f0999634f68e294", - "https://deno.land/x/path_to_regexp@v6.2.1/index.ts": "894060567837bae8fc9c5cbd4d0a05e9024672083d5883b525c031eea940e556", - "https://deno.land/x/rss@0.5.6/deps.ts": "7e1d455d3c3176f5c662e281c40456f2c81380fd3998bb1f84c5e285db4a5a88", - "https://deno.land/x/rss@0.5.6/mod.ts": "e3543b4feeadb666a6973fa1d5139d928fb3d6fde8529021b3ac5e47e55681bb", - "https://deno.land/x/rss@0.5.6/src/deserializer.ts": "897ed3a1ee73775e1aa85e40cfaa7d38966aa52bc801edcf9ce73b621b02ab55", - "https://deno.land/x/rss@0.5.6/src/mappers/mapper.ts": "34356df91fe3ccbadcb3f0798f8a9f5bb7791d6c74030c03b4ea0e22e29b1bfb", - "https://deno.land/x/rss@0.5.6/src/mappers/mapper_legacy.ts": "2ee4a0f2e6546c45f916538fe0ca8c0c402318eb05676b12cf991f56310888f7", - "https://deno.land/x/rss@0.5.6/src/mappers/media_mapper.ts": "d21314dc8d687cc7976c81b94d1e92bcb840f9a3cc59cfeef82a4835d60f6a2f", - "https://deno.land/x/rss@0.5.6/src/mappers/mod.ts": "c7ea71d61bc4b6f6886f53701c1dffd974fe11d6d9d1f811064baeebf48899dc", - "https://deno.land/x/rss@0.5.6/src/resolvers/atom_resolver.ts": "41107cc3c57f7b51b5a8c425bb7bf91773e9d8ace01fa6c0f1df8352532d46dd", - "https://deno.land/x/rss@0.5.6/src/resolvers/dublin_core_resolver.ts": "cdc12e443e3ce7d96b839b7ed6e97d26f0500c4bd1c8051a88871fd9fce3bcd7", - "https://deno.land/x/rss@0.5.6/src/resolvers/media_rss_resolver.ts": "d40784e616b98d83d4a6f0d7b8e31199096a4172f7fe401c871ad504bc556a3b", - "https://deno.land/x/rss@0.5.6/src/resolvers/mod.ts": "063f3341070fa0e69cea48a9387a226e0a1c500193d7e75cb25b0a870717c8fa", - "https://deno.land/x/rss@0.5.6/src/resolvers/rss1_resolver.ts": "76292ebe5ab13aa17408288f62d19bdd78c74b47307e2db7fc0df595305d37a4", - "https://deno.land/x/rss@0.5.6/src/resolvers/rss2_resolver.ts": "a82b0e9e5ba0f73d4bd924de12b0d02575a2fc56142bea534a83e632bb166b97", - "https://deno.land/x/rss@0.5.6/src/resolvers/slash_resolver.ts": "ea904b557a18961077bb974cea64629fd46f0cc5964fd8e66eaaeca5f0682ce4", - "https://deno.land/x/rss@0.5.6/src/types/feed_type.ts": "b2098f23982be344a19f00915437550d1e4002fb653ac9e34455a90119ee4e6c", - "https://deno.land/x/rss@0.5.6/src/types/fields/atom_fields.ts": "85bf1a4b3e917baa3c94f3fba8283b678d5a10e78a16c42cfa19b2ee0bf210b3", - "https://deno.land/x/rss@0.5.6/src/types/fields/dublin_core_fields.ts": "fd6b7e5b660a4f99cabb7a62ae91ab48579de340e57c8dd7ad62c992e64c1644", - "https://deno.land/x/rss@0.5.6/src/types/fields/media_rss_fields.ts": "076b1e23b6ca99002ffb13e5859164acdf45617aea90440acc75fb28f3aa00b4", - "https://deno.land/x/rss@0.5.6/src/types/fields/mod.ts": "cdcbf786f30577d8e95648f10a6a1a069e6f68ba749e3b753caff94f205ddaaa", - "https://deno.land/x/rss@0.5.6/src/types/fields/rss1_fields.ts": "04da9ca7d3fe2b59626c6b3bd18bfd41bf1b1531bc33f2d4f420f10da4e84f7e", - "https://deno.land/x/rss@0.5.6/src/types/fields/rss2_fields.ts": "c5041472e42cdf04e9f6b2e6e2e334b016aff40524c9a4a4f2c3a5875f9508ea", - "https://deno.land/x/rss@0.5.6/src/types/fields/slash_fields.ts": "11a1cf81744fe3b2bddbcf3a3ce6f86406d250198ef8c41e6fa58dd652e2e466", - "https://deno.land/x/rss@0.5.6/src/types/internal/internal_dublin_core.ts": "9f3d63a2386335c8bfe03d3ca3aa6e71f437edb2df0c29f1317308f35cdbcfea", - "https://deno.land/x/rss@0.5.6/src/types/mod.ts": "c8daacf748276644389dd3c8d9c5e6c964707cc59b719b35aa1a424d453d543c", - "https://deno.land/x/rss@0.5.6/src/types/slash.ts": "9a08968ef416f818de3ab9778650fa0b586f9c8c60369bdbd365ba110c5f6f05", - "https://deno.land/x/rss@0.5.6/src/util.ts": "a6611bcea8acf000eb01220092f1bea6c18aaf3ad82578c60563f25f6260c080", - "https://deno.land/x/sax_ts@v1.2.10/src/sax.ts": "3d468d10d13d2690b5910fd12afc3cf52d979c0f144899d5c36da45c0931634e", - "https://deno.land/x/sqlite@v3.7.0/build/sqlite.js": "cc55fef9cd124b2acb624899a5fad413834f4701bcfc21ac275844b822466292", - "https://deno.land/x/sqlite@v3.7.0/build/vfs.js": "08533cc78fb29b9d9bd62f6bb93e5ef333407013fed185776808f11223ba0e70", - "https://deno.land/x/sqlite@v3.7.0/mod.ts": "e09fc79d8065fe222578114b109b1fd60077bff1bb75448532077f784f4d6a83", - "https://deno.land/x/sqlite@v3.7.0/src/constants.ts": "90f3be047ec0a89bcb5d6fc30db121685fc82cb00b1c476124ff47a4b0472aa9", - "https://deno.land/x/sqlite@v3.7.0/src/db.ts": "87e3d222d00dd2e2827816a47976c6359996ab39e69c7f87855ea414585839c6", - "https://deno.land/x/sqlite@v3.7.0/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a", - "https://deno.land/x/sqlite@v3.7.0/src/function.ts": "e4c83b8ec64bf88bafad2407376b0c6a3b54e777593c70336fb40d43a79865f2", - "https://deno.land/x/sqlite@v3.7.0/src/query.ts": "69895232360b4254a8834e3bbea30bc3752cf691dba862f4a393e7a41a2e30eb", - "https://deno.land/x/sqlite@v3.7.0/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487", - "https://raw.githubusercontent.com/mansueli/deno_ics_parser/ecb78556377dee1fb6061f985a9f185ec29304bd/ics_parser.ts": "8da2bc70a11aedcaeba89e08e8935e80ab33f19ce59d6063feba799a586659c6" - }, - "workspace": { - "packageJson": { - "dependencies": [ - "npm:tslib@^2.6.2", - "npm:typescript@^5.2.2", - "npm:wireit@^0.14.4" - ] - } - } -} diff --git a/typescript/packages/collectathon/deps.ts b/typescript/packages/collectathon/deps.ts deleted file mode 100644 index 01dea3f46..000000000 --- a/typescript/packages/collectathon/deps.ts +++ /dev/null @@ -1,17 +0,0 @@ -export { DB } from "https://deno.land/x/sqlite@v3.7.0/mod.ts"; -export { readLines } from "https://deno.land/std@0.181.0/io/mod.ts"; -export { parseFeed } from "https://deno.land/x/rss@0.5.6/mod.ts"; -export { ensureDir, walk } from "https://deno.land/std@0.181.0/fs/mod.ts"; -export { anthropic } from "npm:@ai-sdk/anthropic@0.0.50"; -export { openai } from "npm:@ai-sdk/openai@0.0.60"; -export { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts"; -export * as ai from "npm:ai@3.3.39"; -export { Application, Router } from "https://deno.land/x/oak@v12.1.0/mod.ts"; -export { default as Table } from "npm:easy-table"; -export { default as parseICS } from "https://raw.githubusercontent.com/mansueli/deno_ics_parser/ecb78556377dee1fb6061f985a9f185ec29304bd/ics_parser.ts"; -export { CID } from "npm:multiformats@13.3.0/cid"; -export * as json from 'npm:multiformats@13.3.0/codecs/json' -export { sha256 } from 'npm:multiformats@13.3.0/hashes/sha2' - -import { load } from "https://deno.land/std@0.181.0/dotenv/mod.ts"; -await load({ export: true }); diff --git a/typescript/packages/collectathon/dream.ts b/typescript/packages/collectathon/dream.ts deleted file mode 100644 index 46367d0a2..000000000 --- a/typescript/packages/collectathon/dream.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { db } from "./db.ts"; -import { chat, smart } from "./llm.ts"; -import { CoreMessage } from "npm:ai@3.3.21"; -import { extractJsonShape } from "./schema.ts"; -import { getOrCreateCollection, addItemToCollection } from "./collections.ts"; - -export async function handleDreamCommand(collectionName: string) { - // Fetch the collection items - const items = db.query<[number, string]>( - `SELECT i.id, i.content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collectionName], - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collectionName}`); - return; - } - - // Parse the JSON content of each item - const jsonItems = items.map(([id, content]) => ({ - id, - ...JSON.parse(content), - })); - - // Extract the shape of the JSON items - const itemShape = await extractJsonShape(jsonItems); - - console.log("Items shape:", itemShape); - - // Generate a new item using the LLM - const newItem = await generateNewItem(jsonItems, itemShape); - - // Print the new item and ask for user confirmation - console.log("Generated new item:"); - console.log(JSON.stringify(newItem, null, 2)); - const confirmation = prompt( - "Do you want to add this item to the collection? (y/n): ", - ); - - if (confirmation.toLowerCase() !== "y") { - console.log("Item addition cancelled."); - return; - } - - // Add the new item to the collection - const collectionId = await getOrCreateCollection(collectionName); - const result = db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - "dreamed_item", - `Dreamed item for ${collectionName}`, - JSON.stringify(newItem), - JSON.stringify(newItem), - "Dream", - ], - ); - const itemId = result[0][0] as number; - - await addItemToCollection(itemId, collectionName); - - console.log( - `New item added to collection: ${collectionName} with ID: ${itemId}`, - ); -} - -async function generateNewItem(items: any[], itemShape: string): Promise { - const userMessage = `Given the following array of items and their shape, generate a new item that fits within the set but contains novel ideas or data: - -Items: -${JSON.stringify(items, null, 2)} - -Item Shape: -${itemShape} - -Generate a single new item that fits in the collection. Return only the JSON object for the new item, without any explanation or additional text.`; - - const messages: CoreMessage[] = [ - { role: "user", content: userMessage }, - ]; - - console.log("Dreaming..."); - const response = await smart(messages, false); - return response; -} - -export function addDreamCommand(args: string[]) { - if (args.length !== 1) { - console.log("Usage: dream "); - return; - } - - const collectionName = args[0]; - handleDreamCommand(collectionName); -} diff --git a/typescript/packages/collectathon/github.ts b/typescript/packages/collectathon/github.ts deleted file mode 100644 index 530f402c8..000000000 --- a/typescript/packages/collectathon/github.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { getOrCreateCollection } from "./collections.ts"; -import { db } from "./db.ts"; -import { ensureDir, walk } from "./deps.ts"; - -export async function runCommand(cmd: string[], cwd?: string): Promise { - const command = new Deno.Command(cmd[0], { - args: cmd.slice(1), - cwd, - stdout: "piped", - stderr: "piped", - }); - - const { code, stdout, stderr } = await command.output(); - - if (code === 0) { - return new TextDecoder().decode(stdout).trim(); - } else { - const errorString = new TextDecoder().decode(stderr); - throw new Error(errorString); - } -} - -export async function clipGitHub(url: string, collections: string[]) { - try { - db.query("BEGIN TRANSACTION"); - const repoPath = new URL(url).pathname.split("/").slice(-2).join("/"); - const localPath = `./temp/${repoPath}`; - - await ensureDir("./temp"); - - console.log(`Cloning repository: ${url}`); - await runCommand(["git", "clone", url, localPath]); - - const collectionIds = await Promise.all(collections.map(collectionName => getOrCreateCollection(collectionName))); - let itemCount = 0; - - for await (const entry of walk(localPath, { includeDirs: false })) { - // Skip the .git folder and its contents - if (entry.path.includes("/.git/")) { - continue; - } - - const relativePath = entry.path.replace(localPath + "/", ""); - const content = await Deno.readTextFile(entry.path); - - const contentJson = { - path: relativePath, - content: content, - }; - - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - `${url}/blob/main/${relativePath}`, - relativePath, - JSON.stringify(contentJson), - content, - "GitHub", - ], - ); - const itemId = result[0][0] as number; - - for (const collectionId of collectionIds) { - await db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - } - - itemCount++; - } - - console.log( - `Clipped ${itemCount} files from GitHub repository to collection: ${collections.join(', ')}`, - ); - - // Clean up: remove the cloned repository - await Deno.remove(localPath, { recursive: true }); - - db.query("COMMIT"); - } catch (error) { - console.error(`Error clipping GitHub repository: ${error.message}`); - db.query("ROLLBACK"); - } -} - -export async function syncGitHubIssues(repoUrl: string, collections: string[]) { - try { - db.query("BEGIN TRANSACTION"); - // Update URL parsing logic - const urlParts = new URL(repoUrl).pathname.split("/").filter(Boolean); - const owner = urlParts[0]; - const repo = urlParts[1]; - if (!owner || !repo) { - throw new Error("Invalid GitHub repository URL"); - } - let apiUrl = `https://api.github.com/repos/${owner}/${repo}/issues?state=all&per_page=100`; - let itemCountProcessed = 0; - - while (apiUrl) { - const response = await fetch(apiUrl, { - headers: { - "Accept": "application/vnd.github.v3+json", - "User-Agent": "Collectathon", - }, - }); - - if (!response.ok) { - throw new Error(`GitHub API request failed: ${response.statusText}`); - } - - const issues = await response.json(); - const collectionIds = await Promise.all(collections.map(collectionName => getOrCreateCollection(collectionName))); - - for (const issue of issues) { - const contentJson = { - number: issue.number, - title: issue.title, - state: issue.state, - created_at: issue.created_at, - updated_at: issue.updated_at, - body: issue.body || "", - html_url: issue.html_url, - user: issue.user.login, - labels: issue.labels.map((label: any) => label.name).join(', '), - }; - - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - issue.html_url, - `Issue #${issue.number}: ${issue.title}`, - JSON.stringify(contentJson), - issue.body || "(empty)", - "GitHub Issue", - ], - ); - const itemId = result[0][0] as number; - - for (const collectionId of collectionIds) { - await db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - } - - itemCountProcessed++; - } - - // Check for next page - const linkHeader = response.headers.get('Link'); - apiUrl = getNextPageUrl(linkHeader); - } - - console.log( - `Synced ${itemCountProcessed} issues from GitHub repository to collections: ${collections.join(', ')}`, - ); - - db.query("COMMIT"); - } catch (error) { - console.error(`Error syncing GitHub issues: ${error.message}`); - db.query("ROLLBACK"); - } -} - -function getNextPageUrl(linkHeader: string | null): string | null { - if (!linkHeader) return null; - const links = linkHeader.split(','); - const nextLink = links.find(link => link.includes('rel="next"')); - if (!nextLink) return null; - const match = nextLink.match(/<(.+)>/); - return match ? match[1] : null; -} diff --git a/typescript/packages/collectathon/gmail.ts b/typescript/packages/collectathon/gmail.ts deleted file mode 100644 index f9cbd0dd8..000000000 --- a/typescript/packages/collectathon/gmail.ts +++ /dev/null @@ -1,119 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import process from 'node:process'; -import { authenticate } from 'npm:@google-cloud/local-auth@3.0.1'; -import { OAuth2Client } from "npm:google-auth-library@9.14.2"; -import { google } from "npm:googleapis@144.0.0"; -import { clipEmail, cid } from "./synopsys.ts"; - -type GmailEntity = { - messageId: string; - subject: string; - from: string; - date: string; - snippet: string; -} - -// If modifying these scopes, delete token.json. -const SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']; - -// The file token.json stores the user's access and refresh tokens, and is -// created automatically when the authorization flow completes the first time -const TOKEN_PATH = path.join(process.cwd(), 'token.json'); -const CREDENTIALS_PATH = path.join(process.cwd(), 'credentials.json'); - -async function loadSavedCredentialsIfExist(): Promise { - try { - const content = await fs.readFile(TOKEN_PATH); - const credentials = JSON.parse(content.toString()); - return google.auth.fromJSON(credentials); - } catch (err) { - return null; - } -} - -async function saveCredentials(client: OAuth2Client) { - const content = await fs.readFile(CREDENTIALS_PATH); - const keys = JSON.parse(content.toString()); - const key = keys.installed || keys.web; - const payload = JSON.stringify({ - type: 'authorized_user', - client_id: key.client_id, - client_secret: key.client_secret, - refresh_token: client.credentials.refresh_token, - }); - await fs.writeFile(TOKEN_PATH, payload); -} - -async function authorize(): Promise { - let client: any = await loadSavedCredentialsIfExist(); - if (client) { - return client; - } - client = await authenticate({ - scopes: SCOPES, - keyfilePath: CREDENTIALS_PATH, - }); - if (client.credentials) { - await saveCredentials(client); - } - return client; -} - -async function listMessages(auth: OAuth2Client, maxResults: number = 10) { - const gmail = google.gmail({ version: 'v1', auth }); - const res = await gmail.users.messages.list({ - userId: 'me', - maxResults: maxResults - }); - - return res.data.messages || []; -} - -async function getMessage(auth: OAuth2Client, messageId: string) { - const gmail = google.gmail({ version: 'v1', auth }); - const res = await gmail.users.messages.get({ - userId: 'me', - id: messageId - }); - - return res.data; -} - -async function getMessageDetails(auth: OAuth2Client, messageId: string): Promise { - const gmail = google.gmail({ version: 'v1', auth }); - const res = await gmail.users.messages.get({ - userId: 'me', - id: messageId, - format: 'metadata', - metadataHeaders: ['Subject', 'From', 'Date'], - }); - - const headers = res.data.payload?.headers; - const subject = headers?.find(h => h.name === 'Subject')?.value || 'No subject'; - const from = headers?.find(h => h.name === 'From')?.value || 'Unknown sender'; - const date = headers?.find(h => h.name === 'Date')?.value || 'Unknown date'; - - return { - messageId, - subject, - from, - date, - snippet: res.data.snippet || 'No snippet available', - }; -} - -async function importGmail(message: GmailEntity) { - const entityCid = await cid({ messageId: message.messageId, source: "gmail" }); - clipEmail(message.from, ["gmail"], message, entityCid); -} - -if (import.meta.main) { - const maxResults = parseInt(process.argv[2]) || 10; - const auth = await authorize(); - const messages = await listMessages(auth, maxResults); - for (const message of messages) { - const messageDetails = await getMessageDetails(auth, message.id); - importGmail(messageDetails); - } -} \ No newline at end of file diff --git a/typescript/packages/collectathon/import.ts b/typescript/packages/collectathon/import.ts deleted file mode 100644 index ac47c7877..000000000 --- a/typescript/packages/collectathon/import.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { db } from "./db.ts"; -import { walk, ensureDir } from "./deps.ts"; -import { getOrCreateCollection } from "./collections.ts"; -import { clipGitHub, syncGitHubIssues } from "./github.ts"; -import { clipCalendar } from "./calendar.ts"; -import { clipRSS } from "./rss.ts"; -import { clipWebpage } from "./webpage.ts"; - -export async function clipUrl( - url: string, - collections: string[], - prompt: string | undefined, - htmlSource: string | undefined, -) { - if (url.endsWith(".ics")) { - await clipCalendar(url, collections); - } else if (url.includes("github.com")) { - if (url.includes("issues")) { - await syncGitHubIssues(url, collections); - } else { - await clipGitHub(url, collections); - } - } else if ( - url.includes(".rss") || - url.includes("/RSS") || - url.includes("/feed") || - url.includes("feedformat=") - ) { - await clipRSS(url, collections); - } else { - return await clipWebpage(url, collections, prompt, htmlSource); - } - - return []; -} - -export async function importFiles( - path: string, - collectionName: string, - fileTypeFilter: string = "*", -) { - try { - const fullPath = await Deno.realPath(path); - const fileInfo = await Deno.stat(fullPath); - - db.query("BEGIN TRANSACTION"); - - const collectionId = await getOrCreateCollection(collectionName); - let itemCount = 0; - let updatedCount = 0; - - const gitignorePattern = await getGitignorePattern(fullPath); - - if (fileInfo.isDirectory) { - for await (const entry of walk(fullPath, { - includeDirs: false, - match: [new RegExp(fileTypeFilter.replace("*", ".*"))], - skip: [/node_modules/, /\.git/, ...gitignorePattern], - })) { - const result = await processFile(entry.path, collectionId, fullPath); - if (result === "updated") { - updatedCount++; - } else { - itemCount++; - } - } - } else { - if ( - fileTypeFilter === "*" || - fullPath.endsWith(fileTypeFilter.replace("*", "")) - ) { - if (!isIgnoredFile(fullPath, gitignorePattern)) { - const result = await processFile(fullPath, collectionId, Deno.cwd()); - if (result === "updated") { - updatedCount++; - } else { - itemCount++; - } - } - } - } - - db.query("COMMIT"); - - console.log( - `Imported ${itemCount} new file(s) and updated ${updatedCount} existing file(s) in collection: ${collectionName}`, - ); - } catch (error) { - console.error(`Error importing files: ${error.message}`); - db.query("ROLLBACK"); - } -} - -async function getGitignorePattern(path: string): Promise { - try { - const gitignorePath = `${path}/.gitignore`; - const content = await Deno.readTextFile(gitignorePath); - return content - .split("\n") - .filter((line) => line.trim() && !line.startsWith("#")) - .map((pattern) => new RegExp(pattern.replace(/\*/g, ".*"))); - } catch { - return []; - } -} - -function isIgnoredFile(filePath: string, gitignorePattern: RegExp[]): boolean { - return gitignorePattern.some((pattern) => pattern.test(filePath)); -} - -async function processFile( - filePath: string, - collectionId: number, - basePath: string, -): Promise<"new" | "updated"> { - const relativePath = filePath.replace(basePath, "").replace(/^\//, ""); - const content = await Deno.readTextFile(filePath); - const fileUrl = `file://${filePath}`; - - const contentJson = { - path: relativePath, - content: content, - }; - - // Check if the file already exists in the database - const existingItem = await db.query<[number]>( - "SELECT id FROM items WHERE url = ?", - [fileUrl], - ); - - if (existingItem.length > 0) { - // Update existing record - const itemId = existingItem[0][0]; - await db.query( - "UPDATE items SET title = ?, content = ?, raw_content = ? WHERE id = ?", - [relativePath, JSON.stringify(contentJson), content, itemId], - ); - - // Ensure the item is associated with the current collection - await db.query( - "INSERT OR IGNORE INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - - return "updated"; - } else { - // Insert new record - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - fileUrl, - relativePath, - JSON.stringify(contentJson), - content, - "Local File", - ], - ); - const itemId = result[0][0] as number; - - await db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - - return "new"; - } -} diff --git a/typescript/packages/collectathon/items.ts b/typescript/packages/collectathon/items.ts deleted file mode 100644 index 5dc0299fe..000000000 --- a/typescript/packages/collectathon/items.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { addItemToCollection } from "./collections.ts"; -import { db } from "./db.ts"; - -export function printItem(itemId: number, showRaw: boolean = false) { - const item = db.query< - [number, string, string, string, string, string, string] - >( - "SELECT id, url, title, content, raw_content, source, created_at FROM items WHERE id = ?", - [itemId], - )[0]; - - if (!item) { - console.log(`Item with ID ${itemId} not found.`); - return; - } - - const [id, url, title, content, rawContent, source, createdAt] = item; - - console.log(`Item ID: ${id}`); - console.log(`URL: ${url}`); - console.log(`Title: ${title}`); - console.log(`Source: ${source}`); - console.log(`Created At: ${createdAt}`); - console.log("\nContent:"); - - try { - const contentObj = JSON.parse(content); - console.log(JSON.stringify(contentObj, null, 2)); - } catch (error) { - console.log("Error parsing JSON content:", error.message); - console.log("Raw content:", content); - } - - if (showRaw) { - console.log("\nRaw Content:"); - console.log(rawContent); - } - - // Print associated collections - const collections = db.query<[string]>( - `SELECT c.name - FROM collections c - JOIN item_collections ic ON c.id = ic.collection_id - WHERE ic.item_id = ?`, - [itemId], - ); - - if (collections.length > 0) { - console.log("\nAssociated Collections:"); - collections.forEach(([name]) => console.log(`- ${name}`)); - } -} - -export function getItem(itemId: number): any { - const item = db.query< - [number, string, string, string, string, string, string] - >( - "SELECT id, url, title, content, raw_content, source, created_at FROM items WHERE id = ?", - [itemId], - )[0]; - - if (!item) { - return null; - } - - const [id, url, title, content, rawContent, source, createdAt] = item; - - const collections = db.query<[string]>( - `SELECT c.name - FROM collections c - JOIN item_collections ic ON c.id = ic.collection_id - WHERE ic.item_id = ?`, - [itemId], - ); - - return { - id, - url, - title, - content: JSON.parse(content), - rawContent, - source, - createdAt, - collections: collections.map(([name]) => name), - }; -} - -export function deleteItem(itemId: number) { - try { - db.query("BEGIN TRANSACTION"); - - // Delete from item_collections first to maintain referential integrity - db.query("DELETE FROM item_collections WHERE item_id = ?", [itemId]); - - // Then delete the item itself - const result = db.query("DELETE FROM items WHERE id = ?", [itemId]); - - db.query("COMMIT"); - - if (result.length > 0) { - console.log(`Item with ID ${itemId} has been deleted.`); - } else { - console.log(`No item found with ID ${itemId}.`); - } - } catch (error) { - db.query("ROLLBACK"); - console.error(`Error deleting item: ${error.message}`); - } -} - -export async function editItemCLI( - itemId: number, - editRawContent: boolean = false, -) { - const item = db.query<[string, string]>( - "SELECT content, raw_content FROM items WHERE id = ?", - [itemId], - )[0]; - - if (!item) { - console.log(`Item with ID ${itemId} not found.`); - return; - } - - const [content, rawContent] = item; - const contentToEdit = editRawContent ? rawContent : content; - - // Create a temporary file - const tempFile = await Deno.makeTempFile({ suffix: ".txt" }); - await Deno.writeTextFile(tempFile, contentToEdit); - - // Open the editor - const editor = Deno.env.get("EDITOR") || "nano"; - const process = Deno.run({ - cmd: [editor, tempFile], - stdin: "inherit", - stdout: "inherit", - stderr: "inherit", - }); - - // Wait for the editor to close - await process.status(); - - // Read the edited content - const editedContent = await Deno.readTextFile(tempFile); - - // Update the database if the content has changed - if (editedContent !== contentToEdit) { - try { - if (editRawContent) { - db.query("UPDATE items SET raw_content = ? WHERE id = ?", [ - editedContent, - itemId, - ]); - } else { - db.query("UPDATE items SET content = ? WHERE id = ?", [ - editedContent, - itemId, - ]); - } - console.log(`Item ${itemId} has been updated.`); - } catch (error) { - console.error(`Error updating item: ${error.message}`); - } - } else { - console.log("No changes were made."); - } - - // Clean up the temporary file - await Deno.remove(tempFile); -} - -export function editItemWeb( - itemId: number, - editRawContent: boolean, - newContent: string -) { - try { - if (editRawContent) { - db.query("UPDATE items SET raw_content = ? WHERE id = ?", [ - newContent, - itemId, - ]); - } else { - db.query("UPDATE items SET content = ? WHERE id = ?", [ - newContent, - itemId, - ]); - } - console.log(`Item ${itemId} has been updated.`); - return true; - } catch (error) { - console.error(`Error updating item: ${error.message}`); - return false; - } -} - -export async function createNewItem(content: any, collections?: string[]) { - try { - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - content.url || "", - content.title || "New item", - JSON.stringify(content), - JSON.stringify(content.raw_content || content), - "API", - ] - ); - const itemId = result[0][0] as number; - - if (collections && collections.length > 0) { - for (const collection of collections) { - await addItemToCollection(itemId, collection); - } - } - - return itemId; - } catch (error) { - console.error(`Error creating new item: ${error.message}`); - return null; - } -} - -export function purge() { - try { - db.query("BEGIN TRANSACTION"); - - // Find and delete items that are not members of any collection - const result = db.query(` - DELETE FROM items - WHERE id NOT IN ( - SELECT DISTINCT item_id - FROM item_collections - ) - `); - - const purgedCount = result.length; - - db.query("COMMIT"); - - console.log( - `Purged ${purgedCount} items that were not members of any collection.`, - ); - } catch (error) { - db.query("ROLLBACK"); - console.error(`Error purging items: ${error.message}`); - } -} diff --git a/typescript/packages/collectathon/llm.ts b/typescript/packages/collectathon/llm.ts deleted file mode 100644 index 5d25b65a3..000000000 --- a/typescript/packages/collectathon/llm.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { CoreMessage } from "npm:ai@3.3.21"; -import { ai, anthropic, openai } from "./deps.ts"; -const { streamText, generateText } = ai; - -const SONNET = "claude-3-5-sonnet-latest"; -const HAIKU = "claude-3-haiku-20240307"; -const O1_MINI = "o1-mini"; -const O1_PREVIEW = "o1-preview"; -const model = anthropic(SONNET); -const fastModel = anthropic(HAIKU); -const smartModel = openai(O1_PREVIEW); - -export function grabJson(txt: string) { - // try parsing whole string first - try { - return JSON.parse(txt); - } catch (error) { - // if that fails, try to grab it from the text - } - - const json = txt.match(/```json\n([\s\S]+?)```/)?.[1]; - if (!json) { - console.error("No JSON found in text", txt); - return {}; - } - return JSON.parse(json); -} - -export function grabHtml(txt: string) { - const html = txt.match(/```html\n([\s\S]+?)```/)?.[1]; - if (!html) { - console.error("No HTML found in text", txt); - return "" - } - return html; -} - -export async function chat( - system: string, - messages: CoreMessage[], - silent = false, -) { - const { textStream: analysisStream } = await streamText({ - model: model, - system, - messages, - temperature: 1.0 - }); - - let message = ""; - for await (const delta of analysisStream) { - message += delta; - if (!silent) { - Deno.stdout.writeSync(new TextEncoder().encode(delta)); - } - } - - return message; -} - -export async function completion(system: string, messages: CoreMessage[]) { - const { textStream: analysisStream } = await streamText({ - model: model, - system, - messages, - }); - - let message = ""; - for await (const delta of analysisStream) { - message += delta; - Deno.stdout.writeSync(new TextEncoder().encode(delta)); - } - - const analysis = grabJson(message); - return analysis; -} - -export async function fastCompletion( - system: string, - messages: CoreMessage[], - silent = false, -) { - const { textStream: analysisStream } = await streamText({ - model: fastModel, - system, - messages, - }); - - let message = ""; - for await (const delta of analysisStream) { - message += delta; - if (!silent) { - Deno.stdout.writeSync(new TextEncoder().encode(delta)); - } - } - - const analysis = grabJson(message); - return analysis; -} - -export async function smart( - messages: CoreMessage[], - silent = false, -) { - const result = await generateText({ - model: smartModel, - messages, - temperature: 1.0, - }); - - return grabJson(result.text); -} diff --git a/typescript/packages/collectathon/mail.ts b/typescript/packages/collectathon/mail.ts deleted file mode 100644 index d53659195..000000000 --- a/typescript/packages/collectathon/mail.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { serve } from "https://deno.land/std@0.184.0/http/server.ts"; -import { fastCompletion } from "./llm.ts"; -import { clip, clipEmail } from "./synopsys.ts"; - -const port = 8025; - -export async function ingestEmail(subject: string, body: string): Promise { - const systemPrompt = - "Convert the attached email into a JSON object. The content of the email or subject may have further instructions on how to format the data but you must return a JSON array of objects e.g. a ```json [{...}]``` block, no commentary. Each object must be flat, no nested object hierarachy is permitted."; - const userPrompt = ` -Subject: ${subject} - -Body: -${body} - ---- - -Format the output as a JSON array of one or more objects in a \`\`\`json\`\`\ block. - `; - - const response = await fastCompletion(systemPrompt, [ - { role: "user", content: userPrompt }, - ]); - - return response; -} - -async function handleWebhook(req: Request): Promise { - if (req.method !== 'POST') { - return new Response('Method Not Allowed', { status: 405 }); - } - - try { - const formData = await req.formData(); - const sender = formData.get('sender')?.toString(); - const recipient = formData.get('recipient'); - const subject = formData.get('subject')?.toString(); - const bodyPlain = formData.get('body-plain')?.toString(); - - console.log('Received email:'); - console.log('From:', sender); - console.log('To:', recipient); - ; - const allNames: string[] = []; - - const recipientName = recipient?.toString().split('@')[0]; - if (recipientName) { - allNames.push(recipientName); - } - console.log('Recipient Name:', recipientName); - - const cc = formData.get('cc'); - if (cc) { - const ccNames = cc.toString().split(',').map(email => email.trim().split('@')[0]); - allNames.push(...ccNames); - console.log('CC Names:', ccNames); - } - - console.log('All Names:', allNames); - - console.log('Subject:', subject); - console.log('Body:', bodyPlain); - if (!subject || !bodyPlain || !sender || !recipient) { - throw new Error('Missing data'); - } - const ingested = await ingestEmail(subject, bodyPlain); - for (const entity of ingested) { - console.log('Entity:', entity); - clipEmail(sender, allNames, entity); - } - - return new Response('OK', { status: 200 }); - } catch (error) { - console.error('Error processing webhook:', error); - return new Response('Internal Server Error', { status: 500 }); - } -} - -export async function start() { - - console.log(`Webhook server running on http://localhost:${port}`); - await serve(handleWebhook, { port }); -} diff --git a/typescript/packages/collectathon/main.ts b/typescript/packages/collectathon/main.ts deleted file mode 100644 index 2107a7160..000000000 --- a/typescript/packages/collectathon/main.ts +++ /dev/null @@ -1,257 +0,0 @@ -import { addActionCommand } from "./action.ts"; -import { startChat } from "./chat.ts"; -import { - addItemToCollection, - deleteCollection, - listCollections, - listItems, - removeItemFromCollection, - moveCollection, -} from "./collections.ts"; -import { readLines } from "./deps.ts"; -import { addDreamCommand } from "./dream.ts"; -import { importFiles, clipUrl } from "./import.ts"; -import { deleteItem, editItemCLI, printItem, purge } from "./items.ts"; -import { addRule, applyRules, deleteRule, listRules } from "./rules.ts"; -import { search } from "./search.ts"; -import { handleViewCommandInteractive } from "./view.ts"; - -function listAPI() { - console.log("Available commands:"); - console.log(" clip [-p PROMPT]"); - console.log(" collection list"); - console.log(" collection delete "); - console.log(" collection apply-rules "); - console.log(" collection move "); - console.log(" item list "); - console.log(" item show [-raw]"); - console.log(" item delete "); - console.log(" item edit [-raw]"); - console.log(" item add "); - console.log(" item remove "); - console.log(" item purge"); - console.log(" rule add "); - console.log(" rule list "); - console.log(" rule delete "); - console.log(" chat [COLLECTION2 ...]"); - console.log(" import [FILE_TYPE_FILTER]"); - console.log(" search "); - console.log(" action "); - console.log(" dream "); - console.log(" view "); - console.log(" exit"); -} - -async function main() { - console.log("Welcome to the Collection Clipper CLI!"); - listAPI(); - - for await (const line of readLines(Deno.stdin)) { - const args = line.trim().split(" "); - const command = args.shift()?.toLowerCase(); - - switch (command) { - case "clip": - if (args.length < 2) { - console.log("Usage: clip [-p PROMPT]"); - } else { - const [url, collection, ...rest] = args; - const promptIndex = rest.indexOf("-p"); - const prompt = - promptIndex !== -1 - ? rest.slice(promptIndex + 1).join(" ") - : undefined; - - clipUrl(url, collection, prompt, undefined); - } - break; - case "collection": - if (args.length < 1) { - console.log("Usage: collection [arguments]"); - break; - } - const collectionAction = args.shift(); - switch (collectionAction) { - case "list": - await listCollections(); - break; - case "delete": - if (args.length !== 1) { - console.log("Usage: collection delete "); - } else { - await deleteCollection(args[0]); - } - break; - case "apply-rules": - if (args.length !== 1) { - console.log("Usage: collection apply-rules "); - } else { - await applyRules(args[0]); - } - break; - case "move": - if (args.length !== 2) { - console.log("Usage: collection move "); - } else { - await moveCollection(args[0], args[1]); - } - break; - default: - console.log( - "Unknown collection action. Available actions: list, delete, apply-rules, move", - ); - } - break; - case "item": - if (args.length < 1) { - console.log("Usage: item [arguments]"); - break; - } - const itemAction = args.shift(); - switch (itemAction) { - case "list": - if (args.length !== 1) { - console.log("Usage: item list "); - } else { - await listItems(args[0]); - } - break; - case "show": - if (args.length < 1) { - console.log("Usage: item show [-raw]"); - } else { - const itemId = parseInt(args[0]); - const showRaw = args[1] === "-raw"; - printItem(itemId, showRaw); - } - break; - case "delete": - if (args.length !== 1) { - console.log("Usage: item delete "); - } else { - deleteItem(parseInt(args[0])); - } - break; - case "edit": - if (args.length < 1 || args.length > 2) { - console.log("Usage: item edit [-raw]"); - } else { - const itemId = parseInt(args[0]); - const editRaw = args[1] === "-raw"; - await editItemCLI(itemId, editRaw); - } - break; - case "add": - if (args.length !== 2) { - console.log("Usage: item add "); - } else { - await addItemToCollection(parseInt(args[0]), args[1]); - } - break; - case "remove": - if (args.length !== 2) { - console.log("Usage: item remove "); - } else { - await removeItemFromCollection(parseInt(args[0]), args[1]); - } - break; - case "purge": - await purge(); - break; - default: - console.log( - "Unknown item action. Available actions: list, show, delete, edit, add, remove, purge", - ); - } - break; - case "rule": - if (args.length < 1) { - console.log("Usage: rule [arguments]"); - break; - } - const ruleAction = args.shift(); - switch (ruleAction) { - case "add": - if (args.length < 3) { - console.log( - "Usage: rule add ", - ); - } else { - const collection = args.shift()!; - const targetCollection = args.pop()!; - const rule = args.join(" "); - await addRule(collection, rule, targetCollection); - } - break; - case "list": - if (args.length !== 1) { - console.log("Usage: rule list "); - } else { - await listRules(args[0]); - } - break; - case "delete": - if (args.length !== 1) { - console.log("Usage: rule delete "); - } else { - await deleteRule(parseInt(args[0])); - } - break; - default: - console.log( - "Unknown rule action. Available actions: add, list, delete", - ); - } - break; - case "chat": - if (args.length < 1) { - console.log("Usage: chat [COLLECTION2 ...]"); - } else { - await startChat(args); - } - break; - case "import": - if (args.length < 2 || args.length > 3) { - console.log("Usage: import [FILE_TYPE_FILTER]"); - } else { - const [path, collection, fileTypeFilter] = args; - await importFiles(path, collection, fileTypeFilter); - } - break; - case "search": - if (args.length === 0) { - console.log("Usage: search "); - } else { - const query = args.join(" "); - await search(query); - } - break; - case "action": - addActionCommand(args); - break; - case "dream": - addDreamCommand(args); - break; - case "view": - if (args.length < 2) { - console.log("Usage: view "); - } else { - const collection = args.shift()!; - const initialPrompt = args.join(" "); - await handleViewCommandInteractive(collection, initialPrompt); - } - break; - case "exit": - console.log("Goodbye!"); - Deno.exit(0); - break; - default: - console.log("Unknown command."); - listAPI(); - } - } -} - -if (import.meta.main) { - main(); -} diff --git a/typescript/packages/collectathon/package.json b/typescript/packages/collectathon/package.json deleted file mode 100644 index f3eccf885..000000000 --- a/typescript/packages/collectathon/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@commontools/collectathon", - "author": "The Common Authors", - "version": "0.0.1", - "description": "A link clipping prototype.", - "license": "UNLICENSED", - "private": true, - "type": "module", - "scripts": { - "start": "./start.sh", - "server": "deno run --allow-net --allow-read --allow-write --allow-env --allow-run server.ts", - "cli": "deno run --allow-net --allow-read --allow-write --allow-env --allow-run main.ts", - "build": "wireit", - "clean": "wireit" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/commontoolsinc/labs.git" - }, - "bugs": { - "url": "https://github.com/commontoolsinc/labs/issues" - }, - "homepage": "https://github.com/commontoolsinc/labs#readme", - "exports": "./lib/index.js", - "files": [ - "./lib/*.js" - ], - "dependencies": {}, - "devDependencies": { - "tslib": "^2.6.2", - "typescript": "^5.2.2", - "wireit": "^0.14.4" - }, - "wireit": { - "build": { - "dependencies": [], - "files": [ - "./src/**/*" - ], - "output": [ - "./lib/**/*" - ], - "command": "tsc --build -f" - }, - "clean": { - "command": "rm -rf ./lib ./.wireit" - } - } -} diff --git a/typescript/packages/collectathon/rss.ts b/typescript/packages/collectathon/rss.ts deleted file mode 100644 index 5370de386..000000000 --- a/typescript/packages/collectathon/rss.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { getOrCreateCollection } from "./collections.ts"; -import { db } from "./db.ts"; -import { parseFeed } from "./deps.ts"; - -export async function clipRSS(url: string, collections: string[]) { - try { - db.query("BEGIN TRANSACTION"); - const response = await fetch(url); - const xml = await response.text(); - const feed = await parseFeed(xml); - - const collectionIds = await Promise.all(collections.map(collectionName => - getOrCreateCollection(collectionName) - )); - let itemCount = 0; - - for (const item of feed.entries) { - const contentJson: Record = {}; - - // Extract all key-value pairs from the item - for (const [key, value] of Object.entries(item)) { - if (typeof value === "object" && value !== null) { - if ("value" in value) { - contentJson[key] = value.value; - } else { - contentJson[key] = value; - } - } else { - contentJson[key] = value; - } - } - - const result = db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - item.links[0].href, - item.title?.value, - JSON.stringify(contentJson), - item.description?.value || "", - "RSS", - ], - ); - const itemId = result[0][0] as number; - - for (const collectionId of collectionIds) { - db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - } - - itemCount++; - } - - db.query("COMMIT"); - - console.log( - `Clipped ${itemCount} items from RSS feed to collections: ${collections.join(', ')}`, - ); - } catch (error) { - db.query("ROLLBACK"); - console.error(`Error clipping RSS feed: ${error.message}`); - } -} diff --git a/typescript/packages/collectathon/rules.ts b/typescript/packages/collectathon/rules.ts deleted file mode 100644 index 18626e01c..000000000 --- a/typescript/packages/collectathon/rules.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { addItemToCollection, getOrCreateCollection } from "./collections.ts"; -import { db } from "./db.ts"; -import { completion } from "./llm.ts"; -import { Table } from "./deps.ts"; - -export async function addRule( - collectionName: string, - rule: string, - targetCollection: string, -) { - try { - // Ensure both collections exist - await getOrCreateCollection(collectionName); - await getOrCreateCollection(targetCollection); - - const result = db.query( - "INSERT INTO rules (collection_name, rule, target_collection) VALUES (?, ?, ?) RETURNING id", - [collectionName, rule, targetCollection], - ); - const ruleId = result[0][0] as number; - console.log( - `Added rule (ID: ${ruleId}) to collection "${collectionName}": "${rule}" -> "${targetCollection}"`, - ); - } catch (error) { - console.error(`Error adding rule: ${error.message}`); - } -} - -export async function listRules(collectionName: string) { - const rules = db.query<[number, string, string]>( - "SELECT id, rule, target_collection FROM rules WHERE collection_name = ? ORDER BY id", - [collectionName], - ); - - if (rules.length === 0) { - console.log(`No rules found for collection: ${collectionName}`); - return; - } - - console.log(`Rules for collection "${collectionName}":`); - - const t = new Table(); - - rules.forEach(function (rule) { - t.cell("Rule Id", rule[0]); - t.cell("Rule", rule[1]); - t.cell("Target Collection", rule[2]); - t.newRow(); - }); - - console.log(t.toString()); -} - -export async function deleteRule(ruleId: number) { - try { - db.query("DELETE FROM rules WHERE id = ?", [ruleId]); - console.log(`Deleted rule with ID: ${ruleId}`); - } catch (error) { - console.error(`Error deleting rule: ${error.message}`); - } -} - -export async function applyRules(collectionName: string, ruleId?: number) { - let rules: [number, string, string][]; - - if (ruleId) { - rules = db.query<[number, string, string]>( - "SELECT id, rule, target_collection FROM rules WHERE id = ?", - [ruleId], - ); - if (rules.length === 0) { - console.error(`Rule with ID ${ruleId} not found`); - return; - } - } else { - rules = db.query<[number, string, string]>( - "SELECT id, rule, target_collection FROM rules WHERE collection_name = ?", - [collectionName], - ); - if (rules.length === 0) { - console.log(`No rules found for collection: ${collectionName}`); - return; - } - } - - const items = db.query<[number, string, string]>( - `SELECT i.id, i.content, i.raw_content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collectionName], - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collectionName}`); - return; - } - - for (const [itemId, content, rawContent] of items) { - for (const [ruleId, rule, targetCollection] of rules) { - const systemPrompt = - "You are an expert at evaluating content based on given criteria."; - const userPrompt = ` -Evaluate if the following content matches this rule: "${rule}" - -Content: -${itemId} -${content} -${rawContent} - -Respond with a JSON object of the form - -\`\`\`json -{ "match": true } -\`\`\` or -\`\`\`json -{ "match": false } -\`\`\` - -Say NOTHING else. -`; - - const response = await completion(systemPrompt, [ - { role: "user", content: userPrompt }, - ]); - - if (response.match) { - await addItemToCollection(itemId, targetCollection); - console.log( - `Rule ${ruleId} matched. Added item ${itemId} to collection "${targetCollection}"`, - ); - } - } - } -} diff --git a/typescript/packages/collectathon/schema.ts b/typescript/packages/collectathon/schema.ts deleted file mode 100644 index 88e3c8834..000000000 --- a/typescript/packages/collectathon/schema.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { CoreMessage } from "npm:ai@3.3.21"; -import { fastCompletion } from "./llm.ts"; - -export async function extractJsonShape(items: any[]): Promise { - const systemPrompt = - "Output a json schema that covers all the keys and values of the JSON objects."; - const userMessage = `${JSON.stringify(items, null, 2)}`; - - const messages: CoreMessage[] = [ - { role: "system", content: systemPrompt }, - { role: "user", content: userMessage }, - ]; - - const response = await fastCompletion(systemPrompt, messages); - return response; -} diff --git a/typescript/packages/collectathon/search.ts b/typescript/packages/collectathon/search.ts deleted file mode 100644 index 7de8149be..000000000 --- a/typescript/packages/collectathon/search.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { db } from "./db.ts"; -import { chat, completion } from "./llm.ts"; -import { - addItemToCollection, - getOrCreateCollection, - listCollections, - listItems, -} from "./collections.ts"; -import { CoreMessage } from "npm:ai@3.3.21"; - -export async function search(query: string) { - const searchDate = new Date().toISOString().slice(0, 10).replace(/-/g, ""); - const collectionName = `_search_${searchDate}_${query.replace(/\s+/g, "_")}`; - - // Generate SQL query using LLM - const sqlQuery = await generateSQLQuery(query); - - // Execute the generated SQL query - const results = db.query(sqlQuery); - - // Create ephemeral collection - const collectionId = await getOrCreateCollection(collectionName); - - // Add search results to the ephemeral collection - for (const [itemId] of results) { - await addItemToCollection(itemId, collectionName, true); - } - - await listItems(collectionName); - - console.log(`Search results saved to collection: ${collectionName}`); - console.log(`Found ${results.length} items matching the query.`); -} - -async function generateSQLQuery(userQuery: string): Promise { - const systemPrompt = - "You are an expert in SQLite. Generate a SQLite query to search for items based on the user's query."; - const userPrompt = ` -Generate a SQLite query to search for items based on the following user query: -"${userQuery}" - -Use the following table structure: -- items (id, url, title, content, raw_content, source, created_at) -- item_collections (item_id, collection_id) -- collections (id, name) - -Return only the SQLite query, without any explanation or additional text. -`; - - const messages: CoreMessage[] = [ - { role: "system", content: systemPrompt }, - { role: "user", content: userPrompt }, - ]; - - const response = await chat(systemPrompt, messages); - return response || ""; -} diff --git a/typescript/packages/collectathon/server.ts b/typescript/packages/collectathon/server.ts deleted file mode 100644 index c3c64e6f0..000000000 --- a/typescript/packages/collectathon/server.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { Application, Router, oakCors } from "./deps.ts"; -import { extractEntities } from "./webpage.ts"; -import { db } from "./db.ts"; -import { getOrCreateCollection } from "./collections.ts"; -import { clipUrl } from "./import.ts"; -import { clip } from "./synopsys.ts"; -import { start as mail } from "./mail.ts"; - -const app = new Application(); -const router = new Router(); - -app.use(oakCors()); - -router.get("/suggested-collections", async (ctx) => { - const currentUrl = ctx.request.url.searchParams.get("url"); - if (!currentUrl) { - ctx.response.status = 400; - ctx.response.body = { error: "URL parameter is required" }; - return; - } - - const baseUrl = new URL(currentUrl).origin; - - // First, get collections containing items from the same base URL - const suggestedCollections = db.query<[string]>( - `SELECT DISTINCT c.name - FROM collections c - JOIN item_collections ic ON c.id = ic.collection_id - JOIN items i ON ic.item_id = i.id - WHERE i.url LIKE ? - LIMIT 5`, - [`${baseUrl}%`], - ); - - // If we have less than 5 suggested collections, add recent collections to fill the gap - if (suggestedCollections.length < 5) { - const recentCollections = db.query<[string]>( - `SELECT DISTINCT name - FROM collections - WHERE name NOT IN (${suggestedCollections.map(() => "?").join(",")}) - ORDER BY id DESC - LIMIT ?`, - [ - ...suggestedCollections.map(([name]) => name), - 5 - suggestedCollections.length, - ], - ); - suggestedCollections.push(...recentCollections); - } - - ctx.response.body = suggestedCollections.map(([name]) => name); -}); - -router.get("/search-collections", async (ctx) => { - const query = ctx.request.url.searchParams.get("q"); - if (!query) { - ctx.response.status = 400; - ctx.response.body = { error: "Query parameter 'q' is required" }; - return; - } - const searchResults = db.query<[string]>( - "SELECT name FROM collections WHERE name LIKE ? ORDER BY name LIMIT 5", - [`%${query}%`], - ); - ctx.response.body = searchResults.map(([name]) => name); -}); - -router.post("/clip", async (ctx) => { - try { - const body = ctx.request.body(); - if (body.type === "json") { - const { url, collections, prompt, content } = await body.value; - if (!url || !collections || collections.length === 0) { - throw new Error("URL and collection are required"); - } - console.log( - "Clipping content:", - content, - "to collections:", - collections, - "with prompt:", - prompt, - ); - - let entities; - if (content.type === "webpage") { - entities = await clipUrl(url, collections, prompt, content.html); - } else { - entities = await extractEntities( - JSON.stringify(content), - url, - "If the provided content is already JSON then simply return it. " + - prompt, - ); - await saveEntities(entities, collections); - } - - ctx.response.body = { message: "Content clipped successfully", entities }; - } else { - throw new Error("Invalid request body"); - } - } catch (error) { - console.error("Error processing request:", error); - ctx.response.status = 400; - ctx.response.body = { error: error.message }; - } -}); - -async function saveEntities(entities: any[], collections: string[]) { - const collectionIds = await Promise.all( - collections.map((collectionName) => getOrCreateCollection(collectionName)), - ); - - for (const entity of entities) { - const result = await db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - entity.url || "", - entity.title || `${entity.type} from clipped content`, - JSON.stringify(entity), - JSON.stringify(entity.content), - "Clipped Content", - ], - ); - const itemId = result[0][0] as number; - - await clip(entity.url || "", collections, entity); - - for (const collectionId of collectionIds) { - await db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - } - } -} - -app.use(router.routes()); -app.use(router.allowedMethods()); - -export const PORT = 8001; -let server; - -export async function start() { - console.log(`Server running on http://localhost:${PORT}`); - server = await app.listen({ port: PORT }); -} - -function shutdown() { - console.log("Shutting down server..."); - if (server) { - server.close(); - console.log("Server shut down successfully"); - } - Deno.exit(0); -} - -if (Deno) { - Deno.addSignalListener("SIGINT", shutdown); - Deno.addSignalListener("SIGTERM", shutdown); -} - -if (import.meta.main) { - start(); - mail(); -} diff --git a/typescript/packages/collectathon/spec/dreams.md b/typescript/packages/collectathon/spec/dreams.md deleted file mode 100644 index c9dfafbad..000000000 --- a/typescript/packages/collectathon/spec/dreams.md +++ /dev/null @@ -1,9 +0,0 @@ -## Feature: Dreams - -Implement a new command in the tool, `dream `. - -Similar to the `action` command, it will first determine the "shape" of data in the collection and then pass the shape + array of JSON records to an LLM that will "dream" a new item for the collection, based on the existing ones. The item should contain novel ideas or data but fit within the set it is part of. - -Extract any shared functionality for schema/shape into "schema.ts", the dream logic goes in "dream.ts" - -Return the files in full + the changes for the main.ts file. diff --git a/typescript/packages/collectathon/start.sh b/typescript/packages/collectathon/start.sh deleted file mode 100755 index 87d7a909f..000000000 --- a/typescript/packages/collectathon/start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -killall deno -deno run --allow-net --allow-read --allow-write --allow-env --allow-run server.ts & -deno run --allow-net --allow-read --allow-write --allow-env --allow-run main.ts diff --git a/typescript/packages/collectathon/synopsys.ts b/typescript/packages/collectathon/synopsys.ts deleted file mode 100644 index e551c3e40..000000000 --- a/typescript/packages/collectathon/synopsys.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { CID, json, sha256 } from "./deps.ts"; - -export function entity(id: string) { - return { "/": id }; -} - -type Entity = ReturnType; -type Attribute = string; -type Value = Entity | string | number | boolean; -type Fact = [Entity, Attribute, Value]; - -const SYNOPSYS_URL = Deno.env.get("SYNOPSYS_URL") || "http://localhost:8080"; - -export async function clipEmail( - sender: string, - collections: string[], - entity: any, - entityCid?: string, -) { - entity["import/sender"] = sender; - entity["import/source"] = "Email"; - entity["import/tool"] = "ingest"; - entity["import/time"] = new Date().toISOString(); - - if (!entityCid) { - entityCid = await cid(entity); - } - - const entityFacts = await jsonToFacts(entity, { "/": entityCid } as Entity); - - const collectionsFacts = await Promise.all( - collections.map(async (collectionName) => { - const collection = { name: collectionName, type: "collection" }; - return await jsonToFacts(collection); - }), - ); - - const mergedCollectionFacts = collectionsFacts.flat(); - - const memberFacts = await Promise.all( - collections.map(async (collectionName) => { - const collection = { name: collectionName, type: "collection" }; - const collectionCid = await cid(collection); - return [{ "/": collectionCid }, "member", { "/": entityCid }] as Fact; - }), - ); - - const response = await assert( - ...mergedCollectionFacts, - ...entityFacts, - ...memberFacts, - ); - console.log("assert", response); -} - -export async function clip(url: string, collections: string[], entity: any) { - entity["import/url"] = url; - entity["import/source"] = "Webpage"; - entity["import/tool"] = "clipper"; - entity["import/time"] = new Date().toISOString(); - - const entityCid = await cid(entity); - const entityFacts = await jsonToFacts(entity); - - const collectionsFacts = await Promise.all( - collections.map(async (collectionName) => { - const collection = { name: collectionName, type: "collection" }; - return await jsonToFacts(collection); - }), - ); - - const mergedCollectionFacts = collectionsFacts.flat(); - - const memberFacts = await Promise.all( - collections.map(async (collectionName) => { - const collection = { name: collectionName, type: "collection" }; - const collectionCid = await cid(collection); - return [{ "/": collectionCid }, "member", { "/": entityCid }] as Fact; - }), - ); - - const response = await assert( - ...mergedCollectionFacts, - ...entityFacts, - ...memberFacts, - ); - console.log("assert", response); -} - -export async function cid(data: any) { - const bytes = json.encode(data); - const hash = await sha256.digest(bytes); - const cid = CID.create(1, json.code, hash); - console.log("cid", cid.toString(), data); - return cid.toString(); -} - -// send in a parentEntity if you create the item cid somewhere else -export async function jsonToFacts(data: any, parentEntity?: Entity) { - const facts: Fact[] = []; - const processObject = ( - obj: any, - parentEntity?: Entity, - prefix: string = "", - ) => { - for (const [key, value] of Object.entries(obj)) { - const fullKey = prefix ? `${prefix}/${key}` : key; - if (Array.isArray(value)) { - // Handle arrays - for (const item of value) { - if (typeof item !== "object" && typeof item !== "function") { - facts.push([parentEntity as Entity, fullKey, item as Value]); - } - } - } else if (typeof value === "object" && value !== null) { - if ("/" in value) { - // This is an Entity - facts.push([parentEntity as Entity, fullKey, value as Entity]); - } else { - // Flatten nested objects - processObject(value, parentEntity, fullKey); - } - } else if (typeof value !== "function") { - facts.push([parentEntity as Entity, fullKey, value as Value]); - } - } - }; - - if (typeof data === "object" && data !== null) { - if ("/" in data) { - // The root object is an Entity - processObject(data, data as Entity); - } else { - // The root object is not an Entity, create a new one - const rootEntity = parentEntity || entity(await cid(data)); - processObject(data, rootEntity); - } - } - - return facts.filter((fact) => fact[2] !== null && fact[2] !== undefined); -} - -export async function assert(...facts: Fact[]) { - const body = JSON.stringify(facts.map((f) => ({ Assert: f }))); - console.log("URL", SYNOPSYS_URL, body); - const response = await fetch(SYNOPSYS_URL, { - method: "PATCH", - body, - }); - if (!response.ok) { - throw new Error(`Error asserting facts: ${response.statusText}`); - } - - return await response.json(); -} - -export async function* query() { - const request = await fetch(SYNOPSYS_URL, { - method: "PUT", - body: JSON.stringify({ - select: { - id: "?list", - name: "?name", - todo: [ - { - id: "?item", - title: "?title", - completed: "?done", - }, - ], - }, - where: [ - { Case: ["?list", "name", "?name"] }, - { Case: ["?list", "todo", "?item"] }, - { Case: ["?item", "title", "?title"] }, - { - Or: [ - { Case: ["?item", "done", "?done"] }, - { - And: [ - { Not: { Case: ["?item", "done", "?done"] } }, - { Is: ["?done", false] }, - ], - }, - ], - }, - ], - }), - }); - - const reader = request.body?.getReader(); - const utf8 = new TextDecoder(); - if (!reader) { - throw new Error("No reader"); - } - - while (true) { - const read = await reader.read(); - if (read.done) { - break; - } else { - const [id, event, data] = utf8.decode(read.value).split("\n"); - - yield { - id: id.slice("id:".length), - event: event.slice("event:".length), - data: JSON.parse(data.slice("data:".length)), - }; - } - } -} diff --git a/typescript/packages/collectathon/tests.sh b/typescript/packages/collectathon/tests.sh deleted file mode 100755 index e755610cb..000000000 --- a/typescript/packages/collectathon/tests.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash - -# Test script for Collectathon Web API - -BASE_URL="http://localhost:8001" - -# Colors for output -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -# Function to print colored output -print_result() { - if [ $1 -eq 0 ]; then - echo -e "${GREEN}[SUCCESS]${NC} $2" - else - echo -e "${RED}[FAILED]${NC} $2" - fi -} - -# Test collections -echo "Testing Collections API..." - -# List collections -curl -s "$BASE_URL/collections" | grep "Collections listed in console" -print_result $? "List collections" - -# Create a test collection -curl -s -X POST "$BASE_URL/collections/test_collection" -print_result $? "Create test collection" - -# Move collection -curl -s -X PUT "$BASE_URL/collections/test_collection/move" -H "Content-Type: application/json" -d '{"newName":"moved_test_collection"}' -print_result $? "Move collection" - -# Apply rules to collection -curl -s -X POST "$BASE_URL/collections/moved_test_collection/apply-rules" -print_result $? "Apply rules to collection" - -# Test items -echo -e "\nTesting Items API..." - -# Create a new item -NEW_ITEM_ID=$(curl -s -X POST "$BASE_URL/items" -H "Content-Type: application/json" -d '{"content":{"title":"Test Item","body":"This is a test item"},"collections":["moved_test_collection"]}' | jq -r '.itemId') -print_result $? "Create new item" - -# List items in collection -curl -s "$BASE_URL/collections/moved_test_collection/items" | grep "Items listed for collection moved_test_collection in console" -print_result $? "List items in collection" - -# Get item details -curl -s "$BASE_URL/items/$NEW_ITEM_ID" | grep "Item $NEW_ITEM_ID printed in console" -print_result $? "Get item details" - -# Update item -curl -s -X PUT "$BASE_URL/items/$NEW_ITEM_ID" -H "Content-Type: application/json" -d '{"content":{"title":"Updated Test Item","body":"This item has been updated"},"raw":false}' -print_result $? "Update item" - -# Remove item from collection -curl -s -X DELETE "$BASE_URL/items/$NEW_ITEM_ID/collections/moved_test_collection" -print_result $? "Remove item from collection" - -# Add item to collection -curl -s -X POST "$BASE_URL/items/$NEW_ITEM_ID/collections/moved_test_collection" -print_result $? "Add item to collection" - -# Test rules -echo -e "\nTesting Rules API..." - -# Add rule -curl -s -X POST "$BASE_URL/rules" -H "Content-Type: application/json" -d '{"collection":"moved_test_collection","rule":"title contains Test","targetCollection":"test_target_collection"}' -print_result $? "Add rule" - -# List rules -curl -s "$BASE_URL/collections/moved_test_collection/rules" | grep "Rules listed for collection moved_test_collection in console" -print_result $? "List rules" - -# Test search -echo -e "\nTesting Search API..." - -# Perform search -curl -s "$BASE_URL/search?q=test" | grep "Search results printed in console" -print_result $? "Perform search" - -# Test action -echo -e "\nTesting Action API..." - -# Perform action on collection -curl -s -X POST "$BASE_URL/collections/moved_test_collection/action" -H "Content-Type: application/json" -d '{"prompt":"Summarize the items"}' -print_result $? "Perform action on collection" - -# Test dream -echo -e "\nTesting Dream API..." - -# Generate dream for collection -curl -s -X POST "$BASE_URL/collections/moved_test_collection/dream" -print_result $? "Generate dream for collection" - -# Test view -echo -e "\nTesting View API..." - -# Generate view for collection -VIEW_ID=$(curl -s -X POST "$BASE_URL/collections/moved_test_collection/view" -H "Content-Type: application/json" -d '{"prompt":"Create a table view"}' | jq -r '.viewId') -print_result $? "Generate view for collection" - -# Update view for collection -curl -s -X PUT "$BASE_URL/collections/moved_test_collection/view/$VIEW_ID" -H "Content-Type: application/json" -d '{"prompt":"Update the table view with a new column"}' -print_result $? "Update view for collection" - -# Clean up -echo -e "\nCleaning up..." - -# Delete item -curl -s -X DELETE "$BASE_URL/items/$NEW_ITEM_ID" -print_result $? "Delete item" - -# Delete collection -curl -s -X DELETE "$BASE_URL/collections/moved_test_collection" -print_result $? "Delete collection" - -echo -e "\nTest script completed." diff --git a/typescript/packages/collectathon/tsconfig.json b/typescript/packages/collectathon/tsconfig.json deleted file mode 100644 index f0af7fb66..000000000 --- a/typescript/packages/collectathon/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "lib": ["es2022", "WebWorker"], - "outDir": "./lib", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/typescript/packages/collectathon/view.ts b/typescript/packages/collectathon/view.ts deleted file mode 100644 index cb10717ad..000000000 --- a/typescript/packages/collectathon/view.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { db } from "./db.ts"; -import { chat, grabHtml } from "./llm.ts"; -import { CoreMessage } from "npm:ai@3.3.21"; -import { open } from "https://deno.land/x/open@v0.0.5/index.ts"; -import { PORT } from "./server.ts"; - -export const views = new Map(); - -// 1. Interactive version -export async function handleViewCommandInteractive(collection: string, initialPrompt: string): Promise { - const items = db.query<[number, string]>( - `SELECT i.id, i.content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collection] - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collection}`); - return; - } - - const jsonItems = items.map(([id, content]) => ({ - id, - ...JSON.parse(content), - })); - - let currentPrompt = initialPrompt; - let messages: CoreMessage[] = []; - - while (true) { - const html = await generateHTML(jsonItems, currentPrompt, messages); - const viewId = crypto.randomUUID(); - - // Store the view in the database - db.query( - "INSERT INTO views (id, collection, html) VALUES (?, ?, ?)", - [viewId, collection, html] - ); - - const url = `http://localhost:8001/view/${collection}/${viewId}`; - console.log(`Opening view in browser: ${url}`); - await open(url); - - const nextPrompt = prompt("Enter a new prompt to regenerate the view, or type '/exit' to quit: "); - - if (nextPrompt?.toLowerCase() === '/exit') { - console.log("Exiting view mode."); - break; - } - - currentPrompt = nextPrompt || currentPrompt; - } -} - -// 2. Single-shot version -export async function handleViewCommandSingleShot(collection: string, prompt: string): Promise { - const items = db.query<[number, string]>( - `SELECT i.id, i.content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collection] - ); - - if (items.length === 0) { - console.log(`No items found in collection: ${collection}`); - return ""; - } - - const jsonItems = items.map(([id, content]) => ({ - id, - ...JSON.parse(content), - })); - - const messages: CoreMessage[] = []; - const html = await generateHTML(jsonItems, prompt, messages); - const viewId = crypto.randomUUID(); - - // Store the view in the database - db.query( - "INSERT INTO views (id, collection, html) VALUES (?, ?, ?)", - [viewId, collection, html] - ); - - const url = `http://localhost:${PORT}/view/${collection}/${viewId}`; - console.log(`Opening view in browser: ${url}`); - await open(url); - - return viewId; -} - -// 3. Update existing view version -export async function handleViewCommandUpdate(viewId: string, newPrompt: string): Promise { - const view = db.query<[string, string]>( - "SELECT collection, html FROM views WHERE id = ?", - [viewId] - )[0]; - - if (!view) { - console.log(`No view found with id: ${viewId}`); - return; - } - - const [collection, _] = view; - - const items = db.query<[number, string]>( - `SELECT i.id, i.content - FROM items i - JOIN item_collections ic ON i.id = ic.item_id - JOIN collections c ON ic.collection_id = c.id - WHERE c.name = ?`, - [collection] - ); - - const jsonItems = items.map(([id, content]) => ({ - id, - ...JSON.parse(content), - })); - - const messages: CoreMessage[] = []; - const html = await generateHTML(jsonItems, newPrompt, messages); - - // Update the view in the database - db.query( - "UPDATE views SET html = ? WHERE id = ?", - [html, viewId] - ); - - const url = `http://localhost:${PORT}/view/${collection}/${viewId}`; - console.log(`Updated view. Opening in browser: ${url}`); - await open(url); -} - -async function generateHTML(items: any[], prompt: string, messages: CoreMessage[]): Promise { - const systemPrompt = "You are an expert web developer. Generate a full HTML page including CSS and JavaScript to visualize the given data based on the user's prompt. You can use external libraries from CDNs if needed."; - const userMessage = ` -Generate a full HTML page to visualize this data: -${JSON.stringify(items, null, 2)} - -User's visualization prompt: ${prompt} - -Include all necessary HTML, CSS, and JavaScript in a single file. You can use external libraries from CDNs if needed. -`; - - if (messages.length === 0) { - messages.push({ role: "system", content: systemPrompt }); - } - messages.push({ role: "user", content: userMessage }); - - const response = await chat(systemPrompt, messages, false); - messages.push({ role: "assistant", content: response }); - return grabHtml(response); -} diff --git a/typescript/packages/collectathon/webpage.ts b/typescript/packages/collectathon/webpage.ts deleted file mode 100644 index ee481c3f3..000000000 --- a/typescript/packages/collectathon/webpage.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { getOrCreateCollection } from "./collections.ts"; -import { db } from "./db.ts"; -import { CID, json, sha256 } from "./deps.ts"; -import { completion, fastCompletion } from "./llm.ts"; -import { assert, cid, clip, jsonToFacts } from "./synopsys.ts"; - -export async function extractEntities( - html: string, - url: string, - prompt?: string, -) { - const systemPrompt = - 'Process the attached webpage HTML to fulfill the user\'s request. Respond only with the data extracted as an array e.g. ```json [{}, {}]``` block, no commentary. Each object must be flat, no nested object hierarachy is permitted. Escape all quotes used within strings, e.g. `"` -> `\\"`.'; - const userPrompt = ` -URL: ${url} - -HTML Content: -${html} - -Format the output as a JSON array of one or more objects in a \`\`\`json\`\`\ block. -${prompt ? `${prompt}` : `Extract a summary of the page as a JSON blob.`} -${prompt ? "Infer the shape of the data from the request." : `Use well-known keys for the entities from the set: ["title", "content-type" "author", "date", "content", "src", "summary", "name", "location"] but also include others to fulfill the request.`} - `; - - const response = await completion(systemPrompt, [ - { role: "user", content: userPrompt }, - ]); - - return response; -} - -export async function clipWebpage( - url: string, - collections: string[], - prompt?: string, - html?: string, -) { - try { - db.query("BEGIN TRANSACTION"); - - if (!html) { - console.log("fetching html"); - const response = await fetch(url); - html = await response.text(); - } else { - console.log("using passed html"); - } - - const entities = await extractEntities(html, url, prompt); - - let totalItemCount = 0; - - for (const entity of entities) { - await clip(url, collections, entity); - - for (const collectionName of collections) { - const collectionId = await getOrCreateCollection(collectionName); - - const result = db.query( - "INSERT INTO items (url, title, content, raw_content, source) VALUES (?, ?, ?, ?, ?) RETURNING id", - [ - url, - `${entity.type} from ${url}`, - JSON.stringify(entity), - JSON.stringify(entity.content), - "Webpage", - ], - ); - const itemId = result[0][0] as number; - - db.query( - "INSERT INTO item_collections (item_id, collection_id) VALUES (?, ?)", - [itemId, collectionId], - ); - } - } - - for (const collectionName of collections) { - const itemCount = entities.length; - console.log( - `Clipped ${itemCount} entities from webpage to collection: ${collectionName}`, - ); - totalItemCount += itemCount; - } - - db.query("COMMIT"); - return entities; - } catch (error) { - db.query("ROLLBACK"); - console.error(`Error clipping webpage: ${error.message}`); - } - - return []; -}