diff --git a/background-charm-service/cast-admin.ts b/background-charm-service/cast-admin.ts index 2022c584e..fc00fa5bd 100644 --- a/background-charm-service/cast-admin.ts +++ b/background-charm-service/cast-admin.ts @@ -6,13 +6,13 @@ import { setBobbyServerUrl, storage, } from "@commontools/runner"; -import { type DID, Identity } from "@commontools/identity"; +import { type DID } from "@commontools/identity"; import { createAdminSession } from "@commontools/identity"; import { BG_CELL_CAUSE, BG_SYSTEM_SPACE_ID, - bgUpdaterCharmsSchema, -} from "@commontools/utils"; + BGCharmEntriesSchema, +} from "./src/schema.ts"; import { getIdentity } from "./src/utils.ts"; const { recipePath, quit } = parseArgs( @@ -79,7 +79,7 @@ async function castRecipe() { const targetCell = getCell( spaceId as DID, cause, - bgUpdaterCharmsSchema, + BGCharmEntriesSchema, ); // Ensure the cell is synced diff --git a/background-charm-service/deno.json b/background-charm-service/deno.json index 614d8f161..660aadb31 100644 --- a/background-charm-service/deno.json +++ b/background-charm-service/deno.json @@ -1,4 +1,5 @@ { + "name": "@commontools/background-charm", "tasks": { "start": "deno run -A --unstable-worker-options src/main.ts", "help": "deno run -A src/main.ts --help", @@ -8,5 +9,9 @@ "lint": "deno lint", "fmt": "deno fmt" }, - "imports": {} + "imports": {}, + "exports": { + ".": "./src/lib.ts", + "./schema": "./src/schema.ts" + } } diff --git a/background-charm-service/src/lib.ts b/background-charm-service/src/lib.ts new file mode 100644 index 000000000..0a0d87b02 --- /dev/null +++ b/background-charm-service/src/lib.ts @@ -0,0 +1,11 @@ +export { + BackgroundCharmService, + type BackgroundCharmServiceOptions, +} from "./service.ts"; +export { + BG_CELL_CAUSE, + BG_SYSTEM_SPACE_ID, + type BGCharmEntry, + BGCharmEntrySchema, +} from "./schema.ts"; +export { setBGCharm } from "./utils.ts"; diff --git a/background-charm-service/src/main.ts b/background-charm-service/src/main.ts index f51808c81..4a32aac5c 100644 --- a/background-charm-service/src/main.ts +++ b/background-charm-service/src/main.ts @@ -1,3 +1,4 @@ +import { storage } from "@commontools/runner"; import { BackgroundCharmService } from "./service.ts"; import { getIdentity, log } from "./utils.ts"; import { env } from "./env.ts"; @@ -6,6 +7,7 @@ const identity = await getIdentity(env.IDENTITY, env.OPERATOR_PASS); const service = new BackgroundCharmService({ identity, toolshedUrl: env.TOOLSHED_API_URL, + storage, }); const shutdown = () => { diff --git a/background-charm-service/src/schema.ts b/background-charm-service/src/schema.ts new file mode 100644 index 000000000..f66a2cacc --- /dev/null +++ b/background-charm-service/src/schema.ts @@ -0,0 +1,37 @@ +import { type JSONSchema, type Schema } from "@commontools/builder"; + +// This is the derived space id for toolshed-system +export const BG_SYSTEM_SPACE_ID = + "did:key:z6Mkfuw7h6jDwqVb6wimYGys14JFcyTem4Kqvdj9DjpFhY88"; +export const BG_CELL_CAUSE = "bgUpdater-2025-03-18"; +export const BGCharmEntrySchema = { + type: "object", + properties: { + space: { type: "string" }, + charmId: { type: "string" }, + integration: { type: "string" }, + createdAt: { type: "number" }, + updatedAt: { type: "number" }, + disabledAt: { type: "number", default: 0 }, + lastRun: { type: "number", default: 0 }, + status: { type: "string", default: "" }, + }, + required: [ + "space", + "charmId", + "integration", + "createdAt", + "updatedAt", + "lastRun", + "status", + ], +} as const as JSONSchema; +export type BGCharmEntry = Schema; + +export const BGCharmEntriesSchema = { + type: "array", + items: BGCharmEntrySchema, + default: [], +} as const satisfies JSONSchema; + +export type BGCharmEntries = Schema; diff --git a/background-charm-service/src/service.ts b/background-charm-service/src/service.ts index 2976737fd..cf38b24ae 100644 --- a/background-charm-service/src/service.ts +++ b/background-charm-service/src/service.ts @@ -1,13 +1,20 @@ import { Identity } from "@commontools/identity"; -import { type Cell, storage } from "@commontools/runner"; -import { type BGCharmEntry, getBGUpdaterCharmsCell } from "@commontools/utils"; -import { log } from "./utils.ts"; +import { type Cell, type Storage } from "@commontools/runner"; +import { + BG_CELL_CAUSE, + BG_SYSTEM_SPACE_ID, + type BGCharmEntry, +} from "./schema.ts"; +import { getBGCharms, log } from "./utils.ts"; import { SpaceManager } from "./space-manager.ts"; import { useCancelGroup } from "@commontools/runner"; export interface BackgroundCharmServiceOptions { identity: Identity; toolshedUrl: string; + storage: Storage; + bgSpace?: string; + bgCause?: string; } export class BackgroundCharmService { @@ -16,18 +23,24 @@ export class BackgroundCharmService { private charmSchedulers: Map = new Map(); private identity: Identity; private toolshedUrl: string; + private storage: Storage; + private bgSpace: string; + private bgCause: string; constructor(options: BackgroundCharmServiceOptions) { this.identity = options.identity; this.toolshedUrl = options.toolshedUrl; + this.storage = options.storage; + this.bgSpace = options.bgSpace ?? BG_SYSTEM_SPACE_ID; + this.bgCause = options.bgCause ?? BG_CELL_CAUSE; } async initialize() { - storage.setRemoteStorage(new URL(this.toolshedUrl)); - storage.setSigner(this.identity); - this.charmsCell = await getBGUpdaterCharmsCell(); - await storage.syncCell(this.charmsCell, true); - await storage.synced(); + this.storage.setRemoteStorage(new URL(this.toolshedUrl)); + this.storage.setSigner(this.identity); + this.charmsCell = await getBGCharms({ bgSpace: this.bgSpace, bgCause: this.bgCause, storage: this.storage }); + await this.storage.syncCell(this.charmsCell, true); + await this.storage.synced(); if (this.isRunning) { log("Service is already running"); diff --git a/background-charm-service/src/space-manager.ts b/background-charm-service/src/space-manager.ts index f335ba807..c23b04682 100644 --- a/background-charm-service/src/space-manager.ts +++ b/background-charm-service/src/space-manager.ts @@ -1,11 +1,12 @@ -import { BGCharmEntry, sleep } from "@commontools/utils"; +import { sleep } from "@commontools/utils"; import { Cell } from "@commontools/runner"; +import { type Cancel, useCancelGroup } from "@commontools/runner"; import { WorkerController, WorkerControllerErrorEvent, type WorkerOptions, } from "./worker-controller.ts"; -import { type Cancel, useCancelGroup } from "@commontools/runner"; +import { type BGCharmEntry } from "./schema.ts"; export interface CharmSchedulerOptions extends WorkerOptions { pollingIntervalMs?: number; diff --git a/background-charm-service/src/utils.ts b/background-charm-service/src/utils.ts index 4407eedc4..f69a9e5e8 100644 --- a/background-charm-service/src/utils.ts +++ b/background-charm-service/src/utils.ts @@ -1,6 +1,18 @@ import { Charm } from "@commontools/charm"; -import { Cell, getEntityId } from "@commontools/runner"; +import { + type Cell, + getCell, + getEntityId, + type Storage, +} from "@commontools/runner"; import { Identity, type IdentityCreateConfig } from "@commontools/identity"; +import { ID, type JSONSchema } from "@commontools/builder"; +import { + BG_CELL_CAUSE, + BG_SYSTEM_SPACE_ID, + type BGCharmEntry, + BGCharmEntrySchema, +} from "./schema.ts"; /** * Custom logger that includes timestamp and optionally charm ID @@ -87,3 +99,108 @@ export async function getIdentity( } throw new Error("No IDENTITY or OPERATOR_PASS environemnt set."); } + +export async function setBGCharm({ + space, + charmId, + integration, + storage, + bgSpace, + bgCause, +}: { + space: string; + charmId: string; + integration: string; + storage: Storage; + bgSpace?: string; + bgCause?: string; +}): Promise { + const charmsCell = await getBGCharms({ + bgSpace, + bgCause, + storage, + }); + + console.log( + "charmsCell", + JSON.stringify(charmsCell.getAsCellLink(), null, 2), + ); + + const charms = charmsCell.get() || []; + + const existingCharmIndex = charms.findIndex( + (charm: Cell) => + charm.get().space === space && charm.get().charmId === charmId, + ); + + if (existingCharmIndex === -1) { + console.log("Adding charm to BGUpdater charms cell"); + charmsCell.push({ + [ID]: `${space}/${charmId}`, + space, + charmId, + integration, + createdAt: Date.now(), + updatedAt: Date.now(), + disabledAt: undefined, + lastRun: 0, + status: "Initializing", + } as unknown as Cell); + + // Ensure changes are synced + await storage.synced(); + + return true; + } else { + console.log("Charm already exists in BGUpdater charms cell, re-enabling"); + const existingCharm = charms[existingCharmIndex]; + existingCharm.update({ + disabledAt: 0, + updatedAt: Date.now(), + status: "Re-initializing", + }); + + await storage.synced(); + + return false; + } +} + +export async function getBGCharms( + { bgSpace, bgCause, storage }: { + bgSpace?: string; + bgCause?: string; + storage: Storage; + }, +): Promise< + Cell[]> +> { + bgSpace = bgSpace ?? BG_SYSTEM_SPACE_ID; + bgCause = bgCause ?? BG_CELL_CAUSE; + + if (!storage.hasSigner()) { + throw new Error("Storage has no signer"); + } + + if (!storage.hasRemoteStorage()) { + throw new Error("Storage has no remote storage"); + } + const schema = { + type: "array", + items: { + ...BGCharmEntrySchema, + asCell: true, + }, + default: [], + } as const satisfies JSONSchema; + + const charmsCell = getCell(bgSpace, bgCause, schema); + + // Ensure the cell is synced + // FIXME(ja): does True do the right thing here? Does this mean: I REALLY REALLY + // INSIST THAT YOU HAVE THIS CELL ON THE SERVER! + await storage.syncCell(charmsCell, true); + await storage.synced(); + + return charmsCell; +} diff --git a/background-charm-service/src/worker-controller.ts b/background-charm-service/src/worker-controller.ts index 4ca54279f..2f1aab572 100644 --- a/background-charm-service/src/worker-controller.ts +++ b/background-charm-service/src/worker-controller.ts @@ -1,4 +1,4 @@ -import { BGCharmEntry } from "@commontools/utils"; +import { BGCharmEntry } from "./schema.ts"; import { Cell } from "@commontools/runner"; import { Identity } from "@commontools/identity"; import { defer, type Deferred } from "@commontools/utils/defer"; @@ -6,7 +6,6 @@ import { isWorkerIPCResponse, WorkerIPCMessageType, WorkerIPCRequest, - WorkerIPCResponse, } from "./worker-ipc.ts"; const DEFAULT_TASK_TIMEOUT = 60_000; diff --git a/recipes/bgAdmin.tsx b/recipes/bgAdmin.tsx index b9b15722d..d8e0bfe8f 100644 --- a/recipes/bgAdmin.tsx +++ b/recipes/bgAdmin.tsx @@ -3,13 +3,16 @@ import { derive, handler, JSONSchema, + lift, NAME, recipe, Schema, UI, } from "@commontools/builder"; -// NOTE(ja): this must be the same as the schema in utils/src/updaters.ts +const DISABLED_VIA_UI = "Disabled via UI"; + +// NOTE(ja): this must be the same as the schema in background-charm-service/src/schema.ts const BGCharmEntrySchema = { type: "object", properties: { @@ -34,14 +37,17 @@ const BGCharmEntrySchema = { } as const as JSONSchema; type BGCharmEntry = Schema; +const BGCharmEntriesSchema = { + type: "array", + items: BGCharmEntrySchema, + default: [], +} as const satisfies JSONSchema; +type BGCharmEntries = Schema; + const InputSchema = { type: "object", properties: { - charms: { - type: "array", - items: BGCharmEntrySchema, - default: [], - }, + charms: BGCharmEntriesSchema, }, } as const as JSONSchema; @@ -68,9 +74,164 @@ const deleteCharm = handler< ); const toggleCharm = handler((_, { charm }) => { - charm.disabledAt = charm.disabledAt ? undefined : Date.now(); + if (charm.disabledAt) { + charm.disabledAt = undefined; + charm.status = "Initializing..."; + } else { + charm.disabledAt = Date.now(); + charm.status = DISABLED_VIA_UI; + } +}); + +// Minimal "moment" style formatting to get a string +// representation of an (older) date relative to now, +// e.g. "5 seconds ago". +// * Renders "in the future" for all times in the future, +// we don't currently need e.g. "5 seconds from now". +// * Disregard plural units, "1 minutes ago" is fine. +// * Timezones are hard. Could maybe render "0 years ago". +function fromNow(then: Date): string { + const now = new Date(); + const diffSeconds = Math.floor((now.getTime() - then.getTime()) / 1000); + if (diffSeconds < 0) return "in the future"; + if (diffSeconds === 0) return "now"; + if (diffSeconds < 60) return `${diffSeconds} seconds ago`; + + const diffMinutes = Math.floor(diffSeconds / 60); + if (diffMinutes < 60) return `${diffMinutes} minutes ago`; + + const diffHours = Math.floor(diffMinutes / 60); + if (diffHours < 24) return `${diffHours} hours ago`; + + const diffDays = Math.floor(diffHours / 24); + if (diffDays < 365) return `${Math.floor(diffDays)} days ago`; + + return `${Math.floor(then.getFullYear() - now.getFullYear())} `; +} + +function StatusIcon( + { status, disabledAt }: { status?: string; disabledAt?: number }, +) { + let color; + let title = status; + const SUCCESS = `#4CAF50`; + const UNKNOWN = `#FFC107`; + const DISABLED = `#9E9E9E`; + const FAILURE = `#F44336`; + if (!disabledAt) { + if (status === "Success") { + color = SUCCESS; + title = "Running"; + } else { + color = UNKNOWN; + } + } else { + if (status === DISABLED_VIA_UI) { + color = DISABLED; + } else { + color = FAILURE; + } + } + return ( +
+
+ ); +} + +const BGCharmRow = lift(( + { charm, charms }: { charm: BGCharmEntry; charms: BGCharmEntries }, +) => { + const { integration, createdAt, updatedAt, disabledAt, lastRun, status } = + charm; + const space = charm.space.slice(-4); + const charmId = charm.charmId.slice(-4); + const name = `#${space}/#${charmId}`; + + const createdAtDate = new Date(createdAt); + const updatedAtDate = new Date(updatedAt); + const lastRunDate = lastRun ? new Date(lastRun) : null; + const isSuccessful = status === "Success"; + const statusDisplay = isSuccessful ? "" : status; + const details = `Created ${ + fromNow(createdAtDate) + } (${createdAtDate.toLocaleString()}) +Updated ${fromNow(updatedAtDate)} (${updatedAtDate.toLocaleString()}) +Last run ${lastRunDate ? fromNow(lastRunDate) : "never"} ${ + lastRunDate ? `(${lastRunDate.toLocaleString()})` : "" + }`; + + return ( +
+
+ +
+
+ {name} + {integration} +
+
{statusDisplay}
+
+ +
+
+ ); }); +const css = ` +.bg-charm-container { + display: flex; + flex-direction: column; +} +.bg-charm-container .ellipsis { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.bg-charm-container button { + cursor: pointer; +} +.bg-charm-row { + display: flex; + flex-direction: row; + height: 50px; + align-items: center; +} +.bg-charm-row > * { + padding: 10px; +} +.bg-charm-row .toggle-button, .bg-charm-row .delete { + flex: 0; + display: flex; +} +.bg-charm-row .name { + width: 250px; + cursor: help; +} +.bg-charm-row .integration { + color: #aaa; + padding-left: 3px; +} +.bg-charm-row .status { + flex: 1; +} +.bg-charm-container .delete button { + border: 1px solid black; +} +`; + export default recipe( InputSchema, ResultSchema, @@ -79,87 +240,20 @@ export default recipe( console.log("bg charm list:", charms); }); return { - [NAME]: "BG Updater Management", + [NAME]: "BG Updater Management New", [UI]: ( - - - - - - - - - - - - - - - - - {charms.map((charm) => ( - - - - - - - - - - - - - ))} - -
SpaceCharm IDIntegrationCreated AtUpdated AtLast RunStatusDisabledDelete
- #{derive(charm, (charm) => charm.space.slice(-4))} - - #{derive(charm, (charm) => charm.charmId.slice(-4))} - {charm.integration} - {derive( - charm, - (charm) => new Date(charm.createdAt).toLocaleString(), - )} - - {derive( - charm, - (charm) => new Date(charm.updatedAt).toLocaleString(), - )} - - {derive( - charm, - (charm) => - charm.lastRun - ? new Date(charm.lastRun).toLocaleString() - : "never", - )} - {charm.status} - {derive( - charm, - (charm) => - charm.disabledAt - ? new Date(charm.disabledAt).toLocaleString() - : "enabled", - )}  - - - -
- +
+ +
+ {charms.map((charm) => ( + + + ))} +
+
), charms, }; diff --git a/runner/src/index.ts b/runner/src/index.ts index 486d3bc73..e70c9ed2f 100644 --- a/runner/src/index.ts +++ b/runner/src/index.ts @@ -49,7 +49,7 @@ export { } from "./recipe-map.ts"; // export { addSchema, getSchema, getSchemaId } from "./schema-map.ts"; export { type AddCancel, type Cancel, noOp, useCancelGroup } from "./cancel.ts"; -export { storage } from "./storage.ts"; +export { type Storage, storage } from "./storage.ts"; export { setBobbyServerUrl, syncRecipeBlobby } from "./recipe-sync.ts"; // export { saveSchema, syncSchemaBlobby } from "./schema-sync.ts"; export { diff --git a/toolshed/routes/integrations/google-oauth/google-oauth.handlers.ts b/toolshed/routes/integrations/google-oauth/google-oauth.handlers.ts index 1ca188a69..b65c6e2d0 100644 --- a/toolshed/routes/integrations/google-oauth/google-oauth.handlers.ts +++ b/toolshed/routes/integrations/google-oauth/google-oauth.handlers.ts @@ -10,7 +10,6 @@ import { type AuthData, type CallbackResult, clearAuthData, - codeVerifiers, createBackgroundIntegrationErrorResponse, createBackgroundIntegrationSuccessResponse, createCallbackResponse, @@ -25,9 +24,8 @@ import { getBaseUrl, persistTokens, } from "./google-oauth.utils.ts"; -import { addOrUpdateBGCharm } from "@commontools/utils"; - -import { type CellLink } from "@commontools/runner"; +import { setBGCharm } from "@commontools/background-charm"; +import { type CellLink, storage } from "@commontools/runner"; /** * Google OAuth Login Handler @@ -195,10 +193,11 @@ export const callback: AppRouteHandler = async (c) => { "Adding Google integration charm to Gmail integrations", ); - await addOrUpdateBGCharm({ + await setBGCharm({ space, charmId: integrationCharmId, integration: "google", + storage, }); } else { logger.warn( @@ -357,10 +356,11 @@ export const backgroundIntegration: AppRouteHandler< try { const payload = await c.req.json(); - await addOrUpdateBGCharm({ + await setBGCharm({ space: payload.space, charmId: payload.charmId, integration: payload.integration, + storage, }); return createBackgroundIntegrationSuccessResponse(c, "success"); diff --git a/utils/src/index.ts b/utils/src/index.ts index 6634cb738..3265e0c05 100644 --- a/utils/src/index.ts +++ b/utils/src/index.ts @@ -1,5 +1,4 @@ export * from "./defer.ts"; export * from "./env.ts"; export * from "./isObj.ts"; -export * from "./updaters.ts"; export * from "./sleep.ts"; diff --git a/utils/src/updaters.ts b/utils/src/updaters.ts deleted file mode 100644 index e8f83a466..000000000 --- a/utils/src/updaters.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Cell, getCell, storage } from "@commontools/runner"; -import { ID, JSONSchema, Schema } from "@commontools/builder"; - -// This is the derived space id for toolshed-system -export const BG_SYSTEM_SPACE_ID = - "did:key:z6Mkfuw7h6jDwqVb6wimYGys14JFcyTem4Kqvdj9DjpFhY88"; -export const BG_CELL_CAUSE = "bgUpdater-2025-03-18"; - -export const CharmEntrySchema = { - type: "object", - properties: { - space: { type: "string" }, - charmId: { type: "string" }, - integration: { type: "string" }, - createdAt: { type: "number" }, - updatedAt: { type: "number" }, - disabledAt: { type: "number", default: 0 }, - lastRun: { type: "number", default: 0 }, - status: { type: "string", default: "" }, - }, - required: [ - "space", - "charmId", - "integration", - "createdAt", - "updatedAt", - "lastRun", - "status", - ], -} as const satisfies JSONSchema; -export type BGCharmEntry = Schema; - -export const bgUpdaterCharmsSchema = { - type: "array", - items: CharmEntrySchema, - default: [], -} as const satisfies JSONSchema; - -export type BGUpdaterCharmsSchema = Schema; - -export async function addOrUpdateBGCharm({ - space, - charmId, - integration, -}: { - space: string; - charmId: string; - integration: string; -}): Promise { - const charmsCell = await getBGUpdaterCharmsCell(); - - console.log( - "charmsCell", - JSON.stringify(charmsCell.getAsCellLink(), null, 2), - ); - - const charms = charmsCell.get() || []; - - const existingCharmIndex = charms.findIndex( - (charm: Cell) => - charm.get().space === space && charm.get().charmId === charmId, - ); - - if (existingCharmIndex === -1) { - console.log("Adding charm to BGUpdater charms cell"); - charmsCell.push({ - [ID]: `${space}/${charmId}`, - space, - charmId, - integration, - createdAt: Date.now(), - updatedAt: Date.now(), - disabledAt: undefined, - lastRun: 0, - status: "Initializing", - } as unknown as Cell); - - // Ensure changes are synced - await storage.synced(); - - return true; - } else { - console.log("Charm already exists in BGUpdater charms cell, re-enabling"); - const existingCharm = charms[existingCharmIndex]; - existingCharm.update({ - disabledAt: 0, - updatedAt: Date.now(), - status: "Re-initializing", - }); - - await storage.synced(); - - return false; - } -} - -export async function getBGUpdaterCharmsCell(): Promise< - Cell[]> -> { - if (!storage.hasSigner()) { - throw new Error("Storage has no signer"); - } - - if (!storage.hasRemoteStorage()) { - throw new Error("Storage has no remote storage"); - } - const schema = { - type: "array", - items: { - ...CharmEntrySchema, - asCell: true, - }, - default: [], - } as const satisfies JSONSchema; - - const charmsCell = getCell(BG_SYSTEM_SPACE_ID, BG_CELL_CAUSE, schema); - - // Ensure the cell is synced - // FIXME(ja): does True do the right thing here? Does this mean: I REALLY REALLY - // INSIST THAT YOU HAVE THIS CELL ON THE SERVER! - await storage.syncCell(charmsCell, true); - await storage.synced(); - - return charmsCell; -}