chore: background charm service as a library, iterate on bgAdmin recipe

Author: jsantell

background-charm-service/cast-admin.ts

@@ -6,13 +6,13 @@ import {
   setBobbyServerUrl,
   storage,
 } from "@commontools/runner";
-import { type DID, Identity } from "@commontools/identity";
+import { type DID } from "@commontools/identity";
 import { createAdminSession } from "@commontools/identity";
 import {
   BG_CELL_CAUSE,
   BG_SYSTEM_SPACE_ID,
-  bgUpdaterCharmsSchema,
-} from "@commontools/utils";
+  BGCharmEntriesSchema,
+} from "./src/schema.ts";
 import { getIdentity } from "./src/utils.ts";
 
 const { recipePath, quit } = parseArgs(
@@ -79,7 +79,7 @@ async function castRecipe() {
     const targetCell = getCell(
       spaceId as DID,
       cause,
-      bgUpdaterCharmsSchema,
+      BGCharmEntriesSchema,
     );
 
     // Ensure the cell is synced

background-charm-service/deno.json

@@ -1,4 +1,5 @@
 {
+  "name": "@commontools/background-charm",
   "tasks": {
     "start": "deno run -A --unstable-worker-options src/main.ts",
     "help": "deno run -A src/main.ts --help",
@@ -8,5 +9,9 @@
     "lint": "deno lint",
     "fmt": "deno fmt"
   },
-  "imports": {}
+  "imports": {},
+  "exports": {
+    ".": "./src/lib.ts",
+    "./schema": "./src/schema.ts"
+  }
 }

background-charm-service/src/lib.ts

@@ -0,0 +1,11 @@
+export {
+  BackgroundCharmService,
+  type BackgroundCharmServiceOptions,
+} from "./service.ts";
+export {
+  BG_CELL_CAUSE,
+  BG_SYSTEM_SPACE_ID,
+  type BGCharmEntry,
+  BGCharmEntrySchema,
+} from "./schema.ts";
+export { setBGCharm } from "./utils.ts";

background-charm-service/src/main.ts

@@ -1,3 +1,4 @@
+import { storage } from "@commontools/runner";
 import { BackgroundCharmService } from "./service.ts";
 import { getIdentity, log } from "./utils.ts";
 import { env } from "./env.ts";
@@ -6,6 +7,7 @@ const identity = await getIdentity(env.IDENTITY, env.OPERATOR_PASS);
 const service = new BackgroundCharmService({
   identity,
   toolshedUrl: env.TOOLSHED_API_URL,
+  storage,
 });
 
 const shutdown = () => {

background-charm-service/src/schema.ts

@@ -0,0 +1,37 @@
+import { type JSONSchema, type Schema } from "@commontools/builder";
+
+// This is the derived space id for toolshed-system
+export const BG_SYSTEM_SPACE_ID =
+  "did:key:z6Mkfuw7h6jDwqVb6wimYGys14JFcyTem4Kqvdj9DjpFhY88";
+export const BG_CELL_CAUSE = "bgUpdater-2025-03-18";
+export const BGCharmEntrySchema = {
+  type: "object",
+  properties: {
+    space: { type: "string" },
+    charmId: { type: "string" },
+    integration: { type: "string" },
+    createdAt: { type: "number" },
+    updatedAt: { type: "number" },
+    disabledAt: { type: "number", default: 0 },
+    lastRun: { type: "number", default: 0 },
+    status: { type: "string", default: "" },
+  },
+  required: [
+    "space",
+    "charmId",
+    "integration",
+    "createdAt",
+    "updatedAt",
+    "lastRun",
+    "status",
+  ],
+} as const as JSONSchema;
+export type BGCharmEntry = Schema<typeof BGCharmEntrySchema>;
+
+export const BGCharmEntriesSchema = {
+  type: "array",
+  items: BGCharmEntrySchema,
+  default: [],
+} as const satisfies JSONSchema;
+
+export type BGCharmEntries = Schema<typeof BGCharmEntriesSchema>;

background-charm-service/src/service.ts

@@ -1,13 +1,20 @@
 import { Identity } from "@commontools/identity";
-import { type Cell, storage } from "@commontools/runner";
-import { type BGCharmEntry, getBGUpdaterCharmsCell } from "@commontools/utils";
-import { log } from "./utils.ts";
+import { type Cell, type Storage } from "@commontools/runner";
+import {
+  BG_CELL_CAUSE,
+  BG_SYSTEM_SPACE_ID,
+  type BGCharmEntry,
+} from "./schema.ts";
+import { getBGCharms, log } from "./utils.ts";
 import { SpaceManager } from "./space-manager.ts";
 import { useCancelGroup } from "@commontools/runner";
 
 export interface BackgroundCharmServiceOptions {
   identity: Identity;
   toolshedUrl: string;
+  storage: Storage;
+  bgSpace?: string;
+  bgCause?: string;
 }
 
 export class BackgroundCharmService {
@@ -16,18 +23,24 @@ export class BackgroundCharmService {
   private charmSchedulers: Map<string, SpaceManager> = new Map();
   private identity: Identity;
   private toolshedUrl: string;
+  private storage: Storage;
+  private bgSpace: string;
+  private bgCause: string;
 
   constructor(options: BackgroundCharmServiceOptions) {
     this.identity = options.identity;
     this.toolshedUrl = options.toolshedUrl;
+    this.storage = options.storage;
+    this.bgSpace = options.bgSpace ?? BG_SYSTEM_SPACE_ID;
+    this.bgCause = options.bgCause ?? BG_CELL_CAUSE;
   }
 
   async initialize() {
-    storage.setRemoteStorage(new URL(this.toolshedUrl));
-    storage.setSigner(this.identity);
-    this.charmsCell = await getBGUpdaterCharmsCell();
-    await storage.syncCell(this.charmsCell, true);
-    await storage.synced();
+    this.storage.setRemoteStorage(new URL(this.toolshedUrl));
+    this.storage.setSigner(this.identity);
+    this.charmsCell = await getBGCharms({ bgSpace: this.bgSpace, bgCause: this.bgCause, storage: this.storage });
+    await this.storage.syncCell(this.charmsCell, true);
+    await this.storage.synced();
 
     if (this.isRunning) {
       log("Service is already running");

background-charm-service/src/space-manager.ts

@@ -1,11 +1,12 @@
-import { BGCharmEntry, sleep } from "@commontools/utils";
+import { sleep } from "@commontools/utils";
 import { Cell } from "@commontools/runner";
+import { type Cancel, useCancelGroup } from "@commontools/runner";
 import {
   WorkerController,
   WorkerControllerErrorEvent,
   type WorkerOptions,
 } from "./worker-controller.ts";
-import { type Cancel, useCancelGroup } from "@commontools/runner";
+import { type BGCharmEntry } from "./schema.ts";
 
 export interface CharmSchedulerOptions extends WorkerOptions {
   pollingIntervalMs?: number;

background-charm-service/src/utils.ts

@@ -1,6 +1,18 @@
 import { Charm } from "@commontools/charm";
-import { Cell, getEntityId } from "@commontools/runner";
+import {
+  type Cell,
+  getCell,
+  getEntityId,
+  type Storage,
+} from "@commontools/runner";
 import { Identity, type IdentityCreateConfig } from "@commontools/identity";
+import { ID, type JSONSchema } from "@commontools/builder";
+import {
+  BG_CELL_CAUSE,
+  BG_SYSTEM_SPACE_ID,
+  type BGCharmEntry,
+  BGCharmEntrySchema,
+} from "./schema.ts";
 
 /**
  * Custom logger that includes timestamp and optionally charm ID
@@ -87,3 +99,108 @@ export async function getIdentity(
   }
   throw new Error("No IDENTITY or OPERATOR_PASS environemnt set.");
 }
+
+export async function setBGCharm({
+  space,
+  charmId,
+  integration,
+  storage,
+  bgSpace,
+  bgCause,
+}: {
+  space: string;
+  charmId: string;
+  integration: string;
+  storage: Storage;
+  bgSpace?: string;
+  bgCause?: string;
+}): Promise<boolean> {
+  const charmsCell = await getBGCharms({
+    bgSpace,
+    bgCause,
+    storage,
+  });
+
+  console.log(
+    "charmsCell",
+    JSON.stringify(charmsCell.getAsCellLink(), null, 2),
+  );
+
+  const charms = charmsCell.get() || [];
+
+  const existingCharmIndex = charms.findIndex(
+    (charm: Cell<BGCharmEntry>) =>
+      charm.get().space === space && charm.get().charmId === charmId,
+  );
+
+  if (existingCharmIndex === -1) {
+    console.log("Adding charm to BGUpdater charms cell");
+    charmsCell.push({
+      [ID]: `${space}/${charmId}`,
+      space,
+      charmId,
+      integration,
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+      disabledAt: undefined,
+      lastRun: 0,
+      status: "Initializing",
+    } as unknown as Cell<BGCharmEntry>);
+
+    // Ensure changes are synced
+    await storage.synced();
+
+    return true;
+  } else {
+    console.log("Charm already exists in BGUpdater charms cell, re-enabling");
+    const existingCharm = charms[existingCharmIndex];
+    existingCharm.update({
+      disabledAt: 0,
+      updatedAt: Date.now(),
+      status: "Re-initializing",
+    });
+
+    await storage.synced();
+
+    return false;
+  }
+}
+
+export async function getBGCharms(
+  { bgSpace, bgCause, storage }: {
+    bgSpace?: string;
+    bgCause?: string;
+    storage: Storage;
+  },
+): Promise<
+  Cell<Cell<BGCharmEntry>[]>
+> {
+  bgSpace = bgSpace ?? BG_SYSTEM_SPACE_ID;
+  bgCause = bgCause ?? BG_CELL_CAUSE;
+
+  if (!storage.hasSigner()) {
+    throw new Error("Storage has no signer");
+  }
+
+  if (!storage.hasRemoteStorage()) {
+    throw new Error("Storage has no remote storage");
+  }
+  const schema = {
+    type: "array",
+    items: {
+      ...BGCharmEntrySchema,
+      asCell: true,
+    },
+    default: [],
+  } as const satisfies JSONSchema;
+
+  const charmsCell = getCell(bgSpace, bgCause, schema);
+
+  // Ensure the cell is synced
+  // FIXME(ja): does True do the right thing here? Does this mean: I REALLY REALLY
+  // INSIST THAT YOU HAVE THIS CELL ON THE SERVER!
+  await storage.syncCell(charmsCell, true);
+  await storage.synced();
+
+  return charmsCell;
+}

background-charm-service/src/worker-controller.ts

@@ -1,12 +1,11 @@
-import { BGCharmEntry } from "@commontools/utils";
+import { BGCharmEntry } from "./schema.ts";
 import { Cell } from "@commontools/runner";
 import { Identity } from "@commontools/identity";
 import { defer, type Deferred } from "@commontools/utils/defer";
 import {
   isWorkerIPCResponse,
   WorkerIPCMessageType,
   WorkerIPCRequest,
-  WorkerIPCResponse,
 } from "./worker-ipc.ts";
 
 const DEFAULT_TASK_TIMEOUT = 60_000;