Remove legacy remote storage provider (#1079)

* chore: commit wip with working serverside traverse overhaul

* chore: finished updating tests

* chore: commit wip while debugging lack of schema

* fix: wrong charm title (#1023)

* add gemini-flash-lite (#1035)

support flash-lite

* chore: Do not attempt to serve static directories (#1037)

* Only re-render CharmList when connection status actually changes (#1038)

* Only re-render CharmList when connection status actually changes

* Update types

* track loads differently based on their schema
import as FactModule instead of Fact to avoid collision
expose the schema associated with the facts from QueryView
make schemaContext optional in SchemaPathSelector, since I don't want to traverse commit logs
Change default storage to cached

* Llm feedback (#1036)

User Feedback UI
 - Implemented FeedbackActions and FeedbackDialog components in Jumble to allow users to submit positive or negative feedback on generated content.
 - Added interactive thumbs-up/down buttons with feedback form submission workflow.

Trace Span ID Management
 - Added functions (setLastTraceSpanID, getLastTraceSpanID) to manage trace IDs in the builder environment.
 - Updated builder exports to expose new trace ID functions.

Integration of Trace ID in LLM Client
 - Modified LLMClient to capture x-ct-llm-trace-id from response headers and store it via setLastTraceSpanID.

Backend Feedback Handling
 - Extended toolshed API (/api/ai/llm/feedback) to accept user feedback and relay it to Phoenix observability backend.

Instrumentation and Span Filtering
 - Enhanced OpenTelemetry instrumentation to include specific spans relevant to LLM inference requests for better observability.

* fix schema query with commit+json

* chore: commit wip

* chore: env variable to toggle schema sub

* changed to have subscribing queries not delete themselves with the task/return
changed so the server sends each response to each subscriber, even though there may be duplicates. a schema subscription on the client needs to know its updated list of entities
added lots of debugging to try to track down this stall issue

* we should be able to inspect both the graph/query and the subscribe for the combo

* reverted incomplete change to docIsSyncing that caused me to stall on load

* fix blunder with pulling value from fact
cleaned up logging and comments

* change todo comment for linter

* remove incorrect comment in test

* fix the other half of the value blunder

* Remove unused import

* Added start-ci task for running integration tests that makes BroadcastChannel available.

* Change cache to only use the BroadcastChannel if it's available.

* better import of isObj

* just inline the isObj test

* Remove the BroadcastChannel, since we're not really using it.

* Remove remote provider
Needs more work, since we fail tests which try to use indexdb in deno.

* correct merge error

---------

Co-authored-by: Irakli Gozalishvili <contact@gozala.io>
Co-authored-by: Jesse Andrews <anotherjesse@gmail.com>
Co-authored-by: Jordan Santell <jsantell@users.noreply.github.com>
Co-authored-by: Ben Follington <5009316+bfollington@users.noreply.github.com>
Co-authored-by: Jake Dahn <jake@common.tools>

Author: 6 people

cli/memory_demo.ts

@@ -1,5 +1,5 @@
-import { RemoteStorageProvider } from "../runner/src/storage/remote.ts";
 import { StorageProvider } from "../runner/src/storage/base.ts";
+import { Provider as CachedStorageProvider } from "../runner/src/storage/cache.ts";
 import { EntityId } from "../runner/src/doc-map.ts";
 import { Identity } from "../identity/src/index.ts";
 
@@ -34,7 +34,7 @@ function entity_id(i: number): EntityId {
 async function main() {
   const authority = await Identity.fromPassphrase("ellyse5");
 
-  const storageProvider: StorageProvider = new RemoteStorageProvider({
+  const storageProvider: StorageProvider = new CachedStorageProvider({
     id: import.meta.url,
     address: new URL("/api/storage/memory", BASE_URL),
     space: authority.did(),

runner/src/storage.ts

@@ -8,8 +8,11 @@ import {
 } from "./query-result-proxy.ts";
 import { idle } from "./scheduler.ts";
 import { isStatic, markAsStatic } from "@commontools/builder";
-import { StorageProvider, StorageValue } from "./storage/base.ts";
-import { RemoteStorageProvider } from "./storage/remote.ts";
+import {
+  BaseStorageProvider,
+  StorageProvider,
+  StorageValue,
+} from "./storage/base.ts";
 import {
   Provider as CachedStorageProvider,
   RemoteStorageProviderSettings,
@@ -22,11 +25,7 @@ import { sleep } from "@commontools/utils/sleep";
 import { TransactionResult } from "@commontools/memory";
 import { refer } from "@commontools/memory/reference";
 import { defer } from "@commontools/utils";
-import {
-  Entity,
-  SchemaContext,
-  SchemaNone,
-} from "@commontools/memory/interface";
+import { SchemaContext, SchemaNone } from "@commontools/memory/interface";
 
 export function log(fn: () => any[]) {
   debug(() => {
@@ -319,18 +318,7 @@ class StorageImpl implements Storage {
         ? "volatile"
         : ((import.meta as any).env?.VITE_STORAGE_TYPE ?? "cached");
 
-      if (type === "remote") {
-        if (!this.remoteStorageUrl) {
-          throw new Error("No remote storage URL set");
-        }
-
-        provider = new RemoteStorageProvider({
-          id: this.id,
-          address: new URL("/api/storage/memory", this.remoteStorageUrl!),
-          space: space as `did:${string}:${string}`,
-          as: this.signer,
-        });
-      } else if (type === "volatile") {
+      if (type === "volatile") {
         provider = new VolatileStorageProvider(space);
       } else if (type === "cached") {
         if (!this.remoteStorageUrl) {
@@ -774,7 +762,7 @@ class StorageImpl implements Storage {
           }
 
           const conflictJobIndex = jobs.findIndex((job) =>
-            RemoteStorageProvider.toEntity(job.entityId) === conflict.of
+            BaseStorageProvider.toEntity(job.entityId) === conflict.of
           );
 
           if (conflictJobIndex === -1) {

runner/src/storage/base.ts

@@ -1,5 +1,5 @@
 import type { Cancel, EntityId } from "@commontools/runner";
-import type { Result, Unit } from "@commontools/memory/interface";
+import type { Entity, Result, Unit } from "@commontools/memory/interface";
 import { log } from "../storage.ts";
 import { SchemaContext } from "@commontools/memory/interface";
 
@@ -144,4 +144,21 @@ export abstract class BaseStorageProvider implements StorageProvider {
   abstract destroy(): Promise<void>;
 
   abstract getReplica(): string | undefined;
+
+  static toEntity(source: EntityId): Entity {
+    if (typeof source["/"] === "string") {
+      return `of:${source["/"]}`;
+    } else if (source.toJSON) {
+      return `of:${source.toJSON()["/"]}`;
+    } else {
+      throw Object.assign(
+        new TypeError(
+          `💣 Got entity ID that is neither merkle reference nor {'/'}`,
+        ),
+        {
+          cause: source,
+        },
+      );
+    }
+  }
 }

runner/src/storage/cache.ts

@@ -18,7 +18,11 @@ import type {
   Unit,
 } from "@commontools/memory/interface";
 import type { Cancel, EntityId } from "@commontools/runner";
-import { type StorageProvider, type StorageValue } from "./base.ts";
+import {
+  BaseStorageProvider,
+  type StorageProvider,
+  type StorageValue,
+} from "./base.ts";
 import type { MemorySpaceSession } from "@commontools/memory/consumer";
 import { assert, retract, unclaimed } from "@commontools/memory/fact";
 import { fromString, refer } from "merkle-reference";
@@ -789,29 +793,12 @@ export class Provider implements StorageProvider {
     }
   }
 
-  static toEntity(source: EntityId): Entity {
-    if (typeof source["/"] === "string") {
-      return `of:${source["/"]}`;
-    } else if (source.toJSON) {
-      return `of:${source.toJSON()["/"]}`;
-    } else {
-      throw Object.assign(
-        new TypeError(
-          `💣 Got entity ID that is neither merkle reference nor {'/'}`,
-        ),
-        {
-          cause: source,
-        },
-      );
-    }
-  }
-
   sink<T = any>(
     entityId: EntityId,
     callback: (value: StorageValue<T>) => void,
   ): Cancel {
     const { the } = this;
-    const of = Provider.toEntity(entityId);
+    const of = BaseStorageProvider.toEntity(entityId);
     const { workspace } = this;
     const address = { the, of };
     const subscriber = (revision?: Revision<State>) => {
@@ -836,14 +823,14 @@ export class Provider implements StorageProvider {
     schemaContext?: SchemaContext,
   ) {
     const { the } = this;
-    const of = Provider.toEntity(entityId);
+    const of = BaseStorageProvider.toEntity(entityId);
     return this.workspace.load([[{ the, of }, schemaContext]]);
   }
 
   get<T = any>(entityId: EntityId): StorageValue<T> | undefined {
     const entity = this.workspace.get({
       the: this.the,
-      of: Provider.toEntity(entityId),
+      of: BaseStorageProvider.toEntity(entityId),
     });
 
     return entity?.is as StorageValue<T> | undefined;
@@ -865,7 +852,7 @@ export class Provider implements StorageProvider {
 
     const changes = [];
     for (const { entityId, value } of batch) {
-      const of = Provider.toEntity(entityId);
+      const of = BaseStorageProvider.toEntity(entityId);
       const content = JSON.stringify(value);
 
       const current = workspace.get({ the, of });