Skip to content

feat(runner): createDataCellURI with circular reference detection and relative link rewriting #1917

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 17, 2025
Merged

feat(runner): createDataCellURI with circular reference detection and relative link rewriting #1917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e208e48
feat(runner): inline data URIs before writing to storage
seefeldb Oct 17, 2025
851987d
fix linter error
seefeldb Oct 17, 2025
cadd815
feat(runner): export `createDataCellURI` with circular detection and …
seefeldb Oct 17, 2025
92edcd2
fix(runner): track only current recursion path in cycle detection
seefeldb Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions packages/runner/src/link-utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,52 @@ export function findAndInlineDataURILinks(value: any): any {
}
}

// Helper to create data URIs for testing
export function createDataCellURI(
data: any,
base?: Cell | NormalizedLink,
): string {
const baseId = isCell(base) ? base.getAsNormalizedFullLink().id : base?.id;

function traverseAndAddBaseIdToRelativeLinks(
value: any,
seen: Set<any>,
): any {
if (!isRecord(value)) return value;
if (seen.has(value)) {
throw new Error(`Cycle detected when creating data URI`);
}
seen.add(value);
try {
if (isAnyCellLink(value)) {
const link = parseLink(value);
if (!link.id) {
return createSigilLinkFromParsedLink({ ...link, id: baseId });
} else {
return value;
}
} else if (Array.isArray(value)) {
return value.map((item) =>
traverseAndAddBaseIdToRelativeLinks(item, seen)
);
} else { // isObject
return Object.fromEntries(
Object.entries(value).map((
[key, value],
) => [key, traverseAndAddBaseIdToRelativeLinks(value, seen)]),
);
}
} finally {
seen.delete(value);
}
}
const json = JSON.stringify({
value: traverseAndAddBaseIdToRelativeLinks(data, new Set()),
});
const base64 = btoa(json);
return `data:application/json;charset=utf-8;base64,${base64}`;
}

/**
* Traverse schema and remove all asCell and asStream flags.
*/
Expand Down
60 changes: 28 additions & 32 deletions packages/runner/test/data-uri-inlining.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,15 @@ import { Identity } from "@commontools/identity";
import { StorageManager } from "@commontools/runner/storage/cache.deno";
import { Runtime } from "../src/runtime.ts";
import type { IExtendedStorageTransaction } from "../src/storage/interface.ts";
import { findAndInlineDataURILinks } from "../src/link-utils.ts";
import {
createDataCellURI,
findAndInlineDataURILinks,
} from "../src/link-utils.ts";
import { LINK_V1_TAG } from "../src/sigil-types.ts";

const signer = await Identity.fromPassphrase("test operator");
const space = signer.did();

// Helper to create data URIs for testing
function createDataURI(data: any): string {
const json = JSON.stringify({ value: data });
const base64 = btoa(json);
return `data:application/json;charset=utf-8;base64,${base64}`;
}

describe("data URI inlining", () => {
let storageManager: ReturnType<typeof StorageManager.emulate>;
let runtime: Runtime;
Expand All @@ -39,7 +35,7 @@ describe("data URI inlining", () => {

describe("findAndInlineDataURILinks", () => {
it("should inline simple data URI links", () => {
const dataURI = createDataURI("test data");
const dataURI = createDataCellURI("test data");
const link = {
"/": {
[LINK_V1_TAG]: {
Expand All @@ -54,7 +50,7 @@ describe("data URI inlining", () => {
});

it("should inline data URI links with paths", () => {
const dataURI = createDataURI({ nested: { value: 42 } });
const dataURI = createDataCellURI({ nested: { value: 42 } });
const link = {
"/": {
[LINK_V1_TAG]: {
Expand All @@ -69,8 +65,8 @@ describe("data URI inlining", () => {
});

it("should inline data URI links in arrays", () => {
const dataURI1 = createDataURI("first");
const dataURI2 = createDataURI("second");
const dataURI1 = createDataCellURI("first");
const dataURI2 = createDataCellURI("second");

const array = [
{
Expand All @@ -96,7 +92,7 @@ describe("data URI inlining", () => {
});

it("should inline data URI links in objects", () => {
const dataURI = createDataURI("nested value");
const dataURI = createDataCellURI("nested value");
const obj = {
key1: "regular value",
key2: {
Expand All @@ -120,7 +116,7 @@ describe("data URI inlining", () => {
const innerCell = runtime.getCell(space, "inner", undefined, tx);
innerCell.set({ value: "inner data" });

const dataURI = createDataURI(innerCell.getAsLink());
const dataURI = createDataCellURI(innerCell.getAsLink());
const link = {
"/": {
[LINK_V1_TAG]: {
Expand All @@ -145,7 +141,7 @@ describe("data URI inlining", () => {
const innerCell = runtime.getCell(space, "inner", undefined, tx);
innerCell.set({ nested: { value: "inner data" } });

const dataURI = createDataURI(innerCell.getAsLink());
const dataURI = createDataCellURI(innerCell.getAsLink());
const link = {
"/": {
[LINK_V1_TAG]: {
Expand All @@ -166,7 +162,7 @@ describe("data URI inlining", () => {
});

it("should return undefined for data URIs with invalid paths", () => {
const dataURI = createDataURI({ a: 1 });
const dataURI = createDataCellURI({ a: 1 });
const link = {
"/": {
[LINK_V1_TAG]: {
Expand Down Expand Up @@ -196,7 +192,7 @@ describe("data URI inlining", () => {
});

it("should deeply traverse nested structures", () => {
const dataURI = createDataURI("deep value");
const dataURI = createDataCellURI("deep value");
const complex = {
level1: {
level2: [
Expand All @@ -219,7 +215,7 @@ describe("data URI inlining", () => {
});

it("should handle data URIs with schema", () => {
const dataURI = createDataURI(42);
const dataURI = createDataCellURI(42);
const link = {
"/": {
[LINK_V1_TAG]: {
Expand All @@ -239,7 +235,7 @@ describe("data URI inlining", () => {
const innerCell = runtime.getCell(space, "inner", undefined, tx);
innerCell.set({ nested: { value: "data" } });

const dataURI = createDataURI(innerCell.getAsLink({
const dataURI = createDataCellURI(innerCell.getAsLink({
includeSchema: true,
}));
const link = {
Expand All @@ -266,7 +262,7 @@ describe("data URI inlining", () => {

describe("setRaw with data URI inlining", () => {
it("should inline data URIs when using setRaw", () => {
const dataURI = createDataURI("inlined value");
const dataURI = createDataCellURI("inlined value");
const targetCell = runtime.getCell(space, "target", undefined, tx);

const link = {
Expand All @@ -283,8 +279,8 @@ describe("data URI inlining", () => {
});

it("should inline nested data URIs in objects", () => {
const dataURI1 = createDataURI("value1");
const dataURI2 = createDataURI("value2");
const dataURI1 = createDataCellURI("value1");
const dataURI2 = createDataCellURI("value2");
const targetCell = runtime.getCell(space, "target", undefined, tx);

targetCell.setRaw({
Expand Down Expand Up @@ -313,7 +309,7 @@ describe("data URI inlining", () => {
});

it("should inline data URIs in arrays", () => {
const dataURI = createDataURI("array item");
const dataURI = createDataCellURI("array item");
const targetCell = runtime.getCell(space, "target", undefined, tx);

targetCell.setRaw([
Expand All @@ -334,7 +330,7 @@ describe("data URI inlining", () => {

describe("diffAndUpdate with data URI inlining", () => {
it("should inline data URIs during diffAndUpdate", () => {
const dataURI = createDataURI("updated value");
const dataURI = createDataCellURI("updated value");
const targetCell = runtime.getCell(space, "target", undefined, tx);
targetCell.set({ initial: "value" });

Expand All @@ -352,7 +348,7 @@ describe("data URI inlining", () => {
});

it("should handle data URIs with complex nested structures", () => {
const dataURI = createDataURI({
const dataURI = createDataCellURI({
nested: {
array: [1, 2, 3],
obj: { key: "value" },
Expand All @@ -379,7 +375,7 @@ describe("data URI inlining", () => {
});

it("should not write data URIs to storage", () => {
const dataURI = createDataURI("test");
const dataURI = createDataCellURI("test");
const targetCell = runtime.getCell(space, "target", undefined, tx);

const link = {
Expand Down Expand Up @@ -412,7 +408,7 @@ describe("data URI inlining", () => {
},
},
};
const dataURI = createDataURI({
const dataURI = createDataCellURI({
link: relativeLink,
other: { path: "success" },
});
Expand Down Expand Up @@ -488,7 +484,7 @@ describe("data URI inlining", () => {
};

// Embed the link in a data URI at some intermediate level
const dataURI = createDataURI({ intermediate: linkToOtherDoc });
const dataURI = createDataCellURI({ intermediate: linkToOtherDoc });

// Now create a link that goes through data URI, then through intermediate,
// and then further into the linked document beyond what data URI describes
Expand Down Expand Up @@ -540,7 +536,7 @@ describe("data URI inlining", () => {
},
},
};
const dataURI = createDataURI(relativeLink);
const dataURI = createDataCellURI(relativeLink);

// Link with additional path that goes beyond the relative link
const link = {
Expand Down Expand Up @@ -580,7 +576,7 @@ describe("data URI inlining", () => {
const linkToTarget = targetCell.getAsLink();

// Wrap it in a data URI
const dataURI1 = createDataURI({ wrapped: linkToTarget });
const dataURI1 = createDataCellURI({ wrapped: linkToTarget });

// Create a link to first data URI
const linkToDataURI1 = {
Expand All @@ -593,7 +589,7 @@ describe("data URI inlining", () => {
};

// Wrap that in another data URI
const dataURI2 = createDataURI({ doubleWrapped: linkToDataURI1 });
const dataURI2 = createDataCellURI({ doubleWrapped: linkToDataURI1 });

// Create final link
const finalLink = {
Expand Down Expand Up @@ -646,7 +642,7 @@ describe("data URI inlining", () => {
},
};

const dataURI = createDataURI(linkWithSchema);
const dataURI = createDataCellURI(linkWithSchema);

// Path extends into the linked document
const link = {
Expand Down
Loading