chore: Introduce CTHelpers in ts-transformers serving imported symbols.

As a part of the "pretransform" step, we replace the CTS enable
directive with a wildcard import for "commontools" module that is
used when injecting new functionality via transformations.

Author: jsantell

packages/runner/src/harness/engine.ts

@@ -22,6 +22,7 @@ import * as RuntimeModules from "./runtime-modules.ts";
 import { IRuntime } from "../runtime.ts";
 import * as merkleReference from "merkle-reference";
 import { StaticCache } from "@commontools/static";
+import { pretransformProgram } from "./pretransform.ts";
 
 const RUNTIME_ENGINE_CONSOLE_HOOK = "RUNTIME_ENGINE_CONSOLE_HOOK";
 const INJECTED_SCRIPT =
@@ -147,7 +148,7 @@ export class Engine extends EventTarget implements Harness {
   > {
     const id = options.identifier ?? computeId(program);
     const filename = options.filename ?? `${id}.js`;
-    const mappedProgram = mapPrefixProgramFiles(program, id);
+    const mappedProgram = pretransformProgram(program, id);
     const resolver = new EngineProgramResolver(
       mappedProgram,
       this.ctRuntime.staticCache,
@@ -250,32 +251,3 @@ function computeId(program: Program): string {
   ];
   return merkleReference.refer(source).toString();
 }
-
-// Adds `id` as a prefix to all files in the program.
-// Injects a new entry at root `/index.ts` to re-export
-// the entry contents because otherwise `typescript`
-// flattens the output, eliding the common prefix.
-function mapPrefixProgramFiles(program: RuntimeProgram, id: string): Program {
-  const main = program.main;
-  const exportNameds = `export * from "${prefix(main, id)}";`;
-  const exportDefault = `export { default } from "${prefix(main, id)}";`;
-  const hasDefault = !program.mainExport || program.mainExport === "default";
-  const files = [
-    ...program.files.map((source) => ({
-      name: prefix(source.name, id),
-      contents: source.contents,
-    })),
-    {
-      name: `/index.ts`,
-      contents: `${exportNameds}${hasDefault ? `\n${exportDefault}` : ""}`,
-    },
-  ];
-  return {
-    main: `/index.ts`,
-    files,
-  };
-}
-
-function prefix(filename: string, id: string): string {
-  return `/${id}${filename}`;
-}

packages/runner/src/harness/pretransform.ts

@@ -0,0 +1,60 @@
+import { transformCtDirective } from "@commontools/ts-transformers";
+import { RuntimeProgram } from "./types.ts";
+
+export function pretransformProgram(
+  program: RuntimeProgram,
+  id: string,
+): RuntimeProgram {
+  program = transformInjectHelperModule(program);
+  program = transformProgramWithPrefix(program, id);
+  return program;
+}
+
+// For each source file in the program, replace
+// a `/// <cts-enable />` directive line with an
+// internal import statement for use by the AST transformer
+// to provide access to helpers like `derive`, etc.
+export function transformInjectHelperModule(
+  program: RuntimeProgram,
+): RuntimeProgram {
+  return {
+    main: program.main,
+    files: program.files.map((source) => ({
+      name: source.name,
+      contents: transformCtDirective(source.contents),
+    })),
+    mainExport: program.mainExport,
+  };
+}
+
+// Adds `id` as a prefix to all files in the program.
+// Injects a new entry at root `/index.ts` to re-export
+// the entry contents because otherwise `typescript`
+// flattens the output, eliding the common prefix.
+export function transformProgramWithPrefix(
+  program: RuntimeProgram,
+  id: string,
+): RuntimeProgram {
+  const main = program.main;
+  const exportNameds = `export * from "${prefix(main, id)}";`;
+  const exportDefault = `export { default } from "${prefix(main, id)}";`;
+  const hasDefault = !program.mainExport || program.mainExport === "default";
+  const files = [
+    ...program.files.map((source) => ({
+      name: prefix(source.name, id),
+      contents: source.contents,
+    })),
+    {
+      name: `/index.ts`,
+      contents: `${exportNameds}${hasDefault ? `\n${exportDefault}` : ""}`,
+    },
+  ];
+  return {
+    main: `/index.ts`,
+    files,
+  };
+}
+
+function prefix(filename: string, id: string): string {
+  return `/${id}${filename}`;
+}

packages/ts-transformers/src/core/context.ts

@@ -1,10 +1,10 @@
 import ts from "typescript";
-import { ImportRequirements } from "./imports.ts";
 import {
   DiagnosticInput,
   TransformationDiagnostic,
   TransformationOptions,
 } from "./transformers.ts";
+import { CTHelpers } from "./ct-helpers.ts";
 
 const DEFAULT_OPTIONS: TransformationOptions = {
   mode: "transform",
@@ -16,7 +16,6 @@ export interface TransformationContextConfig {
   sourceFile: ts.SourceFile;
   tsContext: ts.TransformationContext;
   options?: TransformationOptions;
-  imports?: ImportRequirements;
 }
 
 export class TransformationContext {
@@ -25,7 +24,7 @@ export class TransformationContext {
   readonly factory: ts.NodeFactory;
   readonly sourceFile: ts.SourceFile;
   readonly options: TransformationOptions;
-  readonly imports: ImportRequirements;
+  readonly ctHelpers: CTHelpers;
   readonly diagnostics: TransformationDiagnostic[] = [];
   readonly tsContext: ts.TransformationContext;
   #typeCache = new Map<ts.Node, ts.Type>();
@@ -36,7 +35,10 @@ export class TransformationContext {
     this.tsContext = config.tsContext;
     this.factory = config.tsContext.factory;
     this.sourceFile = config.sourceFile;
-    this.imports = config.imports ?? new ImportRequirements();
+    this.ctHelpers = new CTHelpers({
+      factory: this.factory,
+      sourceFile: this.sourceFile,
+    });
     this.options = {
       ...DEFAULT_OPTIONS,
       ...config.options,

packages/ts-transformers/src/core/ct-helpers.ts

@@ -0,0 +1,120 @@
+import ts from "typescript";
+import { TransformationContext } from "./mod.ts";
+
+export const CT_HELPERS_IDENTIFIER = "__ctHelpers";
+
+const CT_HELPERS_SPECIFIER = "commontools";
+
+const HELPERS_STMT =
+  `import * as ${CT_HELPERS_IDENTIFIER} from "${CT_HELPERS_SPECIFIER}";`;
+
+const HELPERS_USED_STMT = `${CT_HELPERS_IDENTIFIER}.NAME; // <internals>`;
+
+export class CTHelpers {
+  #sourceFile: ts.SourceFile;
+  #factory: ts.NodeFactory;
+  #helperIdent?: ts.Identifier;
+
+  constructor(params: Pick<TransformationContext, "sourceFile" | "factory">) {
+    this.#sourceFile = params.sourceFile;
+    this.#factory = params.factory;
+
+    for (const stmt of this.#sourceFile.statements) {
+      const symbol = getCTHelpersIdentifier(stmt);
+      if (symbol) {
+        this.#helperIdent = symbol;
+        break;
+      }
+    }
+  }
+
+  sourceHasHelpers(): boolean {
+    return !!this.#helperIdent;
+  }
+
+  // Returns an PropertyAccessExpression of the requested
+  // helper name e.g. `(__ctHelpers.derive)`.
+  getHelperExpr(
+    name: string,
+  ): ts.PropertyAccessExpression {
+    if (!this.sourceHasHelpers()) {
+      throw new Error("Source file does not contain helpers.");
+    }
+    return this.#factory.createPropertyAccessExpression(
+      this.#helperIdent!,
+      name,
+    );
+  }
+
+  // Returns an QualifiedName of the requested
+  // helper name e.g. `__ctHelpers.JSONSchema`.
+  getHelperQualified(
+    name: string,
+  ): ts.QualifiedName {
+    if (!this.sourceHasHelpers()) {
+      throw new Error("Source file does not contain helpers.");
+    }
+    return this.#factory.createQualifiedName(
+      this.#helperIdent!,
+      name,
+    );
+  }
+}
+
+// Replace a `/// <cts-enable />` directive line with an
+// internal import statement for use by the AST transformer
+// to provide access to helpers like `derive`, etc.
+// This operates on strings, and to be used outside of
+// the TypeScript transformer pipeline, since symbol binding
+// occurs before transformers run.
+//
+// We must also inject usage of the module before the AST transformer
+// pipeline, otherwise the binding fails, and the helper module
+// is not available in the compiled JS.
+//
+// Source maps are derived from this transformation.
+// Take care in maintaining source lines from its input.
+//
+// This injected statement enables subsequent transformations.
+export function transformCtDirective(
+  source: string,
+): string {
+  // Throw when this symbol name is already in use
+  // for some reason.
+  if (source.indexOf(CT_HELPERS_IDENTIFIER) !== -1) {
+    throw new Error(
+      `Source cannot contain reserved '${CT_HELPERS_IDENTIFIER}' symbol.`,
+    );
+  }
+  const lines = source.split("\n");
+  if (!lines[0] || !isCTSEnabled(lines[0])) {
+    return source;
+  }
+  return [
+    HELPERS_STMT,
+    ...lines.slice(1),
+    HELPERS_USED_STMT,
+  ].join("\n");
+}
+
+function isCTSEnabled(line: string) {
+  return /^\/\/\/\s*<cts-enable\s*\/>/m.test(line);
+}
+
+function getCTHelpersIdentifier(
+  statement: ts.Statement,
+): ts.Identifier | undefined {
+  if (!ts.isImportDeclaration(statement)) return;
+  const { importClause, moduleSpecifier } = statement;
+
+  // Check specifier is "commontools"
+  if (!ts.isStringLiteral(moduleSpecifier)) return;
+  if (moduleSpecifier.text !== CT_HELPERS_SPECIFIER) return;
+
+  // Check it is a namespace import `* as __ctHelpers`
+  if (!importClause || !ts.isImportClause(importClause)) return;
+  const { namedBindings } = importClause;
+  if (!namedBindings || !ts.isNamespaceImport(namedBindings)) return;
+  if (namedBindings.name.getText() !== CT_HELPERS_IDENTIFIER) return;
+  return namedBindings.name;
+}