feat(zero-client): Normalize schema (rocicorp#2647)

A normalized schema has:
- All the `Record`s are sorted.
- The `tableName` is validated.
- The `primaryKey` is sorted.
- The `columns` are validated to contain all the primary keys and that
  the primary key columns are not optional and not null or undefined.
- The recursive table schemas are resolved.

Author: arv

packages/replicache-perf/src/benchmarks/replicache.ts

@@ -13,6 +13,7 @@ import type {
 import {assert} from '../../../shared/src/asserts.js';
 import {deepEqual} from '../../../shared/src/json.js';
 import {randomUint64} from '../../../shared/src/random-uint64.js';
+import type {Writable} from '../../../shared/src/writable.js';
 import type {Bencher, Benchmark} from '../benchmark.js';
 import {
   type TestDataObject,
@@ -23,8 +24,6 @@ import {
 
 const valSize = 1024;
 
-type Writable<T> = {-readonly [P in keyof T]: T[P]};
-
 export function benchmarkPopulate(opts: {
   numKeys: number;
   clean: boolean;

packages/shared/src/sorted-entries.ts

@@ -0,0 +1,8 @@
+import {stringCompare} from './string-compare.js';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function sortedEntries<T extends Record<string, any>>(
+  object: T,
+): [keyof T & string, T[keyof T]][] {
+  return Object.entries(object).sort((a, b) => stringCompare(a[0], b[0]));
+}

packages/shared/src/tool/vitest-config.ts

@@ -23,8 +23,7 @@ export const config = {
         log.includes('REPLICACHE LICENSE NOT VALID') ||
         log.includes('enableAnalytics false') ||
         log.includes('no such entity') ||
-        log.includes('TODO: addZQLSubscription') ||
-        log.includes('TODO: removeZQLSubscription')
+        log.includes(`Zero starting up with no server URL`)
       ) {
         return false;
       }

packages/shared/src/writable.ts

@@ -0,0 +1,3 @@
+export type Writable<T> = {
+  -readonly [P in keyof T]: T[P];
+};

packages/zero-cache/src/config/config-query.ts

@@ -1,5 +1,6 @@
 import type {AST} from '../../../zql/src/zql/ast/ast.js';
 import type {Format} from '../../../zql/src/zql/ivm/array-view.js';
+import type {NormalizedTableSchema} from '../../../zql/src/zql/query/normalize-table-schema.js';
 import {AbstractQuery} from '../../../zql/src/zql/query/query-impl.js';
 import type {
   DefaultQueryResultRow,
@@ -15,7 +16,7 @@ export class ConfigQuery<
   TReturn extends QueryType = DefaultQueryResultRow<TTableSchema>,
 > extends AbstractQuery<TTableSchema, TReturn> {
   constructor(
-    schema: TTableSchema,
+    schema: NormalizedTableSchema,
     ast?: AST | undefined,
     format?: Format | undefined,
   ) {
@@ -27,7 +28,7 @@ export class ConfigQuery<
   }
 
   protected _newQuery<TSchema extends TableSchema, TReturn extends QueryType>(
-    schema: TSchema,
+    schema: NormalizedTableSchema,
     ast: AST,
     format: Format | undefined,
   ): Query<TSchema, TReturn> {

packages/zero-cache/src/config/define-config.ts

@@ -5,6 +5,7 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import {normalizeSchema} from '../../../zero-client/src/client/normalized-schema.js';
 import type {AST} from '../../../zql/src/zql/ast/ast.js';
 import type {Query, SchemaToRow} from '../../../zql/src/zql/query/query.js';
 import type {TableSchema} from '../../../zql/src/zql/query/schema.js';
@@ -76,8 +77,9 @@ export function defineConfig<TAuthDataShape, TSchema extends Schema>(
   schema: TSchema,
   definer: (queries: Queries<TSchema>) => ZeroConfig<TAuthDataShape, TSchema>,
 ) {
+  const normalizedSchema = normalizeSchema(schema);
   const queries = {} as Record<string, Query<TableSchema>>;
-  for (const [name, tableSchema] of Object.entries(schema.tables)) {
+  for (const [name, tableSchema] of Object.entries(normalizedSchema.tables)) {
     queries[name] = new ConfigQuery(tableSchema);
   }
 

packages/zero-client/src/client/crud.ts

@@ -1,6 +1,9 @@
 import {promiseVoid} from '../../../shared/src/resolved-promises.js';
 import type {MaybePromise} from '../../../shared/src/types.js';
-import {type PrimaryKeyValueRecord} from '../../../zero-protocol/src/primary-key.js';
+import {
+  type PrimaryKeyValue,
+  type PrimaryKeyValueRecord,
+} from '../../../zero-protocol/src/primary-key.js';
 import {
   CRUD_MUTATION_NAME,
   type CreateOp,
@@ -13,9 +16,10 @@ import {
 } from '../../../zero-protocol/src/push.js';
 import type {Row} from '../../../zql/src/zql/ivm/data.js';
 import type {PrimaryKey} from '../../../zql/src/zql/ivm/schema.js';
+import type {NormalizedPrimaryKey} from '../../../zql/src/zql/query/normalize-table-schema.js';
 import type {SchemaToRow} from '../../../zql/src/zql/query/query.js';
 import {toPrimaryKeyString} from './keys.js';
-import {makeIDFromPrimaryKey} from './make-id-from-primary-key.js';
+import type {NormalizedSchema} from './normalized-schema.js';
 import type {MutatorDefs, WriteTransaction} from './replicache-types.js';
 import type {Schema} from './zero.js';
 
@@ -76,7 +80,7 @@ type ZeroCRUDMutate = {
  * `label` properties.
  */
 export function makeCRUDMutate<const S extends Schema>(
-  schema: S,
+  schema: NormalizedSchema,
   repMutate: ZeroCRUDMutate,
 ): MakeCRUDMutate<S> {
   const {[CRUD_MUTATION_NAME]: zeroCRUD} = repMutate;
@@ -122,40 +126,10 @@ export function makeCRUDMutate<const S extends Schema>(
   return mutate as MakeCRUDMutate<S>;
 }
 
-// type NiceIntersection<S, T> = {[K in keyof (S & T)]: (S & T)[K]};
-
-// type MyRow = {
-//   id: string;
-//   n: number;
-//   b: boolean;
-//   null: null;
-// };
-
-// type XXX = NiceIntersection<
-//   Pick<MyRow, 'id' | 'n'>,
-//   Partial<Omit<MyRow, 'id' | 'n'>>
-//   >
-
-// AsPrimaryKeyValueRecord<{
-//   [K in PK[number]]: R[K] extends PrimaryKeyValue ? R[K] : never;
-// }>;
-//Pick<R, PK[number]>;
-
-// const row = {
-//   id: 'abc',
-//   n: null,
-//   b: true,
-//   null: null,
-// };
-// const pk = ['id', 'n'] as const;
-// export type Test = DeleteID<typeof row, typeof pk>;
-
-// export const id: Test = {id: 'a', n: 456};
-
 /**
  * Creates the `{create, set, update, delete}` object for use outside a batch.
  */
-function makeEntityCRUDMutate<R extends Row, PK extends PrimaryKey>(
+function makeEntityCRUDMutate<R extends Row, PK extends NormalizedPrimaryKey>(
   entityType: string,
   primaryKey: PK,
   zeroCRUD: CRUDMutate,
@@ -207,9 +181,12 @@ function makeEntityCRUDMutate<R extends Row, PK extends PrimaryKey>(
 /**
  * Creates the `{create, set, update, delete}` object for use inside a batch.
  */
-export function makeBatchCRUDMutate<R extends Row, PK extends PrimaryKey>(
+export function makeBatchCRUDMutate<
+  R extends Row,
+  PK extends NormalizedPrimaryKey,
+>(
   tableName: string,
-  schema: Schema,
+  schema: NormalizedSchema,
   ops: CRUDOp[],
 ): RowCRUDMutate<R, PK> {
   const {primaryKey} = schema.tables[tableName];
@@ -267,7 +244,7 @@ export type CRUDMutator = (
   crudArg: CRUDMutationArg,
 ) => Promise<void>;
 
-export function makeCRUDMutator<S extends Schema>(schema: S): CRUDMutator {
+export function makeCRUDMutator(schema: NormalizedSchema): CRUDMutator {
   return async function zeroCRUDMutator(
     tx: WriteTransaction,
     crudArg: CRUDMutationArg,
@@ -294,7 +271,7 @@ export function makeCRUDMutator<S extends Schema>(schema: S): CRUDMutator {
 async function createImpl(
   tx: WriteTransaction,
   arg: CreateOp,
-  schema: Schema,
+  schema: NormalizedSchema,
 ): Promise<void> {
   const key = toPrimaryKeyString(
     arg.entityType,
@@ -306,10 +283,10 @@ async function createImpl(
   }
 }
 
-export async function setImpl(
+async function setImpl(
   tx: WriteTransaction,
   arg: CreateOp | SetOp,
-  schema: Schema,
+  schema: NormalizedSchema,
 ): Promise<void> {
   const key = toPrimaryKeyString(
     arg.entityType,
@@ -319,10 +296,10 @@ export async function setImpl(
   await tx.set(key, arg.value);
 }
 
-export async function updateImpl(
+async function updateImpl(
   tx: WriteTransaction,
   arg: UpdateOp,
-  schema: Schema,
+  schema: NormalizedSchema,
 ): Promise<void> {
   const key = toPrimaryKeyString(
     arg.entityType,
@@ -338,10 +315,10 @@ export async function updateImpl(
   await tx.set(key, next);
 }
 
-export async function deleteImpl(
+async function deleteImpl(
   tx: WriteTransaction,
   arg: DeleteOp,
-  schema: Schema,
+  schema: NormalizedSchema,
 ): Promise<void> {
   const key = toPrimaryKeyString(
     arg.entityType,
@@ -350,3 +327,14 @@ export async function deleteImpl(
   );
   await tx.del(key);
 }
+
+function makeIDFromPrimaryKey(
+  primaryKey: NormalizedPrimaryKey,
+  id: PrimaryKeyValueRecord,
+): PrimaryKeyValueRecord {
+  const rv: Record<string, PrimaryKeyValue> = {};
+  for (const key of primaryKey) {
+    rv[key] = id[key];
+  }
+  return rv;
+}

packages/zero-client/src/client/keys.test.ts

@@ -1,9 +1,25 @@
 import fc from 'fast-check';
 import {expect, test} from 'vitest';
-import type {PrimaryKey} from '../../../zero-protocol/src/primary-key.js';
-import {toPrimaryKeyString} from './keys.js';
+import type {
+  PrimaryKey,
+  PrimaryKeyValueRecord,
+} from '../../../zero-protocol/src/primary-key.js';
+import {normalizePrimaryKey} from '../../../zql/src/zql/query/normalize-table-schema.js';
+import {toPrimaryKeyString as toPrimaryKeyStringImpl} from './keys.js';
 
 test('toPrimaryKeyString', () => {
+  function toPrimaryKeyString(
+    tableName: string,
+    primaryKey: PrimaryKey,
+    id: PrimaryKeyValueRecord,
+  ) {
+    return toPrimaryKeyStringImpl(
+      tableName,
+      normalizePrimaryKey(primaryKey),
+      id,
+    );
+  }
+
   expect(
     toPrimaryKeyString('issue', ['id'], {id: 'issue1'}),
   ).toMatchInlineSnapshot(`"e/issue/issue1"`);
@@ -90,8 +106,16 @@ test('no clashes - single pk', () => {
         fc.tuple(fc.boolean(), fc.boolean()),
       ),
       ([a, b]) => {
-        const keyA = toPrimaryKeyString('issue', ['id'], {id: a});
-        const keyB = toPrimaryKeyString('issue', ['id'], {id: b});
+        const keyA = toPrimaryKeyStringImpl(
+          'issue',
+          normalizePrimaryKey(['id']),
+          {id: a},
+        );
+        const keyB = toPrimaryKeyStringImpl(
+          'issue',
+          normalizePrimaryKey(['id']),
+          {id: b},
+        );
         if (a === b) {
           expect(keyA).toBe(keyB);
         } else {
@@ -103,7 +127,7 @@ test('no clashes - single pk', () => {
 });
 
 test('no clashes - multiple pk', () => {
-  const primaryKey: PrimaryKey = ['id', 'name'];
+  const primaryKey = normalizePrimaryKey(['id', 'name']);
   fc.assert(
     fc.property(
       fc.tuple(
@@ -113,11 +137,11 @@ test('no clashes - multiple pk', () => {
         fc.oneof(fc.string(), fc.double(), fc.boolean()),
       ),
       ([a1, a2, b1, b2]) => {
-        const keyA = toPrimaryKeyString('issue', primaryKey, {
+        const keyA = toPrimaryKeyStringImpl('issue', primaryKey, {
           id: a1,
           name: a2,
         });
-        const keyB = toPrimaryKeyString('issue', primaryKey, {
+        const keyB = toPrimaryKeyStringImpl('issue', primaryKey, {
           id: b1,
           name: b2,
         });

packages/zero-client/src/client/keys.ts

@@ -1,8 +1,6 @@
 import {h64WithReverse} from '../../../shared/src/h64-with-reverse.js';
-import type {
-  PrimaryKey,
-  PrimaryKeyValueRecord,
-} from '../../../zero-protocol/src/primary-key.js';
+import type {PrimaryKeyValueRecord} from '../../../zero-protocol/src/primary-key.js';
+import type {NormalizedPrimaryKey} from '../../../zql/src/zql/query/normalize-table-schema.js';
 
 export const CLIENTS_KEY_PREFIX = 'c/';
 export const DESIRED_QUERIES_KEY_PREFIX = 'd/';
@@ -25,32 +23,17 @@ export function toGotQueriesKey(hash: string): string {
   return GOT_QUERIES_KEY_PREFIX + hash;
 }
 
-/**
- * This returns a new array if the array is not already sorted.
- */
-function maybeSort<T>(arr: readonly T[]): readonly T[] {
-  for (let i = 1; i < arr.length; i++) {
-    if (arr[i] < arr[i - 1]) {
-      return [...arr].sort();
-    }
-  }
-  return arr;
-}
-
 export function toPrimaryKeyString(
   tableName: string,
-  primaryKey: PrimaryKey,
+  primaryKey: NormalizedPrimaryKey,
   id: PrimaryKeyValueRecord,
 ): string {
   if (primaryKey.length === 1) {
     return ENTITIES_KEY_PREFIX + tableName + '/' + id[primaryKey[0]];
   }
 
-  // TODO: Assert that PrimaryKey is always sorted at a higher level.
-  const sorted = maybeSort(primaryKey);
-
-  const arr = sorted.map(k => id[k]);
-  const str = JSON.stringify(arr);
+  const values = primaryKey.map(k => id[k]);
+  const str = JSON.stringify(values);
 
   const idSegment = h64WithReverse(str);
   return ENTITIES_KEY_PREFIX + tableName + '/' + idSegment;