align serializers more with cssom

packages/css-tokenizer/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changes to CSS Tokenizer
 
+### Unreleased (patch)
+
+- align serializers with CSSOM
+
 ### 3.0.3
 
 _October 25, 2024_

packages/css-tokenizer/src/util/mutations.ts

@@ -1,5 +1,5 @@
-import { HYPHEN_MINUS } from '../code-points/code-points';
-import { isHexDigitCodePoint, isIdentCodePoint, isIdentStartCodePoint } from '../code-points/ranges';
+import { HYPHEN_MINUS, NULL, REPLACEMENT_CHARACTER } from '../code-points/code-points';
+import { isIdentCodePoint, isIdentStartCodePoint } from '../code-points/ranges';
 import type { TokenDimension, TokenIdent } from '../interfaces/token';
 
 /**
@@ -13,7 +13,7 @@ export function mutateIdent(ident: TokenIdent, newValue: string): void {
 		codePoints.push(codePoint.codePointAt(0)!);
 	}
 
-	const result = String.fromCodePoint(...ensureThatValueRoundTripsAsIdent(codePoints));
+	const result = String.fromCodePoint(...serializeIdent(codePoints));
 
 	ident[1] = result;
 	ident[4].value = newValue;
@@ -30,7 +30,7 @@ export function mutateUnit(ident: TokenDimension, newUnit: string): void {
 		codePoints.push(codePoint.codePointAt(0)!);
 	}
 
-	const escapedCodePoints = ensureThatValueRoundTripsAsIdent(codePoints);
+	const escapedCodePoints = serializeIdent(codePoints);
 	if (escapedCodePoints[0] === 101) { // `e`
 		insertEscapedCodePoint(escapedCodePoints, 0, escapedCodePoints[0]);
 	}
@@ -44,17 +44,30 @@ export function mutateUnit(ident: TokenDimension, newUnit: string): void {
 	ident[4].unit = newUnit;
 }
 
-function ensureThatValueRoundTripsAsIdent(codePoints: Array<number>): Array<number> {
+function serializeIdent(codePoints: Array<number>): Array<number> {
 	let remainderStartIndex = 0;
 
-	if (codePoints[0] === HYPHEN_MINUS && codePoints[1] === HYPHEN_MINUS) {
+	if (codePoints[0] === NULL) {
+		codePoints.splice(
+			0,
+			1,
+			REPLACEMENT_CHARACTER
+		);
+
+		remainderStartIndex = 1;
+	} else if (codePoints[0] === HYPHEN_MINUS && codePoints[1] === HYPHEN_MINUS) {
 		remainderStartIndex = 2;
 	} else if (codePoints[0] === HYPHEN_MINUS && codePoints[1]) {
 		remainderStartIndex = 2;
 
 		if (!isIdentStartCodePoint(codePoints[1])) {
 			remainderStartIndex += insertEscapedCodePoint(codePoints, 1, codePoints[1]);
 		}
+	} else if (codePoints[0] === HYPHEN_MINUS && !codePoints[1]) {
+		return [
+			92, // `\` backslash
+			codePoints[0]
+		];
 	} else if (isIdentStartCodePoint(codePoints[0])) {
 		remainderStartIndex = 1;
 	} else {
@@ -63,16 +76,38 @@ function ensureThatValueRoundTripsAsIdent(codePoints: Array<number>): Array<numb
 	}
 
 	for (let i = remainderStartIndex; i < codePoints.length; i++) {
+		if (codePoints[i] === NULL) {
+			codePoints.splice(
+				i,
+				1,
+				REPLACEMENT_CHARACTER
+			);
+
+			i++;
+			continue;
+		}
+
 		if (isIdentCodePoint(codePoints[i])) {
 			continue;
 		}
 
-		i += insertEscapedCodePoint(codePoints, i, codePoints[i]);
+		i += insertEscapedCharacter(codePoints, i, codePoints[i]);
 	}
 
 	return codePoints;
 }
 
+function insertEscapedCharacter(codePoints: Array<number>, index: number, codePoint: number): number {
+	codePoints.splice(
+		index,
+		1,
+		92, // `\` backslash
+		codePoint
+	);
+
+	return 1;
+}
+
 function insertEscapedCodePoint(codePoints: Array<number>, index: number, codePoint: number): number {
 	const hexRepresentation = codePoint.toString(16);
 	const codePointsForHexRepresentation: Array<number> = [];
@@ -82,28 +117,13 @@ function insertEscapedCodePoint(codePoints: Array<number>, index: number, codePo
 		codePointsForHexRepresentation.push(x.codePointAt(0)!);
 	}
 
-	const next = codePoints[index + 1];
-	if (
-		(index === codePoints.length - 1) ||
-		(next && isHexDigitCodePoint(next))
-	) {
-		codePoints.splice(
-			index,
-			1,
-			92, // `\` backslash
-			...codePointsForHexRepresentation,
-			32, // ` ` space
-		);
-
-		return 1 + codePointsForHexRepresentation.length;
-	}
-
 	codePoints.splice(
 		index,
 		1,
 		92, // `\` backslash
 		...codePointsForHexRepresentation,
+		32, // ` ` space
 	);
 
-	return codePointsForHexRepresentation.length;
+	return 1 + codePointsForHexRepresentation.length;
 }

packages/css-tokenizer/test/mutations/ident.mjs

@@ -36,13 +36,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 
 	assert.deepEqual(
 		token,
-		['ident-token', 'foo\\2e bar', 0, 6, { value: 'foo.bar' }],
+		['ident-token', 'foo\\.bar', 0, 6, { value: 'foo.bar' }],
 	);
 
 	const raw = stringify(token);
 	assert.deepEqual(
 		raw,
-		'foo\\2e bar',
+		'foo\\.bar',
 	);
 
 	const t = tokenizer({
@@ -52,7 +52,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 	assert.deepEqual(
 		collectTokens(t),
 		[
-			['ident-token', 'foo\\2e bar', 0, 9, { value: 'foo.bar' }],
+			['ident-token', 'foo\\.bar', 0, 7, { value: 'foo.bar' }],
 			['EOF-token', '', -1, -1, undefined],
 		],
 	);
@@ -92,13 +92,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 
 	assert.deepEqual(
 		token,
-		['ident-token', '\\2d ', 0, 2, { value: '-' }],
+		['ident-token', '\\-', 0, 2, { value: '-' }],
 	);
 
 	const raw = stringify(token);
 	assert.deepEqual(
 		raw,
-		'\\2d ',
+		'\\-',
 	);
 
 	const t = tokenizer({
@@ -108,7 +108,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 	assert.deepEqual(
 		collectTokens(t),
 		[
-			['ident-token', '\\2d ', 0, 3, { value: '-' }],
+			['ident-token', '\\-', 0, 1, { value: '-' }],
 			['EOF-token', '', -1, -1, undefined],
 		],
 	);
@@ -148,13 +148,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 
 	assert.deepEqual(
 		token,
-		['ident-token', '--\\25other-prop', 0, 2, { value: '--%other-prop' }],
+		['ident-token', '--\\%other-prop', 0, 2, { value: '--%other-prop' }],
 	);
 
 	const raw = stringify(token);
 	assert.deepEqual(
 		raw,
-		'--\\25other-prop',
+		'--\\%other-prop',
 	);
 
 	const t = tokenizer({
@@ -166,9 +166,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 		[
 			[
 				'ident-token',
-				'--\\25other-prop',
+				'--\\%other-prop',
 				0,
-				14,
+				13,
 				{ value: '--%other-prop' },
 			],
 			['EOF-token', '', -1, -1, undefined],
@@ -182,13 +182,13 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 
 	assert.deepEqual(
 		token,
-		['ident-token', '--\\25 a-prop', 0, 2, { value: '--%a-prop' }],
+		['ident-token', '--\\%a-prop', 0, 2, { value: '--%a-prop' }],
 	);
 
 	const raw = stringify(token);
 	assert.deepEqual(
 		raw,
-		'--\\25 a-prop',
+		'--\\%a-prop',
 	);
 
 	const t = tokenizer({
@@ -200,9 +200,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 		[
 			[
 				'ident-token',
-				'--\\25 a-prop',
+				'--\\%a-prop',
 				0,
-				11,
+				9,
 				{ value: '--%a-prop' },
 			],
 			['EOF-token', '', -1, -1, undefined],
@@ -252,7 +252,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 		token,
 		[
 			'ident-token',
-			'-\\40webkit-width',
+			'-\\40 webkit-width',
 			0,
 			2,
 			{ value: '-@webkit-width' },
@@ -262,7 +262,7 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 	const raw = stringify(token);
 	assert.deepEqual(
 		raw,
-		'-\\40webkit-width',
+		'-\\40 webkit-width',
 	);
 
 	const t = tokenizer({
@@ -274,9 +274,9 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 		[
 			[
 				'ident-token',
-				'-\\40webkit-width',
+				'-\\40 webkit-width',
 				0,
-				15,
+				16,
 				{ value: '-@webkit-width' },
 			],
 			['EOF-token', '', -1, -1, undefined],
@@ -311,3 +311,87 @@ import { mutateIdent, tokenizer, stringify } from '@csstools/css-tokenizer';
 		],
 	);
 }
+
+{
+	const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
+	mutateIdent(token, '-');
+
+	assert.deepEqual(
+		token,
+		['ident-token', '\\-', 0, 2, { value: '-' }],
+	);
+
+	const raw = stringify(token);
+	assert.deepEqual(
+		raw,
+		'\\-',
+	);
+
+	const t = tokenizer({
+		css: raw,
+	});
+
+	assert.deepEqual(
+		collectTokens(t),
+		[
+			['ident-token', '\\-', 0, 1, { value: '-' }],
+			['EOF-token', '', -1, -1, undefined],
+		],
+	);
+}
+
+{
+	const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
+	mutateIdent(token, String.fromCodePoint(0x000));
+
+	assert.deepEqual(
+		token,
+		['ident-token', String.fromCodePoint(0xFFFD), 0, 2, { value: String.fromCodePoint(0x000) }],
+	);
+
+	const raw = stringify(token);
+	assert.deepEqual(
+		raw,
+		String.fromCodePoint(0xFFFD),
+	);
+
+	const t = tokenizer({
+		css: raw,
+	});
+
+	assert.deepEqual(
+		collectTokens(t),
+		[
+			['ident-token', String.fromCodePoint(0xFFFD), 0, 0, { value: String.fromCodePoint(0xFFFD) }],
+			['EOF-token', '', -1, -1, undefined],
+		],
+	);
+}
+
+{
+	const token = ['ident-token', 'prop', 0, 2, { value: 'prop' }];
+	mutateIdent(token, 'aa' + String.fromCodePoint(0x000));
+
+	assert.deepEqual(
+		token,
+		['ident-token', 'aa' + String.fromCodePoint(0xFFFD), 0, 2, { value: 'aa' + String.fromCodePoint(0x000) }],
+	);
+
+	const raw = stringify(token);
+	assert.deepEqual(
+		raw,
+		'aa' + String.fromCodePoint(0xFFFD),
+	);
+
+	const t = tokenizer({
+		css: raw,
+	});
+
+	assert.deepEqual(
+		collectTokens(t),
+		[
+			['ident-token', 'aa' + String.fromCodePoint(0xFFFD), 0, 2, { value: 'aa' + String.fromCodePoint(0xFFFD) }],
+			['EOF-token', '', -1, -1, undefined],
+		],
+	);
+}

packages/media-query-list-parser/test/get-name/0001.mjs

@@ -44,7 +44,7 @@ import { TokenType } from '@csstools/css-tokenizer';
 
 	assert.deepStrictEqual(
 		feature.getNameToken(),
-		['ident-token', 'min\\20width', -1, -1, { value: 'min width' }],
+		['ident-token', 'min\\ width', -1, -1, { value: 'min width' }],
 	);
 }
 
@@ -90,7 +90,7 @@ import { TokenType } from '@csstools/css-tokenizer';
 
 	assert.deepStrictEqual(
 		feature.getNameToken(),
-		['ident-token', 'w\\20 dth', -1, -1, { value: 'w dth' }],
+		['ident-token', 'w\\ dth', -1, -1, { value: 'w dth' }],
 	);
 }