improve performance

packages/css-tokenizer/CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Unreleased (major)
 
 - Updated: Support for Node v18+ (major).
+- Improve performance.
 
 ### 2.4.1
 

packages/css-tokenizer/dist/index.d.ts

@@ -36,22 +36,6 @@
  */
 export declare function cloneTokens(tokens: Array<CSSToken>): Array<CSSToken>;
 
-/**
- * @internal
- */
-export declare type CodePointReader = {
-    representationStart: number;
-    representationEnd: number;
-    cursor: number;
-    codePointSource: Array<number>;
-    representationIndices: Array<number>;
-    source: string;
-    advanceCodePoint(n?: number): void;
-    readCodePoint(n?: number): number | false;
-    unreadCodePoint(n?: number): void;
-    resetRepresentation(): void;
-};
-
 /**
  * The union of all possible CSS tokens
  */
@@ -232,24 +216,6 @@ export declare class ParseErrorWithToken extends ParseError {
     constructor(message: string, sourceStart: number, sourceEnd: number, parserState: Array<string>, token: CSSToken);
 }
 
-/**
- * @internal
- */
-export declare class Reader implements CodePointReader {
-    cursor: number;
-    source: string;
-    codePointSource: Array<number>;
-    representationIndices: Array<number>;
-    length: number;
-    representationStart: number;
-    representationEnd: number;
-    constructor(source: string);
-    advanceCodePoint(n?: number): void;
-    readCodePoint(n?: number): number | false;
-    unreadCodePoint(n?: number): void;
-    resetRepresentation(): void;
-}
-
 /**
  * Concatenate the string representation of a list of tokens.
  * This is not a proper serializer that will handle escaping and whitespace.

packages/css-tokenizer/package.json

@@ -48,7 +48,7 @@
 		"dist"
 	],
 	"devDependencies": {
-		"@rmenke/css-tokenizer-tests": "^1.1.5",
+		"@rmenke/css-tokenizer-tests": "^1.1.6",
 		"postcss": "^8.4.38",
 		"postcss-parser-tests": "^8.8.0"
 	},

packages/css-tokenizer/src/checks/four-code-points-would-start-cdo.ts

@@ -3,5 +3,5 @@ import type { CodePointReader } from '../interfaces/code-point-reader';
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token
 export function checkIfFourCodePointsWouldStartCDO(reader: CodePointReader): boolean {
-	return reader.codePointSource[reader.cursor] === LESS_THAN_SIGN && reader.codePointSource[reader.cursor + 1] === EXCLAMATION_MARK && reader.codePointSource[reader.cursor + 2] === HYPHEN_MINUS && reader.codePointSource[reader.cursor + 3] === HYPHEN_MINUS;
+	return reader.source.codePointAt(reader.cursor) === LESS_THAN_SIGN && reader.source.codePointAt(reader.cursor + 1) === EXCLAMATION_MARK && reader.source.codePointAt(reader.cursor + 2) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 3) === HYPHEN_MINUS;
 }

packages/css-tokenizer/src/checks/three-code-points-would-start-cdc.ts

@@ -3,5 +3,5 @@ import type { CodePointReader } from '../interfaces/code-point-reader';
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token
 export function checkIfThreeCodePointsWouldStartCDC(reader: CodePointReader): boolean {
-	return reader.codePointSource[reader.cursor] === HYPHEN_MINUS && reader.codePointSource[reader.cursor + 1] === HYPHEN_MINUS && reader.codePointSource[reader.cursor + 2] === GREATER_THAN_SIGN;
+	return reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 2) === GREATER_THAN_SIGN;
 }

packages/css-tokenizer/src/checks/three-code-points-would-start-ident-sequence.ts

@@ -7,19 +7,19 @@ import { checkIfTwoCodePointsAreAValidEscape } from './two-code-points-are-valid
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#would-start-an-identifier
 export function checkIfThreeCodePointsWouldStartAnIdentSequence(ctx: Context, reader: CodePointReader): boolean {
 	// // U+002D HYPHEN-MINUS
-	if (reader.codePointSource[reader.cursor] === HYPHEN_MINUS) {
+	if (reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) {
 		// If the second code point is a U+002D HYPHEN-MINUS return true
-		if (reader.codePointSource[reader.cursor + 1] === HYPHEN_MINUS) {
+		if (reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS) {
 			return true;
 		}
 
 		// If the second code point is an ident-start code point return true
-		if (isIdentStartCodePoint(reader.codePointSource[reader.cursor + 1])) {
+		if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor + 1))) {
 			return true;
 		}
 
 		// If the second and third code points are a valid escape return true
-		if (reader.codePointSource[reader.cursor + 1] === REVERSE_SOLIDUS && !isNewLine(reader.codePointSource[reader.cursor + 2])) {
+		if (reader.source.codePointAt(reader.cursor + 1) === REVERSE_SOLIDUS && !isNewLine(reader.source.codePointAt(reader.cursor + 2))) {
 			return true;
 		}
 
@@ -28,7 +28,7 @@ export function checkIfThreeCodePointsWouldStartAnIdentSequence(ctx: Context, re
 
 	// ident-start code point
 	// Return true.
-	if (isIdentStartCodePoint(reader.codePointSource[reader.cursor])) {
+	if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor))) {
 		return true;
 	}
 

packages/css-tokenizer/src/checks/three-code-points-would-start-number.ts

@@ -4,26 +4,26 @@ import type { CodePointReader } from '../interfaces/code-point-reader';
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-number
 export function checkIfThreeCodePointsWouldStartANumber(reader: CodePointReader): boolean {
-	if (reader.codePointSource[reader.cursor] === PLUS_SIGN || reader.codePointSource[reader.cursor] === HYPHEN_MINUS) { // U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-)
+	if (reader.source.codePointAt(reader.cursor) === PLUS_SIGN || reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { // U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-)
 		// If the second code point is a digit, return true.
-		if (isDigitCodePoint(reader.codePointSource[reader.cursor + 1])) {
+		if (isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1))) {
 			return true;
 		}
 
 		// Otherwise, if the second code point is a U+002E FULL STOP (.)
-		if (reader.codePointSource[reader.cursor + 1] === FULL_STOP) {
+		if (reader.source.codePointAt(reader.cursor + 1) === FULL_STOP) {
 			// and the third code point is a digit, return true.
-			return isDigitCodePoint(reader.codePointSource[reader.cursor + 2]);
+			return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 2));
 		}
 
 		// Otherwise, return false.
 		return false;
 
-	} else if (reader.codePointSource[reader.cursor] === FULL_STOP) { // U+002E FULL STOP (.)
+	} else if (reader.source.codePointAt(reader.cursor) === FULL_STOP) { // U+002E FULL STOP (.)
 		// If the second code point is a digit, return true.
 		// Otherwise, return false.
-		return isDigitCodePoint(reader.codePointSource[reader.cursor + 1]);
+		return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1));
 	}
 
-	return isDigitCodePoint(reader.codePointSource[reader.cursor]); // digit
+	return isDigitCodePoint(reader.source.codePointAt(reader.cursor)); // digit
 }

packages/css-tokenizer/src/checks/three-code-points-would-start-unicode-range.ts

@@ -7,15 +7,15 @@ export function checkIfThreeCodePointsWouldStartAUnicodeRange(reader: CodePointR
 	if (
 		// The first code point is either U+0055 LATIN CAPITAL LETTER U (U) or U+0075 LATIN SMALL LETTER U (u)
 		(
-			reader.codePointSource[reader.cursor] === LATIN_SMALL_LETTER_U ||
-			reader.codePointSource[reader.cursor] === LATIN_CAPITAL_LETTER_U
+			reader.source.codePointAt(reader.cursor) === LATIN_SMALL_LETTER_U ||
+			reader.source.codePointAt(reader.cursor) === LATIN_CAPITAL_LETTER_U
 		) &&
 		// The second code point is U+002B PLUS SIGN (+).
-		reader.codePointSource[reader.cursor + 1] === PLUS_SIGN &&
+		reader.source.codePointAt(reader.cursor + 1) === PLUS_SIGN &&
 		// The third code point is either U+003F QUESTION MARK (?) or a hex digit
 		(
-			reader.codePointSource[reader.cursor + 2] === QUESTION_MARK ||
-			isHexDigitCodePoint(reader.codePointSource[reader.cursor + 2])
+			reader.source.codePointAt(reader.cursor + 2) === QUESTION_MARK ||
+			isHexDigitCodePoint(reader.source.codePointAt(reader.cursor + 2))
 		)
 	) {
 		// then return true.

packages/css-tokenizer/src/checks/two-code-points-are-valid-escape.ts

@@ -6,8 +6,8 @@ import type { CodePointReader } from '../interfaces/code-point-reader';
 export function checkIfTwoCodePointsAreAValidEscape(reader: CodePointReader): boolean {
 	return (
 		// If the first code point is not U+005C REVERSE SOLIDUS (\), return false.
-		reader.codePointSource[reader.cursor] === REVERSE_SOLIDUS &&
+		reader.source.codePointAt(reader.cursor) === REVERSE_SOLIDUS &&
 		// Otherwise, if the second code point is a newline, return false.
-		!isNewLine(reader.codePointSource[reader.cursor + 1])
+		!isNewLine(reader.source.codePointAt(reader.cursor + 1))
 	);
 }

packages/css-tokenizer/src/checks/two-code-points-start-comment.ts

@@ -4,7 +4,7 @@ import type { CodePointReader } from '../interfaces/code-point-reader';
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comments
 export function checkIfTwoCodePointsStartAComment(reader: CodePointReader): boolean {
 	return (
-		reader.codePointSource[reader.cursor] === SOLIDUS &&
-		reader.codePointSource[reader.cursor + 1] === ASTERISK
+		reader.source.codePointAt(reader.cursor) === SOLIDUS &&
+		reader.source.codePointAt(reader.cursor + 1) === ASTERISK
 	);
 }

packages/css-tokenizer/src/code-points/ranges.ts

@@ -2,46 +2,46 @@ import { BACKSPACE, DELETE, INFORMATION_SEPARATOR_ONE, LINE_TABULATION, LOW_LINE
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokenizer-definitions
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#digit
-export function isDigitCodePoint(search: number): boolean {
-	return search >= 0x0030 && search <= 0x0039;
+export function isDigitCodePoint(search: number | undefined): search is number {
+	return (typeof search !== "undefined") && search >= 0x0030 && search <= 0x0039;
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#uppercase-letter
-function isUppercaseLetterCodePoint(search: number): boolean {
-	return search >= 0x0041 && search <= 0x005a;
+function isUppercaseLetterCodePoint(search: number | undefined): search is number {
+	return (typeof search !== "undefined") && search >= 0x0041 && search <= 0x005a;
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#lowercase-letter
-function isLowercaseLetterCodePoint(search: number): boolean {
-	return search >= 0x0061 && search <= 0x007a;
+function isLowercaseLetterCodePoint(search: number | undefined): search is number {
+	return (typeof search !== "undefined") && search >= 0x0061 && search <= 0x007a;
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#hex-digit
-export function isHexDigitCodePoint(search: number): boolean {
-	return (
-		isDigitCodePoint(search) || // 0 .. 9
+export function isHexDigitCodePoint(search: number | undefined): search is number {
+	return (typeof search !== "undefined") && (
+		(search >= 0x0030 && search <= 0x0039) || // 0 .. 9
 		(search >= 0x0061 && search <= 0x0066) || // a .. f
 		(search >= 0x0041 && search <= 0x0046)    // A .. F
 	);
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#letter
-function isLetterCodePoint(search: number): boolean {
+function isLetterCodePoint(search: number | undefined): search is number {
 	return isLowercaseLetterCodePoint(search) || isUppercaseLetterCodePoint(search);
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-start-code-point
-export function isIdentStartCodePoint(search: number): boolean {
+export function isIdentStartCodePoint(search: number | undefined): search is number {
 	return isLetterCodePoint(search) || isNonASCII_IdentCodePoint(search) || search === LOW_LINE;
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-code-point
-export function isIdentCodePoint(search: number): boolean {
+export function isIdentCodePoint(search: number | undefined): search is number {
 	return isIdentStartCodePoint(search) || isDigitCodePoint(search) || search === HYPHEN_MINUS;
 }
 
 // https://drafts.csswg.org/css-syntax/#non-ascii-ident-code-point
-function isNonASCII_IdentCodePoint(search: number): boolean {
+function isNonASCII_IdentCodePoint(search: number | undefined): search is number {
 	if (
 		search === 0x00B7 ||
 		search === 0x200C ||
@@ -53,6 +53,10 @@ function isNonASCII_IdentCodePoint(search: number): boolean {
 		return true;
 	}
 
+	if (typeof search === "undefined") {
+		return false;
+	}
+
 	if (
 		(0x00C0 <= search && search <= 0x00D6) ||
 		(0x00D8 <= search && search <= 0x00F6) ||
@@ -71,8 +75,8 @@ function isNonASCII_IdentCodePoint(search: number): boolean {
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#non-printable-code-point
-export function isNonPrintableCodePoint(search: number): boolean {
-	return (
+export function isNonPrintableCodePoint(search: number | undefined): search is number {
+	return (typeof search !== "undefined") &&(
 		(search === LINE_TABULATION) ||
 		(search === DELETE) ||
 		(NULL <= search && search <= BACKSPACE) ||
@@ -81,16 +85,16 @@ export function isNonPrintableCodePoint(search: number): boolean {
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#whitespace
-export function isNewLine(search: number): boolean {
+export function isNewLine(search: number | undefined): search is number {
 	return search === LINE_FEED || search === CARRIAGE_RETURN || search === FORM_FEED;
 }
 
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#whitespace
-export function isWhitespace(search: number): boolean {
+export function isWhitespace(search: number | undefined): search is number {
 	return search === SPACE || search === LINE_FEED || search === CHARACTER_TABULATION || search === CARRIAGE_RETURN || search === FORM_FEED;
 }
 
 // https://infra.spec.whatwg.org/#surrogate
-export function isSurrogate(search: number): boolean {
-	return search >= 0xd800 && search <= 0xdfff;
+export function isSurrogate(search: number | undefined): search is number {
+	return (typeof search !== "undefined") && search >= 0xd800 && search <= 0xdfff;
 }

packages/css-tokenizer/src/consume/bad-url.ts

@@ -7,11 +7,12 @@ import { consumeEscapedCodePoint } from './escaped-code-point';
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-remnants-of-bad-url
 export function consumeBadURL(ctx: Context, reader: CodePointReader): void {
 	while (true) {
-		if (reader.codePointSource[reader.cursor] === undefined) {
+		const codePoint = reader.source.codePointAt(reader.cursor);
+		if (typeof codePoint === "undefined") {
 			return;
 		}
 
-		if (reader.codePointSource[reader.cursor] === RIGHT_PARENTHESIS) {
+		if (codePoint === RIGHT_PARENTHESIS) {
 			reader.advanceCodePoint();
 			return;
 		}

packages/css-tokenizer/src/consume/comment.ts

@@ -11,7 +11,7 @@ export function consumeComment(ctx: Context, reader: CodePointReader): TokenComm
 
 	while (true) {
 		const codePoint = reader.readCodePoint();
-		if (codePoint === false) {
+		if (typeof codePoint === "undefined") {
 			const token: CSSToken = [
 				TokenType.Comment,
 				reader.source.slice(reader.representationStart, reader.representationEnd + 1),
@@ -38,11 +38,11 @@ export function consumeComment(ctx: Context, reader: CodePointReader): TokenComm
 			continue;
 		}
 
-		if (reader.codePointSource[reader.cursor] === undefined) {
+		if (typeof reader.source.codePointAt(reader.cursor) === "undefined") {
 			continue;
 		}
 
-		if (reader.codePointSource[reader.cursor] === SOLIDUS) {
+		if (reader.source.codePointAt(reader.cursor) === SOLIDUS) {
 			reader.advanceCodePoint();
 			break;
 		}

packages/css-tokenizer/src/consume/escaped-code-point.ts

@@ -7,7 +7,7 @@ import { ParseError, ParseErrorMessage } from '../interfaces/error';
 // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-escaped-code-point
 export function consumeEscapedCodePoint(ctx: Context, reader: CodePointReader): number {
 	const codePoint = reader.readCodePoint();
-	if (codePoint === false) {
+	if (typeof codePoint === "undefined") {
 		ctx.onParseError(new ParseError(
 			ParseErrorMessage.UnexpectedEOFInEscapedCodePoint,
 			reader.representationStart,
@@ -24,12 +24,13 @@ export function consumeEscapedCodePoint(ctx: Context, reader: CodePointReader):
 	if (isHexDigitCodePoint(codePoint)) {
 		const hexSequence: Array<number> = [codePoint];
 
-		while ((reader.codePointSource[reader.cursor] !== undefined) && isHexDigitCodePoint(reader.codePointSource[reader.cursor]) && hexSequence.length < 6) {
-			hexSequence.push(reader.codePointSource[reader.cursor]);
+		let nextCodePoint: number | undefined;
+		while ((typeof (nextCodePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && isHexDigitCodePoint(nextCodePoint) && hexSequence.length < 6) {
+			hexSequence.push(nextCodePoint);
 			reader.advanceCodePoint();
 		}
 
-		if (isWhitespace(reader.codePointSource[reader.cursor])) {
+		if (isWhitespace(reader.source.codePointAt(reader.cursor))) {
 			reader.advanceCodePoint();
 		}
 

packages/css-tokenizer/src/consume/hash-token.ts

@@ -11,9 +11,10 @@ import { consumeIdentSequence } from './ident-sequence';
 export function consumeHashToken(ctx: Context, reader: CodePointReader): TokenDelim|TokenHash {
 	reader.advanceCodePoint();
 
+	const codePoint = reader.source.codePointAt(reader.cursor);
 	if (
-		(reader.codePointSource[reader.cursor] !== undefined) && (
-			isIdentCodePoint(reader.codePointSource[reader.cursor]) ||
+		(typeof codePoint !== "undefined") && (
+			isIdentCodePoint(codePoint) ||
 			checkIfTwoCodePointsAreAValidEscape(reader)
 		)
 	) {