next

Author: jonathantneal

README.md

@@ -31,21 +31,19 @@ Tokenize CSS in JavaScript:
 ```js
 import { tokenize } from '@csstools/tokenizer'
 
-for (const token of tokenize(cssText)) {
+tokenize(cssText, (token) => {
   console.log(token) // logs an individual CSSToken
-}
+})
 ```
 
 Tokenize CSS in _classical_ NodeJS:
 
 ```js
 const { tokenizer } = require('@csstools/tokenizer')
 
-let iterator = tokenizer(cssText), iteration
-
-while (!(iteration = iterator()).done) {
-  console.log(iteration.value) // logs an individual CSSToken
-}
+tokenize(cssText, (token) => {
+  console.log(token) // logs an individual CSSToken
+})
 ```
 
 Tokenize CSS in client-side scripts:
@@ -62,17 +60,6 @@ for (const token of tokenize(cssText)) {
 </script>
 ```
 
-Tokenize CSS in _classical_ client-side scripts:
-
-```html
-<script src="http://unpkg.com/@csstools/tokenizer"></script>
-<script>
-
-const tokens = Array.from(tokenizeCSS(cssText)) // an array of CSSTokens
-
-</script>
-```
-
 ## How it works
 
 The CSS tokenizer separates a string of CSS into tokens.
@@ -82,58 +69,43 @@ interface CSSToken {
   /** Position in the string at which the token was retrieved. */
   tick: number
 
-  /** Number identifying the kind of token. */
-  type:
-    | 1 // Symbol
-    | 2 // Comment
-    | 3 // Space
-    | 4 // Word
-    | 5 // Function
-    | 6 // Atword
-    | 7 // Hash
-    | 8 // String
-    | 9 // Number
-  
-  /** Code, like the character code of a symbol, or the character code of the opening parenthesis of a function. */
-  code: number
-
-  /** Lead, like the opening of a comment, the quotation mark of a string, or the name of a function. */
-  lead: string,
-
-  /** Data, like the numbers before a unit, the word after an at-sign, or the opening parenthesis of a Function. */
-  data: string,
-
-  /** Tail, like the unit after a number, or the closing of a comment. */
-  tail: string,
+  /** Number identifying the kind of token; or the character code of a symbol. */
+  type: number
+
+  /** Data, the number of characters in the token. */
+  data: number,
+
+  /** Tail, the number of characters in a unit after a number, or in the closing of a comment. */
+  edge: string,
 }
 ```
 
 As an example, the CSS string `@media` would become a **Atword** token where `@` and `media` are recognized as distinct parts of that token. As another example, the CSS string `5px` would become a **Number** token where `5` and `px` are recognized as distinct parts of that token. As a final example, the string `5px 10px` would become 3 tokens; the **Number** as mentioned before (`5px`), a **Space** token that represents a single space (` `), and then another **Number** token (`10px`).
 
 ## Benchmarks
 
-As of August 23, 2021, these benchmarks were averaged from my local machine:
+As of April 11, 2022, these benchmarks were reported from my local machine:
 
 ```
 Benchmark: Tailwind CSS
   ┌────────────────────────────────────────────────────┬───────┬────────┬────────┐
   │                      (index)                       │  ms   │ ms/50k │ tokens │
   ├────────────────────────────────────────────────────┼───────┼────────┼────────┤
-  │ CSSTree 1 x 35.04 ops/sec ±6.55% (64 runs sampled) │ 28.54 │  1.51  │ 946205 │
-  │ CSSTree 2 x 41.76 ops/sec ±7.57% (58 runs sampled) │ 23.95 │  1.27  │ 946205 │
-  │ PostCSS 8 x 14.18 ops/sec ±3.31% (40 runs sampled) │ 70.54 │  3.77  │ 935282 │
-  │ Tokenizer x 17.40 ops/sec ±0.98% (48 runs sampled) │ 57.48 │  3.04  │ 946206 │
+  │ CSSTree 1 x 35.86 ops/sec ±6.82% (65 runs sampled) │ 27.88 │  1.47  │ 946205 │
+  │ CSSTree 2 x 43.15 ops/sec ±7.57% (60 runs sampled) │ 23.17 │  1.22  │ 946205 │
+  │ PostCSS 8 x 14.82 ops/sec ±2.33% (42 runs sampled) │ 67.46 │  3.61  │ 935282 │
+  │ CSS Tools x 36.76 ops/sec ±0.14% (65 runs sampled) │ 27.2  │  1.44  │ 946205 │
   └────────────────────────────────────────────────────┴───────┴────────┴────────┘
 
-Benchmark: Bootstrap
-  ┌───────────────────────────────────────────────────┬──────┬────────┬────────┐
-  │                      (index)                      │  ms  │ ms/50k │ tokens │
-  ├───────────────────────────────────────────────────┼──────┼────────┼────────┤
-  │ CSSTree 1 x 600 ops/sec ±0.87% (96 runs sampled)  │ 1.67 │  1.41  │ 59236  │
-  │ CSSTree 2 x 695 ops/sec ±0.08% (100 runs sampled) │ 1.44 │  1.21  │ 59236  │
-  │ PostCSS 8 x 432 ops/sec ±0.94% (94 runs sampled)  │ 2.31 │  2.26  │ 51170  │
-  │ Tokenizer x 288 ops/sec ±0.40% (93 runs sampled)  │ 3.48 │  2.93  │ 59237  │
-  └───────────────────────────────────────────────────┴──────┴────────┴────────┘
+Benchmark: Bootstrap CSS
+  ┌──────────────────────────────────────────────────┬──────┬────────┬────────┐
+  │                     (index)                      │  ms  │ ms/50k │ tokens │
+  ├──────────────────────────────────────────────────┼──────┼────────┼────────┤
+  │ CSSTree 1 x 608 ops/sec ±0.26% (97 runs sampled) │ 1.65 │  1.38  │ 59543  │
+  │ CSSTree 2 x 702 ops/sec ±0.16% (97 runs sampled) │ 1.42 │  1.2   │ 59543  │
+  │ PostCSS 8 x 440 ops/sec ±0.10% (94 runs sampled) │ 2.27 │  2.21  │ 51453  │
+  │ CSS Tools x 623 ops/sec ±0.15% (97 runs sampled) │ 1.61 │  1.35  │ 59543  │
+  └──────────────────────────────────────────────────┴──────┴────────┴────────┘
 ```
 
 ## Development
@@ -161,7 +133,7 @@ npm run benchmark
 
 The **test** command tests the coverage and accuracy of the tokenizer.
 
-As of September 26, 2020, this tokenizer has 100% test coverage:
+As of April 18, 2022, this tokenizer maintains 100% test coverage:
 
 ```sh
 npm run test

cmd/benchmark.mjs

@@ -2,7 +2,7 @@ import * as _ from './_.mjs'
 
 const isWatch = process.argv.includes('--watch')
 
-console.log(`Starting compilation...`)
+console.log(`Building...`)
 
 _.rmdir('dist')
 

cmd/build.mjs

@@ -2,7 +2,7 @@ import * as _ from './_.mjs'
 
 const isWatch = process.argv.includes('--watch')
 
-console.log(`Starting compilation...`)
+console.log(`Building...`)
 
 _.rmdir('dist')
 _.mkdir('dist')

cmd/publish.mjs

@@ -4,7 +4,7 @@ import './build.mjs'
 
 const isDryRun = Boolean(process?.env?.npm_config_dryRun)
 
-console.log(), console.log('Starting publishing...')
+console.log(), console.log('Publishing...')
 
 _.question.options = { input: process.stdin, output: process.stdout }
 

cmd/test.mjs

@@ -4,7 +4,7 @@ const isWatch = process.argv.includes('--watch')
 
 const jestBin = 'node_modules/jest/bin/jest.js'
 
-console.log(`Starting testing...`)
+console.log(`Testing...`)
 console.log()
 
 _.rmdir('coverage')

package.json

@@ -46,27 +46,27 @@
     "access": "public"
   },
   "devDependencies": {
-    "@babel/core": "7.15.0",
-    "@babel/preset-env": "7.15.0",
-    "@babel/preset-typescript": "7.15.0",
-    "@rollup/plugin-babel": "5.3.0",
+    "@babel/core": "7.17.9",
+    "@babel/preset-env": "7.16.11",
+    "@babel/preset-typescript": "7.16.7",
+    "@rollup/plugin-babel": "5.3.1",
     "@types/benchmark": "2.1.1",
-    "@types/jest": "27.0.1",
-    "@types/node": "16.7.4",
+    "@types/jest": "27.4.1",
+    "@types/node": "17.0.23",
     "benchmark": "2.1.4",
-    "bootstrap": "5.1.0",
+    "bootstrap": "5.1.3",
     "codecov": "3.8.3",
     "css-tree1": "npm:css-tree@^1.1.3",
-    "css-tree2": "npm:css-tree@^2.0.1",
-    "jest": "27.1.0",
-    "magic-string": "0.25.7",
-    "postcss": "8.3.6",
-    "rollup": "2.56.3",
+    "css-tree2": "npm:css-tree@^2.1.0",
+    "jest": "27.5.1",
+    "magic-string": "0.26.1",
+    "postcss": "8.4.12",
+    "rollup": "2.70.1",
     "rollup-plugin-bundle-size": "1.0.3",
     "rollup-plugin-terser": "7.0.2",
     "tailwindcss": "2.2.8",
-    "tsc-watch": "4.5.0",
+    "tsc-watch": "5.0.3",
     "tslib": "2.3.1",
-    "typescript": "4.4.2"
+    "typescript": "4.6.3"
   }
 }

rollup.config.js

@@ -29,17 +29,19 @@ const resolveJsTsExtension = {
 	}
 }
 
-const reduceImpact = {
+const reduceImpact = (trim = 0) => ({
 	name: 'reduce-impact',
 	renderChunk(code) {
 		const str = new MagicString(code)
-		str.trim(';').remove(0, 4)
+
+		str.trim(';').remove(0, trim)
+
 		return {
 			code: str.toString(),
 			map: str.generateMap({ hires: true }),
 		}
 	}
-}
+})
 
 const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 	{
@@ -55,6 +57,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 		plugins: [
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
+			terser({ ...terserOptions }),
+			reduceImpact(0),
 			bundleSize()
 		],
 	},
@@ -71,6 +75,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 		plugins: [
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
+			terser({ ...terserOptions }),
+			reduceImpact(13),
 			bundleSize()
 		],
 	},
@@ -89,7 +95,7 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
 			terser({ ...terserOptions }),
-			reduceImpact,
+			reduceImpact(4),
 			bundleSize(),
 		]
 	},
@@ -106,6 +112,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 		plugins: [
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
+			terser({ ...terserOptions }),
+			reduceImpact(0),
 			bundleSize()
 		],
 	},
@@ -122,6 +130,8 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 		plugins: [
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
+			terser({ ...terserOptions }),
+			reduceImpact(13),
 			bundleSize()
 		],
 	},
@@ -140,7 +150,7 @@ const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([
 			resolveJsTsExtension,
 			babel({ ...babelOptions }),
 			terser({ ...terserOptions }),
-			reduceImpact,
+			reduceImpact(4),
 			bundleSize(),
 		]
 	},