Fallback to RegEx based parser when using custom transformers or extractors (tailwindlabs#11335)

* handle custom `transformer` and `extractor` implementations

Right now the Rust based parser can't work with custom `transfomers` or
`extractors`.

In a perfect world we can implement as many custom parsers/extractors in
Rust such that we don't need this at all. In an almost perfect world we
can pass the transformer and extractor to the Rust based parser and call
the callback functions to handle all of this. This is probably what we
are going to do in the future but this requires more work to make sure
that:

1. It works as expected
2. Doesn't result in (major) performance issues

Since it currently doesn't work with the Rust based parser, we can
implement a fix for this in the meantime before we reach the "perfect"
solution.

One solution to this problem is to check if we do have a custom
transformer or a custom extractor and if we do, then we can bail on the
Rust parser completely and just use the current regex based parser.

An alternative solution, the solution implemented here, is that we group
the `changedContent` into 2 buckets. The bucket where we rely on the
default transformer and extractor and a bucket where a custom
transformer or extractor is used.

Then, the bucket where we use the default transformer and extractor can
still rely on the way faster Rust based parser. For the other bucket we
fallback to the regex based parser.

The nice part about this is that we can use both parsers at the same
time, and the majority of the use cases should use the faster Rust based
parser.

* update changelog

Author: RobinMalfait

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Ensure `repeating-conic-gradient` is detected as an image ([#11180](https://github.com/tailwindlabs/tailwindcss/pull/11180))
 - Remove `autoprefixer` dependency ([#11315](https://github.com/tailwindlabs/tailwindcss/pull/11315))
 - Fix source maps issue resulting in a crash ([#11319](https://github.com/tailwindlabs/tailwindcss/pull/11319))
+- Fallback to RegEx based parser when using custom transformers or extractors  ([#11335](https://github.com/tailwindlabs/tailwindcss/pull/11335))
 
 ### Added
 

src/lib/expandTailwindAtRules.js

@@ -26,7 +26,10 @@ function getExtractor(context, fileExtension) {
     extractors[fileExtension] ||
     extractors.DEFAULT ||
     builtInExtractors[fileExtension] ||
-    builtInExtractors.DEFAULT(context)
+    // Because we call `DEFAULT(context)`, the returning function is always a new function without a
+    // stable identity. Marking it with `DEFAULT_EXTRACTOR` allows us to check if it is the default
+    // extractor without relying on the function identity.
+    Object.assign(builtInExtractors.DEFAULT(context), { DEFAULT_EXTRACTOR: true })
   )
 }
 
@@ -133,20 +136,35 @@ export default function expandTailwindAtRules(context) {
     env.DEBUG && console.time('Reading changed files')
 
     if (flagEnabled(context.tailwindConfig, 'oxideParser')) {
-      // TODO: Pass through or implement `extractor`
-      for (let candidate of parseCandidateStrings(
-        context.changedContent,
-        IO.Parallel | Parsing.Parallel
-        // Object.assign({}, builtInTransformers, context.tailwindConfig.content.transform)
-      )) {
-        candidates.add(candidate)
+      let rustParserContent = []
+      let regexParserContent = []
+
+      for (let item of context.changedContent) {
+        let transformer = getTransformer(context.tailwindConfig, item.extension)
+        let extractor = getExtractor(context, item.extension)
+
+        if (transformer === builtInTransformers.DEFAULT && extractor?.DEFAULT_EXTRACTOR === true) {
+          rustParserContent.push(item)
+        } else {
+          regexParserContent.push([item, { transformer, extractor }])
+        }
+      }
+
+      if (rustParserContent.length > 0) {
+        for (let candidate of parseCandidateStrings(
+          rustParserContent,
+          IO.Parallel | Parsing.Parallel
+        )) {
+          candidates.add(candidate)
+        }
       }
 
-      // for (let { file, content, extension } of context.changedContent) {
-      //   let transformer = getTransformer(context.tailwindConfig, extension)
-      //   let extractor = getExtractor(context, extension)
-      //   getClassCandidatesOxide(file, transformer(content), extractor, candidates, seen)
-      // }
+      if (regexParserContent.length > 0) {
+        for (let [{ file, content }, { transformer, extractor }] of regexParserContent) {
+          content = file ? fs.readFileSync(file, 'utf8') : content
+          getClassCandidates(transformer(content), extractor, candidates, seen)
+        }
+      }
     } else {
       for (let { file, content, extension } of context.changedContent) {
         let transformer = getTransformer(context.tailwindConfig, extension)
@@ -165,15 +183,16 @@ export default function expandTailwindAtRules(context) {
 
     env.DEBUG && console.time('Generate rules')
     env.DEBUG && console.time('Sorting candidates')
-    let sortedCandidates = flagEnabled(context.tailwindConfig, 'oxideParser')
-      ? candidates
-      : new Set(
-          [...candidates].sort((a, z) => {
-            if (a === z) return 0
-            if (a < z) return -1
-            return 1
-          })
-        )
+    // TODO: only sort if we are not using the oxide parser (flagEnabled(context.tailwindConfig,
+    // 'oxideParser')) AND if we got all the candidates form the oxideParser alone. This will not
+    // be the case currently if you have custom transformers / extractors.
+    let sortedCandidates = new Set(
+      [...candidates].sort((a, z) => {
+        if (a === z) return 0
+        if (a < z) return -1
+        return 1
+      })
+    )
     env.DEBUG && console.timeEnd('Sorting candidates')
     generateRules(sortedCandidates, context)
     env.DEBUG && console.timeEnd('Generate rules')