Cleanups and test utf8 special characters in paths

Author: philipp-spiess

crates/oxide/src/extractor/pre_processors/ruby.rs

@@ -3,7 +3,7 @@
 use crate::cursor;
 use crate::extractor::bracket_stack;
 use crate::extractor::pre_processors::pre_processor::PreProcessor;
-use crate::pre_process_input;
+use crate::scanner::pre_process_input;
 use bstr::ByteSlice;
 use fancy_regex::Regex;
 use std::sync;

crates/oxide/src/scanner/sources.rs

@@ -147,17 +147,18 @@ impl PublicSourceEntry {
 
         let base: PathBuf = self.base.clone().into();
         let base = match static_part {
-            Some(static_part) => base.join(static_part),
-            None => base,
-        };
-
-        // TODO: If the base does not exist on disk, try removing the last slash and try again.
-        let base = match dunce::canonicalize(&base) {
-            Ok(base) => base,
-            Err(err) => {
-                event!(tracing::Level::ERROR, "Failed to resolve glob: {:?}", err);
-                return;
+            Some(static_part) => {
+                // TODO: If the base does not exist on disk, try removing the last slash and try
+                // again.
+                match dunce::canonicalize(base.join(static_part)) {
+                    Ok(base) => base,
+                    Err(err) => {
+                        event!(tracing::Level::ERROR, "Failed to resolve glob: {:?}", err);
+                        return;
+                    }
+                }
             }
+            None => base,
         };
 
         let pattern = match dynamic_part {

crates/oxide/tests/scanner.rs

@@ -1585,4 +1585,30 @@ mod scanner {
 
         assert!(candidates.is_empty());
     }
+
+    #[test]
+    fn test_works_with_utf8_special_character_paths() {
+        let ScanResult {
+            candidates,
+            files,
+            globs,
+            normalized_sources,
+        } = scan_with_globs(
+            &[
+                ("src/💩.js", "content-['src/💩.js']"),
+                ("src/🤦‍♂️.tsx", "content-['src/🤦‍♂️.tsx']"),
+                ("src/🤦‍♂️/foo.tsx", "content-['src/🤦‍♂️/foo.tsx']"),
+            ],
+            vec!["@source '**/*'", "@source not 'src/🤦‍♂️'"],
+        );
+
+        assert_eq!(
+            candidates,
+            vec!["content-['src/💩.js']", "content-['src/🤦‍♂️.tsx']"]
+        );
+
+        assert_eq!(files, vec!["src/💩.js", "src/🤦‍♂️.tsx"]);
+        assert_eq!(globs, vec!["*", "src/*/*.{aspx,astro,cjs,cts,eex,erb,gjs,gts,haml,handlebars,hbs,heex,html,jade,js,jsx,liquid,md,mdx,mjs,mts,mustache,njk,nunjucks,php,pug,py,razor,rb,rhtml,rs,slim,svelte,tpl,ts,tsx,twig,vue}"]);
+        assert_eq!(normalized_sources, vec!["**/*"]);
+    }
 }