Fix a bug in unicode-range tokenization.

Author: SimonSapin

src/css-parsing-tests/component_value_list.json

@@ -388,14 +388,15 @@
     ["unicode-range", 16, 2097152]
 ],
 
-"ù+12 Ü+12 u +12 U+ 12 U+12 - 20 U+1?2 U+1?-50", [
+"ù+12 Ü+12 u +12 U+ 12 U+12 - 20 U+1?2 U+1?-50 U+1- 2", [
     ["ident", "ù"], ["number", "+12", 12, "integer"], " ",
     ["ident", "Ü"], ["number", "+12", 12, "integer"], " ",
     ["ident", "u"], " ", ["number", "+12", 12, "integer"], " ",
     ["ident", "U"], "+", " ", ["number", "12", 12, "integer"], " ",
     ["unicode-range", 18, 18], " ", "-", " ", ["number", "20", 20, "integer"], " ",
     ["unicode-range", 16, 31], ["number", "2", 2, "integer"], " ",
-    ["unicode-range", 16, 31], ["number", "-50", -50, "integer"]
+    ["unicode-range", 16, 31], ["number", "-50", -50, "integer"], " ",
+    ["unicode-range", 1, 1], "-", " ", ["number", "2", 2, "integer"]
 ],
 
 "~=|=^=$=*=||<!------> |/**/| ~/**/=", [

src/tokenizer.rs

@@ -833,7 +833,9 @@ fn consume_unicode_range<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     } else {
         start = first;
         hex.truncate(0);
-        if !tokenizer.is_eof() && tokenizer.next_char() == '-' {
+        if tokenizer.has_at_least(1) &&
+           tokenizer.next_char() == '-' &&
+           matches!(tokenizer.char_at(1), '0'...'9' | 'A'...'F' | 'a'...'f') {
             tokenizer.advance(1);
             while hex.len() < 6 && !tokenizer.is_eof() {
                 let c = tokenizer.next_char();