Skip to content

Commit 46e0e80

Browse files
SimonSapinSimonSapin
committed
Remove unicode-range tokens, per spec change.
w3c/csswg-drafts@01c55ee They’re being replaced by a micro-syntax / parsing algorithm based on other tokens: https://drafts.csswg.org/css-syntax/#urange
1 parent 87a1eee commit 46e0e80

File tree

5 files changed

+4
-160
lines changed

5 files changed

+4
-160
lines changed

‎src/css-parsing-tests/README.rst‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -228,10 +228,6 @@ Component values
228228
the value as a number, the type as the string ``"integer"`` or ``"number"``,
229229
and the unit as a string.
230230

231-
<unicode-range>
232-
Array of length 3: the string ``"unicode-range"``,
233-
followed by the *start* and *end* integers as two numbers.
234-
235231
<include-match>
236232
The string ``"~="``.
237233

‎src/css-parsing-tests/component_value_list.json‎

Lines changed: 0 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -325,80 +325,6 @@
325325
["dimension", "12", 12, "integer", "rêd"]
326326
],
327327

328-
"u+1 U+10 U+100 U+1000 U+10000 U+100000 U+1000000", [
329-
["unicode-range", 1, 1], " ",
330-
["unicode-range", 16, 16], " ",
331-
["unicode-range", 256, 256], " ",
332-
["unicode-range", 4096, 4096], " ",
333-
["unicode-range", 65536, 65536], " ",
334-
["unicode-range", 1048576, 1048576], " ",
335-
["unicode-range", 1048576, 1048576], ["number", "0", 0, "integer"]
336-
],
337-
338-
"u+? u+1? U+10? U+100? U+1000? U+10000? U+100000?", [
339-
["unicode-range", 0, 15], " ",
340-
["unicode-range", 16, 31], " ",
341-
["unicode-range", 256, 271], " ",
342-
["unicode-range", 4096, 4111], " ",
343-
["unicode-range", 65536, 65551], " ",
344-
["unicode-range", 1048576, 1048591], " ",
345-
["unicode-range", 1048576, 1048576], "?"
346-
],
347-
348-
"u+?? U+1?? U+10?? U+100?? U+1000?? U+10000??", [
349-
["unicode-range", 0, 255], " ",
350-
["unicode-range", 256, 511], " ",
351-
["unicode-range", 4096, 4351], " ",
352-
["unicode-range", 65536, 65791], " ",
353-
["unicode-range", 1048576, 1048831], " ",
354-
["unicode-range", 1048576, 1048591], "?"
355-
],
356-
357-
"u+??? U+1??? U+10??? U+100??? U+1000???", [
358-
["unicode-range", 0, 4095], " ",
359-
["unicode-range", 4096, 8191], " ",
360-
["unicode-range", 65536, 69631], " ",
361-
["unicode-range", 1048576, 1052671], " ",
362-
["unicode-range", 1048576, 1048831], "?"
363-
],
364-
365-
"u+???? U+1???? U+10???? U+100????", [
366-
["unicode-range", 0, 65535], " ",
367-
["unicode-range", 65536, 131071], " ",
368-
["unicode-range", 1048576, 1114111], " ",
369-
["unicode-range", 1048576, 1052671], "?"
370-
],
371-
372-
"u+????? U+1????? U+10?????", [
373-
["unicode-range", 0, 1048575], " ",
374-
["unicode-range", 1048576, 2097151], " ",
375-
["unicode-range", 1048576, 1114111], "?"
376-
],
377-
378-
"u+?????? U+1??????", [
379-
["unicode-range", 0, 16777215], " ",
380-
["unicode-range", 1048576, 2097151], "?"
381-
],
382-
383-
"u+20-3F U+100000-2 U+1000000-2 U+10-200000", [
384-
["unicode-range", 32, 63], " ",
385-
["unicode-range", 1048576, 2], " ",
386-
["unicode-range", 1048576, 1048576], ["number", "0", 0, "integer"],
387-
["number", "-2", -2, "integer"], " ",
388-
["unicode-range", 16, 2097152]
389-
],
390-
391-
"ù+12 Ü+12 u +12 U+ 12 U+12 - 20 U+1?2 U+1?-50 U+1- 2", [
392-
["ident", "ù"], ["number", "+12", 12, "integer"], " ",
393-
["ident", "Ü"], ["number", "+12", 12, "integer"], " ",
394-
["ident", "u"], " ", ["number", "+12", 12, "integer"], " ",
395-
["ident", "U"], "+", " ", ["number", "12", 12, "integer"], " ",
396-
["unicode-range", 18, 18], " ", "-", " ", ["number", "20", 20, "integer"], " ",
397-
["unicode-range", 16, 31], ["number", "2", 2, "integer"], " ",
398-
["unicode-range", 16, 31], ["number", "-50", -50, "integer"], " ",
399-
["unicode-range", 1, 1], "-", " ", ["number", "2", 2, "integer"]
400-
],
401-
402328
"~=|=^=$=*=||<!------> |/**/| ~/**/=", [
403329
"~=", "|=", "^=", "$=", "*=", "||", "<!--", ["ident", "----"], ">",
404330
" ", "|", "|", " ", "~", "="

‎src/serializer.rs‎

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
44

55
use std::ascii::AsciiExt;
6-
use std::cmp;
76
use std::fmt::{self, Write};
87

98
use super::{Token, NumericValue, PercentageValue};
@@ -110,28 +109,6 @@ impl<'a> ToCss for Token<'a> {
110109
}
111110
},
112111

113-
Token::UnicodeRange(start, end) => {
114-
try!(dest.write_str("U+"));
115-
let bits = cmp::min(start.trailing_zeros(), (!end).trailing_zeros());
116-
let question_marks = bits / 4;
117-
let bits = question_marks * 4;
118-
let truncated_start = start >> bits;
119-
let truncated_end = end >> bits;
120-
if truncated_start == truncated_end {
121-
if truncated_start != 0 {
122-
try!(write!(dest, "{:X}", truncated_start));
123-
}
124-
for _ in 0..question_marks {
125-
try!(dest.write_str("?"));
126-
}
127-
} else {
128-
try!(write!(dest, "{:X}", start));
129-
if end != start {
130-
try!(write!(dest, "-{:X}", end));
131-
}
132-
}
133-
}
134-
135112
Token::WhiteSpace(content) => try!(dest.write_str(content)),
136113
Token::Comment(content) => try!(write!(dest, "/*{}*/", content)),
137114
Token::Colon => try!(dest.write_str(":")),
@@ -343,17 +320,14 @@ impl TokenSerializationType {
343320
match self.0 {
344321
Ident => matches!(other.0,
345322
Ident | Function | UrlOrBadUrl | DelimMinus | Number | Percentage | Dimension |
346-
UnicodeRange | CDC | OpenParen),
323+
CDC | OpenParen),
347324
AtKeywordOrHash | Dimension => matches!(other.0,
348325
Ident | Function | UrlOrBadUrl | DelimMinus | Number | Percentage | Dimension |
349-
UnicodeRange | CDC),
326+
CDC),
350327
DelimHash | DelimMinus | Number => matches!(other.0,
351-
Ident | Function | UrlOrBadUrl | DelimMinus | Number | Percentage | Dimension |
352-
UnicodeRange),
328+
Ident | Function | UrlOrBadUrl | DelimMinus | Number | Percentage | Dimension),
353329
DelimAt => matches!(other.0,
354-
Ident | Function | UrlOrBadUrl | DelimMinus | UnicodeRange),
355-
UnicodeRange => matches!(other.0,
356-
Ident | Function | Number | Percentage | Dimension | DelimQuestion),
330+
Ident | Function | UrlOrBadUrl | DelimMinus),
357331
DelimDotOrPlus => matches!(other.0, Number | Percentage | Dimension),
358332
DelimAssorted | DelimAsterisk => matches!(other.0, DelimEquals),
359333
DelimBar => matches!(other.0, DelimEquals | DelimBar | DashMatch),
@@ -372,7 +346,6 @@ enum TokenSerializationTypeVariants {
372346
Number,
373347
Dimension,
374348
Percentage,
375-
UnicodeRange,
376349
UrlOrBadUrl,
377350
Function,
378351
Ident,
@@ -417,7 +390,6 @@ impl<'a> Token<'a> {
417390
Token::Number(_) => Number,
418391
Token::Percentage(_) => Percentage,
419392
Token::Dimension(..) => Dimension,
420-
Token::UnicodeRange(..) => UnicodeRange,
421393
Token::WhiteSpace(_) => WhiteSpace,
422394
Token::Comment(_) => DelimSlash,
423395
Token::DashMatch => DashMatch,

‎src/tests.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -761,8 +761,6 @@ fn one_component_value_to_json(token: Token, input: &mut Parser) -> Json {
761761
v
762762
}),
763763

764-
Token::UnicodeRange(start, end) => JArray!["unicode-range", start, end],
765-
766764
Token::WhiteSpace(_) => " ".to_json(),
767765
Token::Comment(_) => "/**/".to_json(),
768766
Token::Colon => ":".to_json(),

‎src/tokenizer.rs‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,6 @@ pub enum Token<'a> {
6262
/// A [`<dimension-token>`](https://drafts.csswg.org/css-syntax/#dimension-token-diagram)
6363
Dimension(NumericValue, Cow<'a, str>),
6464

65-
/// A [`<unicode-range-token>`](https://drafts.csswg.org/css-syntax/#unicode-range-token-diagram)
66-
///
67-
/// Components are the start and end code points, respectively.
68-
///
69-
/// The tokenizer only reads up to 6 hex digit (up to 0xFF_FFFF),
70-
/// but does not check that code points are within the range of Unicode (up to U+10_FFFF).
71-
UnicodeRange(u32, u32),
72-
7365
/// A [`<whitespace-token>`](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram)
7466
WhiteSpace(&'a str),
7567

@@ -522,13 +514,6 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
522514
if is_ident_start(tokenizer) { AtKeyword(consume_name(tokenizer)) }
523515
else { Delim('@') }
524516
},
525-
b'u' | b'U' => {
526-
if tokenizer.has_at_least(2)
527-
&& tokenizer.byte_at(1) == b'+'
528-
&& matches!(tokenizer.byte_at(2), b'0'...b'9' | b'a'...b'f' | b'A'...b'F' | b'?')
529-
{ consume_unicode_range(tokenizer) }
530-
else { consume_ident_like(tokenizer) }
531-
},
532517
b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => { consume_ident_like(tokenizer) },
533518
b'[' => { tokenizer.advance(1); SquareBracketBlock },
534519
b'\\' => {
@@ -1017,39 +1002,6 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
10171002
}
10181003
}
10191004

1020-
1021-
1022-
fn consume_unicode_range<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
1023-
tokenizer.advance(2); // Skip U+
1024-
let (hex_value, hex_digits) = consume_hex_digits(tokenizer);
1025-
let max_question_marks = 6 - hex_digits;
1026-
let mut question_marks = 0;
1027-
while question_marks < max_question_marks && !tokenizer.is_eof()
1028-
&& tokenizer.next_byte_unchecked() == b'?' {
1029-
question_marks += 1;
1030-
tokenizer.advance(1)
1031-
}
1032-
let start;
1033-
let end;
1034-
if question_marks > 0 {
1035-
start = hex_value << (question_marks * 4);
1036-
end = ((hex_value + 1) << (question_marks * 4)) - 1;
1037-
} else {
1038-
start = hex_value;
1039-
if tokenizer.has_at_least(1) &&
1040-
tokenizer.next_byte_unchecked() == b'-' &&
1041-
matches!(tokenizer.byte_at(1), b'0'...b'9' | b'A'...b'F' | b'a'...b'f') {
1042-
tokenizer.advance(1);
1043-
let (hex_value, _) = consume_hex_digits(tokenizer);
1044-
end = hex_value;
1045-
} else {
1046-
end = start;
1047-
}
1048-
}
1049-
UnicodeRange(start, end)
1050-
}
1051-
1052-
10531005
// (value, number of digits up to 6)
10541006
fn consume_hex_digits<'a>(tokenizer: &mut Tokenizer<'a>) -> (u32, u32) {
10551007
let mut value = 0;

0 commit comments

Comments
 (0)