Add a Comma token.

SimonSapin · SimonSapin · commit c9a49646338a · 2013-08-02T15:36:13.000+01:00
diff --git a/ast.rs b/ast.rs
@@ -36,6 +36,7 @@ pub enum ComponentValue {
     WhiteSpace,
     Colon,  // :
     Semicolon,  // ;
+    Comma,  // ,
     IncludeMath, // ~=
     DashMatch, // |=
     PrefixMatch, // ^=
diff --git a/tests.rs b/tests.rs
@@ -312,6 +312,7 @@ impl ToJson for ComponentValue {
             WhiteSpace => JString(~" "),
             Colon => JString(~":"),
             Semicolon => JString(~";"),
+            Comma => JString(~","),
             IncludeMath => JString(~"~="),
             DashMatch => JString(~"|="),
             PrefixMatch => JString(~"^="),
diff --git a/tokenizer.rs b/tokenizer.rs
@@ -106,6 +106,7 @@ pub fn next_component_value(parser: &mut Parser) -> Option<(ComponentValue, Sour
                 Delim(c)
             }
         },
+        ',' => { parser.position += 1; Comma },
         '-' => {
             if (
                 parser.position + 1 < parser.length

Original file line number	Diff line number	Diff line change
`@@ -106,6 +106,7 @@ pub fn next_component_value(parser: &mut Parser) -> Option<(ComponentValue, Sour`
`106`	`106`	`Delim(c)`
`107`	`107`	`}`
`108`	`108`	`},`
	`109`	`+ ',' => { parser.position += 1; Comma },`
`109`	`110`	`'-' => {`
`110`	`111`	`if (`
`111`	`112`	`parser.position + 1 < parser.length`