Auto merge of #194 - servo:bad, r=emilio

Fix serialization of bad-string and bad-url tokens

This will help fix https://bugzilla.mozilla.org/show_bug.cgi?id=1396664.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/rust-cssparser/194)
<!-- Reviewable:end -->

Author: bors-servo

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cssparser"
-version = "0.20.0"
+version = "0.20.1"
 authors = [ "Simon Sapin <simon.sapin@exyr.org>" ]
 
 description = "Rust implementation of CSS Syntax Level 3"

src/serializer.rs

@@ -122,8 +122,20 @@ impl<'a> ToCss for Token<'a> {
             Token::SquareBracketBlock => dest.write_str("[")?,
             Token::CurlyBracketBlock => dest.write_str("{")?,
 
-            Token::BadUrl(_) => dest.write_str("url(<bad url>)")?,
-            Token::BadString(_) => dest.write_str("\"<bad string>\n")?,
+            Token::BadUrl(ref contents) => {
+                dest.write_str("url(")?;
+                dest.write_str(contents)?;
+                dest.write_char(')')?;
+            }
+            Token::BadString(ref value) => {
+                // During tokenization, an unescaped newline after a quote causes
+                // the token to be a BadString instead of a QuotedString.
+                // The BadString token ends just before the newline
+                // (which is in a separate WhiteSpace token),
+                // and therefore does not have a closing quote.
+                dest.write_char('"')?;
+                CssStringWriter::new(dest).write_str(value)?;
+            },
             Token::CloseParenthesis => dest.write_str(")")?,
             Token::CloseSquareBracket => dest.write_str("]")?,
             Token::CloseCurlyBracket => dest.write_str("}")?,

src/tests.rs

@@ -441,6 +441,26 @@ fn serializer(preserve_comments: bool) {
     });
 }
 
+#[test]
+fn serialize_bad_tokens() {
+    let mut input = ParserInput::new("url(foo\\) b\\)ar)'ba\\'\"z\n4");
+    let mut parser = Parser::new(&mut input);
+
+    let token = parser.next().unwrap().clone();
+    assert!(matches!(token, Token::BadUrl(_)));
+    assert_eq!(token.to_css_string(), "url(foo\\) b\\)ar)");
+
+    let token = parser.next().unwrap().clone();
+    assert!(matches!(token, Token::BadString(_)));
+    assert_eq!(token.to_css_string(), "\"ba'\\\"z");
+
+    let token = parser.next().unwrap().clone();
+    assert!(matches!(token, Token::Number { .. }));
+    assert_eq!(token.to_css_string(), "4");
+
+    assert!(parser.next().is_err());
+}
+
 #[test]
 fn serialize_current_color() {
     let c = Color::CurrentColor;

src/tokenizer.rs

@@ -1289,8 +1289,9 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
         while !tokenizer.is_eof() {
             match_byte! { tokenizer.next_byte_unchecked(),
                 b')' => {
+                    let contents = tokenizer.slice_from(start_pos).into();
                     tokenizer.advance(1);
-                    break
+                    return BadUrl(contents)
                 }
                 b'\\' => {
                     tokenizer.advance(1);