Extract source map URL from directive comments

Change the parser to extract the source map URL from directive comments.
The relevant spec is here:

https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.lmz475t4mvbx

This is part of similar work being done in M-C in
https://bugzilla.mozilla.org/show_bug.cgi?id=1388855

Author: tromey

src/parser.rs

@@ -268,6 +268,11 @@ impl<'i: 't, 't> Parser<'i, 't> {
         self.input.tokenizer.current_source_location()
     }
 
+    /// The source map URL.
+    pub fn current_source_map_url(&self) -> Option<&str> {
+        self.input.tokenizer.current_source_map_url()
+    }
+
     /// Return the current internal state of the parser (including position within the input).
     ///
     /// This state can later be restored with the `Parser::reset` method.

src/size_of_tests.rs

@@ -36,8 +36,8 @@ size_of_test!(token, Token, 32);
 size_of_test!(std_cow_str, Cow<'static, str>, 32);
 size_of_test!(cow_rc_str, CowRcStr, 16);
 
-size_of_test!(tokenizer, ::tokenizer::Tokenizer, 40);
-size_of_test!(parser_input, ::parser::ParserInput, 112);
+size_of_test!(tokenizer, ::tokenizer::Tokenizer, 56);
+size_of_test!(parser_input, ::parser::ParserInput, 128);
 size_of_test!(parser, ::parser::Parser, 16);
 size_of_test!(source_position, ::SourcePosition, 8);
 size_of_test!(parser_state, ::ParserState, 24);

src/tests.rs

@@ -979,3 +979,28 @@ fn parse_entirely_reports_first_error() {
     let result: Result<(), _> = parser.parse_entirely(|_| Err(ParseError::Custom(E::Foo)));
     assert_eq!(result, Err(ParseError::Custom(E::Foo)));
 }
+
+#[test]
+fn parse_comments() {
+    let tests = vec![
+        ("/*# sourceMappingURL=here*/", Some("here")),
+        ("/*# sourceMappingURL=here  */", Some("here")),
+        ("/*@ sourceMappingURL=here*/", Some("here")),
+        ("/*@ sourceMappingURL=there*/ /*# sourceMappingURL=here*/", Some("here")),
+        ("/*# sourceMappingURL=here there  */", Some("here")),
+        ("/*# sourceMappingURL=  here  */", Some("")),
+        ("/*# sourceMappingURL=*/", Some("")),
+        ("/*# sourceMappingUR=here  */", None),
+        ("/*! sourceMappingURL=here  */", None),
+        ("/*# sourceMappingURL = here  */", None),
+        ("/*   # sourceMappingURL=here   */", None)
+    ];
+
+    for test in tests {
+        let mut input = ParserInput::new(test.0);
+        let mut parser = Parser::new(&mut input);
+        while let Ok(_) = parser.next_including_whitespace() {
+        }
+        assert_eq!(parser.current_source_map_url(), test.1);
+    }
+}

src/tokenizer.rs

@@ -209,6 +209,7 @@ pub struct Tokenizer<'a> {
     current_line_number: u32,
     var_functions: SeenStatus,
     viewport_percentages: SeenStatus,
+    source_map_url: Option<&'a str>,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq)]
@@ -234,6 +235,7 @@ impl<'a> Tokenizer<'a> {
             current_line_number: first_line_number,
             var_functions: SeenStatus::DontCare,
             viewport_percentages: SeenStatus::DontCare,
+            source_map_url: None,
         }
     }
 
@@ -300,6 +302,11 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
+    #[inline]
+    pub fn current_source_map_url(&self) -> Option<&'a str> {
+        self.source_map_url
+    }
+
     #[inline]
     pub fn state(&self) -> ParserState {
         ParserState {
@@ -597,6 +604,53 @@ fn consume_whitespace<'a>(tokenizer: &mut Tokenizer<'a>, newline: bool, is_cr: b
 fn consume_comment<'a>(tokenizer: &mut Tokenizer<'a>) -> &'a str {
     tokenizer.advance(2);  // consume "/*"
     let start_position = tokenizer.position();
+
+    // If there is a source map directive, extract the URL.
+    if tokenizer.starts_with(b"#") || tokenizer.starts_with(b"@") {
+        tokenizer.advance(1);
+        let directive_text = b" sourceMappingURL=";
+        if tokenizer.starts_with(directive_text) {
+            tokenizer.advance(directive_text.len());
+
+            // Scan for the next whitespace.
+            let url_start_position = tokenizer.position();
+            let mut url_end_position = tokenizer.position();
+            while !tokenizer.is_eof() {
+                match_byte! { tokenizer.next_byte_unchecked(),
+                    b' ' | b'\t' => {
+                        url_end_position = tokenizer.position();
+                        tokenizer.advance(1);
+                        break
+                    }
+                    b'\n' | b'\x0C' => {
+                        url_end_position = tokenizer.position();
+                        tokenizer.advance(1);
+                        tokenizer.seen_newline(false);
+                        break
+                    }
+                    b'\r' => {
+                        url_end_position = tokenizer.position();
+                        tokenizer.advance(1);
+                        tokenizer.seen_newline(true);
+                        break
+                    }
+                    b'*' => {
+                        if tokenizer.starts_with(b"*/") {
+                            // End of comment, so let the later loop finish up.
+                            url_end_position = tokenizer.position();
+                            break
+                        }
+                        tokenizer.advance(1);
+                    }
+                    _ => {
+                        tokenizer.advance(1);
+                    }
+                }
+            }
+            tokenizer.source_map_url = Some(tokenizer.slice(url_start_position..url_end_position))
+        }
+    }
+
     while !tokenizer.is_eof() {
         match_byte! { tokenizer.next_byte_unchecked(),
             b'*' => {