Add parsing from bytes, with rust-encoding.

New dependency: https://github.com/lifthrasiir/rust-encoding

Author: SimonSapin

from_bytes.rs

@@ -0,0 +1,102 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use std::str;
+
+use encoding::label::encoding_from_whatwg_label;
+use encoding::all::UTF_8;
+use encoding::Encoding;
+use encoding::DecodeReplace;
+use encoding::decode;
+
+use tokenizer::{tokenize, Tokenizer};
+use parser::{parse_stylesheet_rules, StylesheetParser};
+
+
+/// Determine the character encoding of a CSS stylesheet and decode it.
+///
+/// This is based on the presence of a :abbr:`BOM (Byte Order Mark)`,
+/// an `@charset` rule,
+/// and encoding meta-information.
+///
+/// :param css_bytes: A byte string.
+/// :param protocol_encoding:
+///     The encoding label, if any, defined by HTTP or equivalent protocol.
+///     (e.g. via the `charset` parameter of the `Content-Type` header.)
+/// :param environment_encoding:
+///     An optional `Encoding` object
+///     for the `environment encoding
+///     <http://www.w3.org/TR/css-syntax/#environment-encoding>`_,
+///     if any.
+/// :returns:
+///     A 2-tuple of a decoded Unicode string
+///     and the `Encoding` object that was used.
+pub fn decode_stylesheet_bytes(css: &[u8], protocol_encoding_label: Option<&str>,
+                               environment_encoding: Option<&'static Encoding>)
+                            -> (~str, &'static Encoding) {
+    // http://dev.w3.org/csswg/css-syntax/#the-input-byte-stream
+    match protocol_encoding_label {
+        None => (),
+        Some(label) => match encoding_from_whatwg_label(label) {
+            None => (),
+            Some(fallback) => return decode_replace(css, fallback)
+        }
+    }
+    if css.starts_with("@charset \"".as_bytes()) {
+        // 10 is "@charset \"".len()
+        // 100 is arbitrary so that no encoding label is more than 100-10 bytes.
+        match css.slice(10, css.len().min(&100)).position_elem(&('"' as u8)) {
+            None => (),
+            Some(label_length)
+            => if css.slice_from(10 + label_length).starts_with("\";".as_bytes()) {
+                let label = css.slice(10, 10 + label_length);
+                let label = str::from_chars(label.iter().map(|&b| b as char).to_owned_vec());
+                match encoding_from_whatwg_label(label) {
+                    None => (),
+                    Some(fallback) => match fallback.name() {
+                        "utf-16be" | "utf-16le"
+                        => return decode_replace(css, UTF_8 as &'static Encoding),
+                        _ => return decode_replace(css, fallback),
+                    }
+                }
+            }
+        }
+    }
+    match environment_encoding {
+        None => (),
+        Some(fallback) => return decode_replace(css, fallback)
+    }
+    return decode_replace(css, UTF_8 as &'static Encoding)
+}
+
+
+#[inline]
+fn decode_replace(input: &[u8], fallback_encoding: &'static Encoding)-> (~str, &'static Encoding) {
+    let (result, used_encoding) = decode(input, DecodeReplace, fallback_encoding);
+    (result.unwrap(), used_encoding)
+}
+
+
+/// Parse stylesheet from bytes.
+///
+/// :param css_bytes: A byte string.
+/// :param protocol_encoding:
+///     The encoding label, if any, defined by HTTP or equivalent protocol.
+///     (e.g. via the `charset` parameter of the `Content-Type` header.)
+/// :param environment_encoding:
+///     An optional `Encoding` object
+///     for the `environment encoding
+///     <http://www.w3.org/TR/css-syntax/#environment-encoding>`_,
+///     if any.
+/// :returns:
+///     A 2-tuple of a Iterator<Result<Rule, SyntaxError>>
+///     and the `Encoding` object that was used.
+pub fn parse_stylesheet_rules_from_bytes(
+        css_bytes: &[u8], protocol_encoding_label: Option<&str>,
+        environment_encoding: Option<&'static Encoding>)
+     -> (StylesheetParser<Tokenizer>, &'static Encoding) {
+    let (css_unicode, encoding) = decode_stylesheet_bytes(
+        css_bytes, protocol_encoding_label, environment_encoding);
+    (parse_stylesheet_rules(tokenize(css_unicode)), encoding)
+}

lib.rs

@@ -7,17 +7,20 @@
 #[feature(globs, macro_rules)];
 
 extern mod extra;
+extern mod encoding;  // https://github.com/lifthrasiir/rust-encoding
 
 pub use tokenizer::tokenize;
 pub use parser::{parse_stylesheet_rules, parse_rule_list, parse_declaration_list,
                  parse_one_rule, parse_one_declaration, parse_one_component_value};
+pub use from_bytes::{decode_stylesheet_bytes, parse_stylesheet_rules_from_bytes};
 pub use color::{RGBA, Color, CurrentColor};
 pub use nth::parse_nth;
 pub use serializer::{ToCss, serialize_identifier, serialize_string};
 
 pub mod ast;
 mod tokenizer;
 mod parser;
+mod from_bytes;
 mod color;
 mod nth;
 mod serializer;

parser.rs

@@ -98,7 +98,9 @@ pub fn parse_one_component_value<T: Iterator<Node>>(mut iter: T)
 //  ***********  End of public API  ***********
 
 
-struct StylesheetParser<T>{ iter: T }
+// used in from_bytes.rs but not reexported in the crate top-level
+pub struct StylesheetParser<T>{ iter: T }
+
 struct RuleListParser<T>{ iter: T }
 struct DeclarationListParser<T>{ iter: T }
 

tests.rs

@@ -9,6 +9,8 @@ use extra::{tempfile, json};
 use extra::json::ToJson;
 use extra::test;
 
+use encoding::label::encoding_from_whatwg_label;
+
 use super::*;
 use ast::*;
 
@@ -56,23 +58,34 @@ fn assert_json_eq(results: json::Json, expected: json::Json, message: ~str) {
 }
 
 
-fn run_json_tests<T: ToJson>(json_data: &str, parse: &fn (input: ~str) -> T) {
+fn run_raw_json_tests(json_data: &str, run: &fn (json::Json, json::Json)) {
     let items = match json::from_str(json_data) {
         Ok(json::List(items)) => items,
         _ => fail!("Invalid JSON")
     };
     assert!(items.len() % 2 == 0);
-    let mut input: Option<~str> = None;
+    let mut input = None;
     for item in items.move_iter() {
         match (&input, item) {
-            (&None, json::String(string)) => input = Some(string),
+            (&None, json_obj) => input = Some(json_obj),
             (&Some(_), expected) => {
                 let input = input.take_unwrap();
+                run(input, expected)
+            },
+        };
+    }
+}
+
+
+fn run_json_tests<T: ToJson>(json_data: &str, parse: &fn (input: ~str) -> T) {
+    do run_raw_json_tests(json_data) |input, expected| {
+        match input {
+            json::String(input) => {
                 let result = parse(input.to_owned()).to_json();
                 assert_json_eq(result, expected, input);
             },
             _ => fail!("Unexpected JSON")
-        };
+        }
     }
 }
 
@@ -133,6 +146,43 @@ fn one_rule() {
 }
 
 
+#[test]
+fn stylesheet_from_bytes() {
+    do run_raw_json_tests(include_str!("css-parsing-tests/stylesheet_bytes.json"))
+    |input, expected| {
+        let map = match input {
+            json::Object(map) => map,
+            _ => fail!("Unexpected JSON")
+        };
+
+        let result = {
+            let css = get_string(map, &~"css_bytes").unwrap().iter().map(|c| {
+                assert!(c as u32 <= 0xFF);
+                c as u8
+            }).to_owned_vec();
+            let protocol_encoding_label = get_string(map, &~"protocol_encoding");
+            let environment_encoding = get_string(map, &~"environment_encoding")
+                .and_then(encoding_from_whatwg_label);
+
+            let (mut rules, used_encoding) = parse_stylesheet_rules_from_bytes(
+                css, protocol_encoding_label, environment_encoding);
+
+            (rules.to_owned_vec(), used_encoding.name().to_owned()).to_json()
+        };
+        assert_json_eq(result, expected, json::Object(map).to_str());
+    }
+
+    fn get_string<'a>(map: &'a json::Object, key: &~str) -> Option<&'a str> {
+        match map.find(key) {
+            Some(&json::String(ref s)) => Some(s.as_slice()),
+            Some(&json::Null) => None,
+            None => None,
+            _ => fail!("Unexpected JSON"),
+        }
+    }
+}
+
+
 fn run_color_tests(json_data: &str, to_json: &fn(result: Option<Color>) -> json::Json) {
     do run_json_tests(json_data) |input| {
         match parse_one_component_value(tokenize(input)) {