fix: utf-8 characters support, emoji also 😸.

Author: evilebottnawi

lib/processCss.js

@@ -1,9 +1,8 @@
 /*
-	MIT License http://www.opensource.org/licenses/mit-license.php
-	Author Tobias Koppers @sokra
-*/
+ MIT License http://www.opensource.org/licenses/mit-license.php
+ Author Tobias Koppers @sokra
+ */
 var formatCodeFrame = require("babel-code-frame");
-var Tokenizer = require("css-selector-tokenizer");
 var postcss = require("postcss");
 var loaderUtils = require("loader-utils");
 var assign = require("object-assign");
@@ -42,15 +41,19 @@ var parserPlugin = postcss.plugin("css-loader-parser", function(options) {
 
 		if(options.import) {
 			css.walkAtRules(/import/i, function(rule) {
-				var values = Tokenizer.parseValues(rule.params);
-				var url = values.nodes[0].nodes[0];
-				if(url.type === "url") {
-					url = url.url;
-				} else if(url.type === "string") {
-					url = url.value;
+				var tokens = valueParser(rule.params);
+				var importNode = tokens.nodes[0];
+				var url = null;
+				if (importNode.type === "function" && importNode.value === 'url') {
+					url = importNode.nodes[0].value;
+				} else if(importNode.type === "string") {
+					url = importNode.value;
 				} else throw rule.error("Unexpected format" + rule.params);
-				values.nodes[0].nodes.shift();
-				var mediaQuery = Tokenizer.stringifyValues(values);
+				if (!url) {
+					return;
+				}
+				tokens.nodes.shift();
+				var mediaQuery = tokens.toString().trim();
 				if(loaderUtils.isUrlRequest(url, options.root) && options.mode === "global") {
 					url = loaderUtils.urlToRequest(url, options.root);
 				}
@@ -86,45 +89,45 @@ var parserPlugin = postcss.plugin("css-loader-parser", function(options) {
 			exports[exportName] = replaceImportsInString(exports[exportName]);
 		});
 
-		function processNode(item) {
-			switch (item.type) {
-				case "value":
-					item.nodes.forEach(processNode);
-					break;
-				case "nested-item":
-					item.nodes.forEach(processNode);
-					break;
-				case "item":
-					var importIndex = imports["$" + item.name];
+		css.walkDecls(function(decl) {
+			var tokens = valueParser(decl.value);
+			tokens.walk(function (node) {
+				if (node.type === 'word') {
+					var importIndex = imports["$" + node.value];
 					if (typeof importIndex === "number") {
-						item.name = "___CSS_LOADER_IMPORT___" + importIndex + "___";
+						node.value = "___CSS_LOADER_IMPORT___" + importIndex + "___";
 					}
-					break;
-				case "url":
-					if (options.url && !/^#/.test(item.url) && loaderUtils.isUrlRequest(item.url, options.root)) {
-						// Don't remove quotes around url when contain space
-						if (item.url.indexOf(" ") === -1) {
-							item.stringType = "";
-						}
-						delete item.innerSpacingBefore;
-						delete item.innerSpacingAfter;
-						var url = item.url;
-						item.url = "___CSS_LOADER_URL___" + urlItems.length + "___";
-						urlItems.push({
-							url: url
-						});
+				}
+
+				if (options.url && node.type === 'function' && node.value === 'url') {
+					delete node.before;
+					delete node.after;
+					if (!node.nodes) {
+						return;
 					}
-					break;
-			}
-		}
 
-		css.walkDecls(function(decl) {
-			var values = Tokenizer.parseValues(decl.value);
-			values.nodes.forEach(function(value) {
-				value.nodes.forEach(processNode);
+					node.nodes.forEach(function (nestedNode) {
+						if (nestedNode.type !== 'word' && nestedNode.type !== 'string') {
+							return;
+						}
+						var value = nestedNode.value;
+						if (!/^#/.test(value) && loaderUtils.isUrlRequest(value, options.root)) {
+							// Don't remove quotes around url when contain space
+							if (value.indexOf(" ") === -1) {
+								nestedNode.quote = "";
+							}
+							var url = value;
+							nestedNode.value = "___CSS_LOADER_URL___" + urlItems.length + "___";
+							urlItems.push({
+								url: url
+							});
+						}
+					})
+				}
 			});
-			decl.value = Tokenizer.stringifyValues(values);
+			decl.value = tokens.toString();
 		});
+
 		css.walkAtRules(function(atrule) {
 			if(typeof atrule.params === "string") {
 				atrule.params = replaceImportsInString(atrule.params);

package.json

@@ -13,7 +13,6 @@
   ],
   "dependencies": {
     "babel-code-frame": "^6.11.0",
-    "css-selector-tokenizer": "^0.7.0",
     "cssnano": ">=2.6.1 <4",
     "loader-utils": "^1.0.2",
     "lodash.camelcase": "^4.3.0",

test/importTest.js

@@ -9,23 +9,29 @@ describe("import", function() {
 	], "", {
 		"./test.css": [[2, ".test{a: b}", ""]]
 	});
-	test("import camelcase", "@IMPORT url(test.css);\n.class { a: b c d; }", [
+	test("import 2", "@import url('test.css');\n.class { a: b c d; }", [
+		[2, ".test{a: b}", "screen"],
+		[1, ".class { a: b c d; }", ""]
+	], "", {
+		"./test.css": [[2, ".test{a: b}", "screen"]]
+	});
+	test("import non ascii characters (emoji)", "@import url(😸.css);\n.class { a: b c d; }", [
 		[2, ".test{a: b}", ""],
 		[1, ".class { a: b c d; }", ""]
 	], "", {
-		"./test.css": [[2, ".test{a: b}", ""]]
+		"./😸.css": [[2, ".test{a: b}", ""]]
 	});
-	test("import with string", "@import \"test.css\";\n.class { a: b c d; }", [
+	test("import camelcase", "@IMPORT url(test.css);\n.class { a: b c d; }", [
 		[2, ".test{a: b}", ""],
 		[1, ".class { a: b c d; }", ""]
 	], "", {
 		"./test.css": [[2, ".test{a: b}", ""]]
 	});
-	test("import 2", "@import url('test.css');\n.class { a: b c d; }", [
-		[2, ".test{a: b}", "screen"],
+	test("import with string", "@import \"test.css\";\n.class { a: b c d; }", [
+		[2, ".test{a: b}", ""],
 		[1, ".class { a: b c d; }", ""]
 	], "", {
-		"./test.css": [[2, ".test{a: b}", "screen"]]
+		"./test.css": [[2, ".test{a: b}", ""]]
 	});
 	test("import with media", "@import url('~test/css') screen and print;\n.class { a: b c d; }", [
 		[3, ".test{a: b}", "((min-width: 100px)) and (screen and print)"],

test/simpleTest.js

@@ -29,6 +29,9 @@ describe("simple", function() {
 	/*test("escape characters (two)", ".class { content: \"\\F10C \\F10D\" }", [
 		[1, ".class { content: \"\\F10C \\F10D\" }", ""]
 	]);*/
+	/*test("non-ascii characters", "body { font-family: '😸'; }", [
+		[1, "body { font-family: '😸'; }", ""]
+	]);*/
 	testMinimize("minimized simple", ".class { a: b c d; }", [
 		[1, ".class{a:b c d}", ""]
 	]);