From fe1a7534f024fc53b4af935f16574fd0702633d3 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Thu, 4 Jun 2015 03:43:32 +0200 Subject: [PATCH 01/11] Fix tests --- test/fixtures/cwd.css | 4 ++-- test/fixtures/ignore.css | 2 +- test/index.js | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/fixtures/cwd.css b/test/fixtures/cwd.css index d46d79eb..bdc682c0 100755 --- a/test/fixtures/cwd.css +++ b/test/fixtures/cwd.css @@ -1,2 +1,2 @@ -@import "test/fixtures/imports/foo.css"; -@import "test/fixtures/imports/foo-recursive.css"; +@import "imports/foo.css"; +@import "imports/foo-recursive.css"; diff --git a/test/fixtures/ignore.css b/test/fixtures/ignore.css index 71a7951e..24265277 100644 --- a/test/fixtures/ignore.css +++ b/test/fixtures/ignore.css @@ -1,4 +1,4 @@ -@import "ignore.css" (min-width: 25em); +@import "imports/ignore.css" (min-width: 25em); @import "http://css"; @import "https://css"; @import 'http://css'; diff --git a/test/index.js b/test/index.js index 28683e78..ff3b2acd 100755 --- a/test/index.js +++ b/test/index.js @@ -17,6 +17,8 @@ function read(name) { function compareFixtures(t, name, msg, opts, postcssOpts) { opts = opts || {path: importsDir} + postcssOpts = postcssOpts || {} + postcssOpts.from = "test/fixtures/" + name + ".css" var actual = postcss() .use(atImport(opts)) .process(read("fixtures/" + name), postcssOpts) From f183e59de53855fed4de8c11bede256310897e5f Mon Sep 17 00:00:00 2001 From: andyjansson Date: Thu, 4 Jun 2015 04:11:07 +0200 Subject: [PATCH 02/11] Add cache --- .gitignore | 1 + README.md | 7 +++ index.js | 120 ++++++++++++++++++++++++++++++++++++++++++++------ test/index.js | 3 +- 4 files changed, 116 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 7ab649f4..f875ebf0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules test/fixtures/*.actual.css +test/cache \ No newline at end of file diff --git a/README.md b/README.md index 82b006ae..d819337f 100755 --- a/README.md +++ b/README.md @@ -93,6 +93,13 @@ Default: `process.cwd()` or _dirname of [the postcss `from`](https://github.com/ A string or an array of paths in where to look for files. _Note: nested `@import` will additionally benefit of the relative dirname of imported files._ +#### `cacheDir` + +Type: `String` +Default: `null` + +The location of the cache. When set, the compiled output will be stored in this directory. The cache will automatically be invalidated when a file in the dependency graph is modified. + #### `transform` Type: `Function` diff --git a/index.js b/index.js index a1405f88..fe26e9e4 100755 --- a/index.js +++ b/index.js @@ -68,9 +68,25 @@ function AtImport(options) { var hashFiles = {} - parseStyles(styles, options, insertRules, importedFiles, ignoredAtRules, null, hashFiles) + var cache; + if (options.cacheDir) { + if (!fs.existsSync(options.cacheDir)) { + fs.mkdirSync(options.cacheDir); + } + try { + cache = require(path.resolve(options.cacheDir, "imports.json")) + } catch (err) { + cache = {} + } + } + + parseStyles(styles, options, insertRules, importedFiles, ignoredAtRules, null, hashFiles, cache) addIgnoredAtRulesOnTop(styles, ignoredAtRules) + if (cache) { + fs.writeFileSync(path.resolve(options.cacheDir, "imports.json"), JSON.stringify(cache)) + } + if (typeof options.onImport === "function") { options.onImport(Object.keys(importedFiles)) } @@ -83,7 +99,16 @@ function AtImport(options) { * @param {Object} styles * @param {Object} options */ -function parseStyles(styles, options, cb, importedFiles, ignoredAtRules, media, hashFiles) { +function parseStyles(styles, options, cb, globallyImportedFiles, ignoredAtRules, media, hashFiles, cache) { + if (options.from && !media && cache && IsCached(options.from, options, cache)) { + var newStyles = readFromCache(options, options.from, cache) + var newNodes = newStyles.nodes + newNodes.forEach(function(node) {node.parent = styles}) + styles.source = newStyles.source + styles.nodes = newStyles.nodes + styles.semicolon = newStyles.semicolon + return + } var imports = [] styles.eachAtRule("import", function checkAtRule(atRule) { if (options.glob && glob.hasMagic(atRule.params)) { @@ -93,11 +118,72 @@ function parseStyles(styles, options, cb, importedFiles, ignoredAtRules, media, imports.push(atRule) } }) + + var locallyImportedFiles = [] imports.forEach(function(atRule) { helpers.try(function transformAtImport() { - readAtImport(atRule, options, cb, importedFiles, ignoredAtRules, media, hashFiles) + readAtImport(atRule, options, cb, globallyImportedFiles, locallyImportedFiles, ignoredAtRules, media, hashFiles, cache) }, atRule.source) }) + if (options.from && cache && !media) { + var cacheFilename; + if (locallyImportedFiles.length > 0) { + var css = styles.toResult().css + cacheFilename = path.resolve(options.cacheDir, hash(css) + ".css") + fs.writeFileSync(cacheFilename, css) + } + cache[options.from] = cache[options.from] || {}; + + cache[options.from].mtime = fs.statSync(options.from).mtime.getTime(); + cache[options.from].dependencies = locallyImportedFiles; + cache[options.from].cache = cacheFilename + } +} + +function IsCached(filename, options, cache) { + var importCache = cache[filename] + + if (!IsDependencyCached(filename, options, cache)) { + if (importCache && importCache.cache) { + fs.unlink(importCache.cache, function(err) { + if (err) { + throw err + } + }) + } + return false + } + + if (!importCache.cache) { + return false + } + + return true +} + +function readFromCache(options, filename, cache) { + var fileContent = fs.readFileSync(cache[filename].cache) + return postcss.parse(fileContent, options) +} + +function IsDependencyCached(filename, options, cache) { + if (!filename) { + return false + } + var importCache = cache[filename] + if (!importCache) { + return false + } + var mtime = fs.statSync(filename).mtime.getTime(); + if (mtime !== importCache.mtime) { + return false + } + for (var i = 0; i < importCache.dependencies.length; i++) { + if (!IsDependencyCached(importCache.dependencies[i], options, cache)) { + return false + } + } + return true } /** @@ -172,7 +258,7 @@ function addIgnoredAtRulesOnTop(styles, ignoredAtRules) { * @param {Object} atRule postcss atRule * @param {Object} options */ -function readAtImport(atRule, options, cb, importedFiles, ignoredAtRules, media, hashFiles) { +function readAtImport(atRule, options, cb, globallyImportedFiles, locallyImportedFiles, ignoredAtRules, media, hashFiles, cache) { // parse-import module parse entire line // @todo extract what can be interesting from this one var parsedAtImport = parseImport(atRule.params, atRule.source) @@ -197,19 +283,20 @@ function readAtImport(atRule, options, cb, importedFiles, ignoredAtRules, media, var resolvedFilename = resolveFilename(parsedAtImport.uri, options.root, options.path, atRule.source) // skip files already imported at the same scope - if (importedFiles[resolvedFilename] && importedFiles[resolvedFilename][media]) { + if (globallyImportedFiles[resolvedFilename] && globallyImportedFiles[resolvedFilename][media]) { detach(atRule) return } // save imported files to skip them next time - if (!importedFiles[resolvedFilename]) { - importedFiles[resolvedFilename] = {} + if (!globallyImportedFiles[resolvedFilename]) { + globallyImportedFiles[resolvedFilename] = {} } - importedFiles[resolvedFilename][media] = true - - - readImportedContent(atRule, parsedAtImport, clone(options), resolvedFilename, cb, importedFiles, ignoredAtRules, media, hashFiles) + globallyImportedFiles[resolvedFilename][media] = true + if (locallyImportedFiles.indexOf(resolvedFilename) === -1) { + locallyImportedFiles.push(resolvedFilename) + } + readImportedContent(atRule, parsedAtImport, clone(options), resolvedFilename, cb, globallyImportedFiles, ignoredAtRules, media, hashFiles, cache) } /** @@ -221,7 +308,7 @@ function readAtImport(atRule, options, cb, importedFiles, ignoredAtRules, media, * @param {String} resolvedFilename * @param {Function} cb */ -function readImportedContent(atRule, parsedAtImport, options, resolvedFilename, cb, importedFiles, ignoredAtRules, media, hashFiles) { +function readImportedContent(atRule, parsedAtImport, options, resolvedFilename, cb, globallyImportedFiles, ignoredAtRules, media, hashFiles, cache) { // add directory containing the @imported file in the paths // to allow local import from this file var dirname = path.dirname(resolvedFilename) @@ -229,12 +316,17 @@ function readImportedContent(atRule, parsedAtImport, options, resolvedFilename, options.path = options.path.slice() options.path.unshift(dirname) } - options.from = resolvedFilename var fileContent = readFile(resolvedFilename, options.encoding, options.transform || function(value) { return value }) if (fileContent.trim() === "") { console.log(helpers.message(resolvedFilename + " is empty", atRule.source)) + if (cache) { + cache[resolvedFilename] = { + mtime: fs.statSync(resolvedFilename).mtime.getTime(), + dependencies: [] + } + } detach(atRule) return } @@ -260,7 +352,7 @@ function readImportedContent(atRule, parsedAtImport, options, resolvedFilename, var newStyles = postcss.parse(fileContent, options) // recursion: import @import from imported file - parseStyles(newStyles, options, cb, importedFiles, ignoredAtRules, parsedAtImport.media, hashFiles) + parseStyles(newStyles, options, cb, globallyImportedFiles, ignoredAtRules, parsedAtImport.media, hashFiles, cache) cb(atRule, parsedAtImport, newStyles, resolvedFilename) } diff --git a/test/index.js b/test/index.js index ff3b2acd..8275478e 100755 --- a/test/index.js +++ b/test/index.js @@ -10,13 +10,14 @@ var postcss = require("postcss") var fixturesDir = path.join(__dirname, "fixtures") var importsDir = path.join(fixturesDir, "imports") +var cacheDir = path.join(__dirname, "cache") function read(name) { return fs.readFileSync("test/" + name + ".css", "utf8").trim() } function compareFixtures(t, name, msg, opts, postcssOpts) { - opts = opts || {path: importsDir} + opts = opts || {path: importsDir, cacheDir: cacheDir} postcssOpts = postcssOpts || {} postcssOpts.from = "test/fixtures/" + name + ".css" var actual = postcss() From 190df56ab8cc186cf38aa1d741ce82b3e422f573 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Thu, 4 Jun 2015 09:48:58 +0200 Subject: [PATCH 03/11] Move cache to its own test --- test/fixtures/cwd.css | 4 ++-- test/fixtures/ignore.css | 2 +- test/index.js | 22 ++++++++++++++++++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/test/fixtures/cwd.css b/test/fixtures/cwd.css index bdc682c0..d46d79eb 100755 --- a/test/fixtures/cwd.css +++ b/test/fixtures/cwd.css @@ -1,2 +1,2 @@ -@import "imports/foo.css"; -@import "imports/foo-recursive.css"; +@import "test/fixtures/imports/foo.css"; +@import "test/fixtures/imports/foo-recursive.css"; diff --git a/test/fixtures/ignore.css b/test/fixtures/ignore.css index 24265277..71a7951e 100644 --- a/test/fixtures/ignore.css +++ b/test/fixtures/ignore.css @@ -1,4 +1,4 @@ -@import "imports/ignore.css" (min-width: 25em); +@import "ignore.css" (min-width: 25em); @import "http://css"; @import "https://css"; @import 'http://css'; diff --git a/test/index.js b/test/index.js index 8275478e..d21d7a4c 100755 --- a/test/index.js +++ b/test/index.js @@ -17,9 +17,8 @@ function read(name) { } function compareFixtures(t, name, msg, opts, postcssOpts) { - opts = opts || {path: importsDir, cacheDir: cacheDir} + opts = opts || {path: importsDir} postcssOpts = postcssOpts || {} - postcssOpts.from = "test/fixtures/" + name + ".css" var actual = postcss() .use(atImport(opts)) .process(read("fixtures/" + name), postcssOpts) @@ -193,3 +192,22 @@ test("works with no styles at all", function(t) { t.end() }) + +test("works with caching", function(t) { + var opts = {path: importsDir, cacheDir: cacheDir} + var postcssOpts = {from: "test/fixtures/relative-to-source.css"} + var name = "relative-to-source" + var actual = postcss() + .use(atImport(opts)) + .process(read("fixtures/" + name), postcssOpts) + .css.trim() + var expected = read("fixtures/" + name + ".expected") + t.equal(actual, expected, "put content in cache") + actual = postcss() + .use(atImport(opts)) + .process(read("fixtures/" + name), postcssOpts) + .css.trim() + expected = read("fixtures/" + name + ".expected") + t.equal(actual, expected, "read content from cache") + t.end() +}) From c8c3bbd0b0afaed1cf0275c3e1a24a7654f5d1f6 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Mon, 8 Jun 2015 00:09:29 +0200 Subject: [PATCH 04/11] Add test to check cache contents --- test/index.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/index.js b/test/index.js index d21d7a4c..b7e7ac38 100755 --- a/test/index.js +++ b/test/index.js @@ -195,19 +195,24 @@ test("works with no styles at all", function(t) { test("works with caching", function(t) { var opts = {path: importsDir, cacheDir: cacheDir} - var postcssOpts = {from: "test/fixtures/relative-to-source.css"} - var name = "relative-to-source" - var actual = postcss() + var cssFileName = path.resolve("test/fixtures/relative-to-source.css") + var postcssOpts = {from: cssFileName} + var css = fs.readFileSync(cssFileName) + var output = postcss() .use(atImport(opts)) - .process(read("fixtures/" + name), postcssOpts) + .process(css, postcssOpts) .css.trim() - var expected = read("fixtures/" + name + ".expected") - t.equal(actual, expected, "put content in cache") - actual = postcss() + + var imports = JSON.parse(fs.readFileSync(cacheDir + "/imports.json")) + var cacheFileName = imports[cssFileName].cache; + var cache = fs.readFileSync(cacheFileName, "utf8").trim() + + t.equal(output, cache, "should put output in cache") + + output = postcss() .use(atImport(opts)) - .process(read("fixtures/" + name), postcssOpts) + .process(css, postcssOpts) .css.trim() - expected = read("fixtures/" + name + ".expected") - t.equal(actual, expected, "read content from cache") + t.equal(output, cache, "should return identical result on subsequent calls") t.end() }) From b69f080762aa404c92cb0dd0450ca1d5e9c67ceb Mon Sep 17 00:00:00 2001 From: Jed Mao Date: Fri, 19 Jun 2015 17:22:21 -0500 Subject: [PATCH 05/11] Fix plugins section of README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 48ee80bb..59f0e63b 100755 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ A function to transform the content of imported files. Take one argument (file c #### `plugins` -Type: `Array` +Type: `Array` Default: `undefined` An array of plugins to be applied on each imported file. From 3642fff1bc2e68587872d1dc986eaac02982d1ee Mon Sep 17 00:00:00 2001 From: Jed Mao Date: Fri, 19 Jun 2015 23:41:53 -0500 Subject: [PATCH 06/11] Refactor index.js --- index.js | 409 ++++++++++++++++++++++++------------------------------- 1 file changed, 179 insertions(+), 230 deletions(-) diff --git a/index.js b/index.js index 2e52947d..b6af9052 100755 --- a/index.js +++ b/index.js @@ -32,20 +32,20 @@ var warnNodesMessage = /** * Inline `@import`ed files * - * @param {Object} options + * @param {Object} pluginOptions */ -function AtImport(options) { - options = assign({}, options || {}) - options.root = options.root || process.cwd() - options.path = ( +function AtImport(pluginOptions) { + pluginOptions = assign({}, pluginOptions || {}) + pluginOptions.root = pluginOptions.root || process.cwd() + pluginOptions.path = ( // convert string to an array of a single element - typeof options.path === "string" ? - [options.path] : - (options.path || []) // fallback to empty array + typeof pluginOptions.path === "string" ? + [pluginOptions.path] : + (pluginOptions.path || []) // fallback to empty array ) return function(styles, result) { - const opts = assign({}, options || {}) + const opts = assign({}, pluginOptions || {}) // auto add from option if possible if ( !opts.from && @@ -77,77 +77,188 @@ function AtImport(options) { var hashFiles = {} - return parseStyles( - result, - styles, - opts, - importedFiles, - ignoredAtRules, - null, - hashFiles, - createProcessor(result, options.plugins) - ).then(function() { + var processor = createProcessor() + + function createProcessor() { + var plugins = opts.plugins + if (plugins) { + if (!Array.isArray(plugins)) { + throw new Error("plugins option must be an array") + } + return postcss(plugins) + } + return postcss() + } + + return parseStyles(styles, null, opts).then(function() { addIgnoredAtRulesOnTop(styles, ignoredAtRules) if (typeof opts.onImport === "function") { opts.onImport(Object.keys(importedFiles)) } }) - } -} -function createProcessor(result, plugins) { - if (plugins) { - if (!Array.isArray(plugins)) { - throw new Error("plugins option must be an array") + /** + * lookup for @import rules + * + * @param {Object} css + * @param {Object} options + */ + function parseStyles(css, media, options) { + var imports = [] + css.eachAtRule("import", function checkAtRule(atRule) { + if (atRule.nodes) { + result.warn(warnNodesMessage, {node: atRule}) + } + if (options.glob && glob.hasMagic(atRule.params)) { + imports = parseGlob(atRule, options, imports) + } + else { + imports.push(atRule) + } + }) + return Promise.all(imports.map(function(atRule) { + return helpers.try(function transformAtImport() { + return readAtImport(atRule, media, options) + }, atRule.source) + })) } - return postcss(plugins) - } - return postcss() -} -/** - * lookup for @import rules - * - * @param {Object} styles - * @param {Object} options - */ -function parseStyles( - result, - styles, - options, - importedFiles, - ignoredAtRules, - media, - hashFiles, - processor -) { - var imports = [] - styles.eachAtRule("import", function checkAtRule(atRule) { - if (atRule.nodes) { - result.warn(warnNodesMessage, {node: atRule}) - } - if (options.glob && glob.hasMagic(atRule.params)) { - imports = parseGlob(atRule, options, imports) - } - else { - imports.push(atRule) - } - }) - return Promise.all(imports.map(function(atRule) { - return helpers.try(function transformAtImport() { - return readAtImport( - result, + /** + * parse @import rules & inline appropriate rules + * + * @param {Object} atRule postcss atRule + * @param {Object} options + */ + function readAtImport(atRule, media, options) { + // parse-import module parse entire line + // @todo extract what can be interesting from this one + var parsedAtImport = parseImport(atRule.params, atRule.source) + + // adjust media according to current scope + media = parsedAtImport.media + ? (media ? media + " and " : "") + parsedAtImport.media + : (media ? media : null) + + // just update protocol base uri (protocol://url) or protocol-relative + // (//url) if media needed + if (parsedAtImport.uri.match(/^(?:[a-z]+:)?\/\//i)) { + parsedAtImport.media = media + + // save + ignoredAtRules.push([atRule, parsedAtImport]) + + // detach + detach(atRule) + + return resolvedPromise + } + + addInputToPath(options) + var resolvedFilename = resolveFilename( + parsedAtImport.uri, + options.root, + options.path, + atRule.source, + options.resolve + ) + + // skip files already imported at the same scope + if ( + importedFiles[resolvedFilename] && + importedFiles[resolvedFilename][media] + ) { + detach(atRule) + return resolvedPromise + } + + // save imported files to skip them next time + if (!importedFiles[resolvedFilename]) { + importedFiles[resolvedFilename] = {} + } + importedFiles[resolvedFilename][media] = true + + return readImportedContent( atRule, + parsedAtImport, options, - importedFiles, - ignoredAtRules, - media, - hashFiles, - processor + resolvedFilename, + media + ) + } + + /** + * insert imported content at the right place + * + * @param {Object} atRule + * @param {Object} parsedAtImport + * @param {Object} options + * @param {String} resolvedFilename + */ + function readImportedContent( + atRule, + parsedAtImport, + options, + resolvedFilename, + media + ) { + // add directory containing the @imported file in the paths + // to allow local import from this file + options = clone(options) + var dirname = path.dirname(resolvedFilename) + if (options.path.indexOf(dirname) === -1) { + options.path = options.path.slice() + options.path.unshift(dirname) + } + + options.from = resolvedFilename + var fileContent = readFile( + resolvedFilename, + options.encoding, + options.transform || function(value) { + return value + } ) - }, atRule.source) - })) + + if (fileContent.trim() === "") { + result.warn(resolvedFilename + " is empty", {node: atRule}) + detach(atRule) + return resolvedPromise + } + + // skip files wich only contain @import rules + var newFileContent = fileContent.replace(/@import (.*);/, "") + if (newFileContent.trim() !== "") { + var fileContentHash = hash(fileContent) + + // skip files already imported at the same scope and same hash + if (hashFiles[fileContentHash] && hashFiles[fileContentHash][media]) { + detach(atRule) + return resolvedPromise + } + + // save hash files to skip them next time + if (!hashFiles[fileContentHash]) { + hashFiles[fileContentHash] = {} + } + hashFiles[fileContentHash][media] = true + } + + var newStyles = postcss.parse(fileContent, options) + + // recursion: import @import from imported file + return parseStyles(newStyles, parsedAtImport.media, options) + .then(function() { + return processor.process(newStyles) + .then(function(newResult) { + result.messages = result.messages.concat(newResult.messages) + }) + }) + .then(function() { + insertRules(atRule, parsedAtImport, newStyles) + }) + } + } } /** @@ -226,168 +337,6 @@ function addIgnoredAtRulesOnTop(styles, ignoredAtRules) { } } -/** - * parse @import rules & inline appropriate rules - * - * @param {Object} atRule postcss atRule - * @param {Object} options - */ -function readAtImport( - result, - atRule, - options, - importedFiles, - ignoredAtRules, - media, - hashFiles, - processor -) { - // parse-import module parse entire line - // @todo extract what can be interesting from this one - var parsedAtImport = parseImport(atRule.params, atRule.source) - - // adjust media according to current scope - media = parsedAtImport.media - ? (media ? media + " and " : "") + parsedAtImport.media - : (media ? media : null) - - // just update protocol base uri (protocol://url) or protocol-relative - // (//url) if media needed - if (parsedAtImport.uri.match(/^(?:[a-z]+:)?\/\//i)) { - parsedAtImport.media = media - - // save - ignoredAtRules.push([atRule, parsedAtImport]) - - // detach - detach(atRule) - - return resolvedPromise - } - - addInputToPath(options) - var resolvedFilename = resolveFilename( - parsedAtImport.uri, - options.root, - options.path, - atRule.source, - options.resolve - ) - - // skip files already imported at the same scope - if ( - importedFiles[resolvedFilename] && - importedFiles[resolvedFilename][media] - ) { - detach(atRule) - return resolvedPromise - } - - // save imported files to skip them next time - if (!importedFiles[resolvedFilename]) { - importedFiles[resolvedFilename] = {} - } - importedFiles[resolvedFilename][media] = true - - return readImportedContent( - result, - atRule, - parsedAtImport, - clone(options), - resolvedFilename, - importedFiles, - ignoredAtRules, - media, - hashFiles, - processor - ) -} - -/** - * insert imported content at the right place - * - * @param {Object} atRule - * @param {Object} parsedAtImport - * @param {Object} options - * @param {String} resolvedFilename - */ -function readImportedContent( - result, - atRule, - parsedAtImport, - options, - resolvedFilename, - importedFiles, - ignoredAtRules, - media, - hashFiles, - processor -) { - // add directory containing the @imported file in the paths - // to allow local import from this file - var dirname = path.dirname(resolvedFilename) - if (options.path.indexOf(dirname) === -1) { - options.path = options.path.slice() - options.path.unshift(dirname) - } - - options.from = resolvedFilename - var fileContent = readFile( - resolvedFilename, - options.encoding, - options.transform || function(value) { - return value - } - ) - - if (fileContent.trim() === "") { - result.warn(resolvedFilename + " is empty", {node: atRule}) - detach(atRule) - return resolvedPromise - } - - // skip files wich only contain @import rules - var newFileContent = fileContent.replace(/@import (.*);/, "") - if (newFileContent.trim() !== "") { - var fileContentHash = hash(fileContent) - - // skip files already imported at the same scope and same hash - if (hashFiles[fileContentHash] && hashFiles[fileContentHash][media]) { - detach(atRule) - return resolvedPromise - } - - // save hash files to skip them next time - if (!hashFiles[fileContentHash]) { - hashFiles[fileContentHash] = {} - } - hashFiles[fileContentHash][media] = true - } - - var newStyles = postcss.parse(fileContent, options) - - // recursion: import @import from imported file - return parseStyles( - result, - newStyles, - options, - importedFiles, - ignoredAtRules, - parsedAtImport.media, - hashFiles, - processor - ) - .then(function() { - return processor.process(newStyles) - .then(function(newResult) { - result.messages = result.messages.concat(newResult.messages) - }) - }) - .then(function() { - insertRules(atRule, parsedAtImport, newStyles) - }) -} - /** * insert new imported rules at the right place * From 389121c7ac6746947f5fc99aa9b3c61b1110956f Mon Sep 17 00:00:00 2001 From: Jed Mao Date: Wed, 24 Jun 2015 18:20:57 -0500 Subject: [PATCH 07/11] Introduce caching --- .gitignore | 1 + README.md | 7 +++ index.js | 144 ++++++++++++++++++++++++++++++++++++++++++++------ package.json | 1 + test/index.js | 28 ++++++++++ 5 files changed, 164 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 7ab649f4..5c937481 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules +test/cache test/fixtures/*.actual.css diff --git a/README.md b/README.md index 59f0e63b..070e321a 100755 --- a/README.md +++ b/README.md @@ -93,6 +93,13 @@ Default: `process.cwd()` or _dirname of [the postcss `from`](https://github.com/ A string or an array of paths in where to look for files. _Note: nested `@import` will additionally benefit of the relative dirname of imported files._ +#### `cacheDir` + +Type: `String` +Default: `null` + +The location of the cache. When set, the compiled output will be stored in this directory. The cache will automatically be invalidated when a file in the dependency graph is modified. + #### `transform` Type: `Function` diff --git a/index.js b/index.js index b6af9052..db6952ae 100755 --- a/index.js +++ b/index.js @@ -11,6 +11,7 @@ var postcss = require("postcss") var helpers = require("postcss-message-helpers") var hash = require("string-hash") var glob = require("glob") +var mkdirp = require("mkdirp") var Promise = global.Promise || require("es6-promise").Promise var resolvedPromise = new Promise(function(resolvePromise) { @@ -67,9 +68,9 @@ function AtImport(pluginOptions) { opts.path.push(process.cwd()) } - var importedFiles = {} + var globallyImportedFiles = {} if (opts.from) { - importedFiles[opts.from] = { + globallyImportedFiles[opts.from] = { "": true, } } @@ -79,6 +80,19 @@ function AtImport(pluginOptions) { var processor = createProcessor() + var cache + var cacheFile + if (opts.cacheDir) { + mkdirp.sync(opts.cacheDir) + cacheFile = path.resolve(opts.cacheDir, "import-cache.json") + try { + cache = require(cacheFile) + } + catch (err) { + cache = {} + } + } + function createProcessor() { var plugins = opts.plugins if (plugins) { @@ -93,8 +107,12 @@ function AtImport(pluginOptions) { return parseStyles(styles, null, opts).then(function() { addIgnoredAtRulesOnTop(styles, ignoredAtRules) + if (cache) { + fs.writeFileSync(cacheFile, JSON.stringify(cache)) + } + if (typeof opts.onImport === "function") { - opts.onImport(Object.keys(importedFiles)) + opts.onImport(Object.keys(globallyImportedFiles)) } }) @@ -105,6 +123,23 @@ function AtImport(pluginOptions) { * @param {Object} options */ function parseStyles(css, media, options) { + if ( + options.from && + !media && + cache && + isCached(options.from, options) + ) { + return readFromCache(options.from).then(function(newStyles) { + newStyles.nodes.forEach(function(node) { + node.parent = styles + }) + assign(styles, { + source: newStyles.source, + nodes: newStyles.nodes, + semicolon: newStyles.semicolon, + }) + }) + } var imports = [] css.eachAtRule("import", function checkAtRule(atRule) { if (atRule.nodes) { @@ -117,11 +152,73 @@ function AtImport(pluginOptions) { imports.push(atRule) } }) + var locallyImportedFiles = [] return Promise.all(imports.map(function(atRule) { return helpers.try(function transformAtImport() { - return readAtImport(atRule, media, options) + return readAtImport(atRule, media, locallyImportedFiles, options) }, atRule.source) - })) + })).then(function() { + if (options.from && cache && !media) { + var cacheFilename + if (locallyImportedFiles.length) { + var cssOut = styles.toResult().css + cacheFilename = path.resolve( + options.cacheDir, + hash(cssOut) + ".css" + ) + fs.writeFileSync(cacheFilename, cssOut) + } + assign(cache[options.from] || {}, { + modified: getModifiedFileTime(options.from), + dependencies: locallyImportedFiles, + cache: cacheFilename, + }) + } + }) + + function readFromCache(filename) { + return new Promise(function(resolvePromise, rejectPromise) { + fs.readFile(cache[filename].cache, function(err, contents) { + if (err) { + rejectPromise(err) + return + } + postcss.parse(contents, options).then(function(cachedResult) { + resolvePromise(cachedResult) + }) + }) + }) + } + } + + function isCached(filename) { + var importCache = cache[filename] + + if (!isDependencyCached(filename) && importCache.cache) { + fs.unlink(importCache.cache, function(err) { + if (err) { + throw err + } + }) + } + + return !!importCache.cache + + function isDependencyCached() { + if (!filename) { + return false + } + if (!importCache) { + return false + } + var modified = getModifiedFileTime(filename) + if (modified !== importCache.modified) { + return false + } + return !importCache.dependencies.some(function(dependency) { + return !isDependencyCached(dependency) + }) + } } /** @@ -130,7 +227,7 @@ function AtImport(pluginOptions) { * @param {Object} atRule postcss atRule * @param {Object} options */ - function readAtImport(atRule, media, options) { + function readAtImport(atRule, media, locallyImportedFiles, options) { // parse-import module parse entire line // @todo extract what can be interesting from this one var parsedAtImport = parseImport(atRule.params, atRule.source) @@ -163,20 +260,23 @@ function AtImport(pluginOptions) { options.resolve ) - // skip files already imported at the same scope - if ( - importedFiles[resolvedFilename] && - importedFiles[resolvedFilename][media] - ) { - detach(atRule) - return resolvedPromise + var globallyImportedFile = globallyImportedFiles[resolvedFilename] + if (globallyImportedFile) { + if (globallyImportedFile[media]) { + // skip files already imported at the same scope + detach(atRule) + return resolvedPromise + } + } + else { + // save imported files to skip them next time + globallyImportedFiles[resolvedFilename] = {} } + globallyImportedFiles[resolvedFilename][media] = true - // save imported files to skip them next time - if (!importedFiles[resolvedFilename]) { - importedFiles[resolvedFilename] = {} + if (locallyImportedFiles.indexOf(resolvedFilename) === -1) { + locallyImportedFiles.push(resolvedFilename) } - importedFiles[resolvedFilename][media] = true return readImportedContent( atRule, @@ -222,6 +322,12 @@ function AtImport(pluginOptions) { if (fileContent.trim() === "") { result.warn(resolvedFilename + " is empty", {node: atRule}) + if (cache) { + cache[resolvedFilename] = { + modified: getModifiedFileTime(resolvedFilename), + dependencies: [], + } + } detach(atRule) return resolvedPromise } @@ -490,6 +596,10 @@ function detach(node) { node.parent.nodes.splice(node.parent.nodes.indexOf(node), 1) } +function getModifiedFileTime(filename) { + return fs.statSync(filename).mtime.getTime() +} + module.exports = postcss.plugin( "postcss-import", AtImport diff --git a/package.json b/package.json index 0e7e16fd..8bf48cfe 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "devDependencies": { "css-whitespace": "^1.1.0", "eslint": "^0.23.0", + "mkdirp": "^0.5.1", "tape": "^3.0.0" }, "scripts": { diff --git a/test/index.js b/test/index.js index 2df946a5..ac9341d7 100755 --- a/test/index.js +++ b/test/index.js @@ -9,6 +9,7 @@ var postcss = require("postcss") var fixturesDir = path.join(__dirname, "fixtures") var importsDir = path.join(fixturesDir, "imports") +var cacheDir = path.join(__dirname, "cache") function read(name) { return fs.readFileSync("test/" + name + ".css", "utf8").trim() @@ -322,3 +323,30 @@ test("plugins option", function(t) { t.pass("should remain silent when value is an empty array") }) }) + +test("works with caching", function(t) { + var opts = {path: importsDir, cacheDir: cacheDir} + var cssFileName = path.resolve("test/fixtures/relative-to-source.css") + var postcssOpts = {from: cssFileName} + var css = fs.readFileSync(cssFileName) + postcss() + .use(atImport(opts)) + .process(css, postcssOpts) + .then(trimResultCss) + .then(function(output) { + var imports = JSON.parse(fs.readFileSync(cacheDir + "/import-cache.json")) + var cacheFileName = imports[cssFileName].cache + var cache = fs.readFileSync(cacheFileName, "utf8").trim() + + t.equal(output, cache, "should put output in cache") + + postcss() + .use(atImport(opts)) + .process(css, postcssOpts) + .then(trimResultCss) + .then(function(output2) { + t.equal(output2, cache, "should return identical result on subsequent calls") + t.end() + }) + }) +}) From 936a93e4a2dcaa6ce46709717b751ecc22bc8a81 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Sun, 28 Jun 2015 01:19:13 +0200 Subject: [PATCH 08/11] Fix cache --- index.js | 55 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index db6952ae..5c2e8fb2 100755 --- a/index.js +++ b/index.js @@ -131,9 +131,9 @@ function AtImport(pluginOptions) { ) { return readFromCache(options.from).then(function(newStyles) { newStyles.nodes.forEach(function(node) { - node.parent = styles + node.parent = css }) - assign(styles, { + assign(css, { source: newStyles.source, nodes: newStyles.nodes, semicolon: newStyles.semicolon, @@ -168,7 +168,8 @@ function AtImport(pluginOptions) { ) fs.writeFileSync(cacheFilename, cssOut) } - assign(cache[options.from] || {}, { + cache[options.from] = cache[options.from] || {} + assign(cache[options.from], { modified: getModifiedFileTime(options.from), dependencies: locallyImportedFiles, cache: cacheFilename, @@ -183,9 +184,7 @@ function AtImport(pluginOptions) { rejectPromise(err) return } - postcss.parse(contents, options).then(function(cachedResult) { - resolvePromise(cachedResult) - }) + resolvePromise(postcss.parse(contents, options)) }) }) } @@ -194,31 +193,35 @@ function AtImport(pluginOptions) { function isCached(filename) { var importCache = cache[filename] - if (!isDependencyCached(filename) && importCache.cache) { - fs.unlink(importCache.cache, function(err) { - if (err) { - throw err - } - }) + if (!isDependencyCached(filename)) { + if (importCache && importCache.cache) { + fs.unlink(importCache.cache, function(err) { + if (err) { + throw err + } + }) + } + return false } return !!importCache.cache + } - function isDependencyCached() { - if (!filename) { - return false - } - if (!importCache) { - return false - } - var modified = getModifiedFileTime(filename) - if (modified !== importCache.modified) { - return false - } - return !importCache.dependencies.some(function(dependency) { - return !isDependencyCached(dependency) - }) + function isDependencyCached(filename) { + if (!filename) { + return false } + var importCache = cache[filename] + if (!importCache) { + return false + } + var modified = getModifiedFileTime(filename) + if (modified !== importCache.modified) { + return false + } + return !importCache.dependencies.some(function(dependency) { + return !isDependencyCached(dependency) + }) } /** From d0b054cc3aa608841579edb96156bb5e3c85859f Mon Sep 17 00:00:00 2001 From: andyjansson Date: Sun, 28 Jun 2015 02:15:57 +0200 Subject: [PATCH 09/11] Fix cache --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 5c2e8fb2..83965da8 100755 --- a/index.js +++ b/index.js @@ -161,7 +161,7 @@ function AtImport(pluginOptions) { if (options.from && cache && !media) { var cacheFilename if (locallyImportedFiles.length) { - var cssOut = styles.toResult().css + var cssOut = css.toResult().css cacheFilename = path.resolve( options.cacheDir, hash(cssOut) + ".css" From 20bced0242c71466e8f0b468445d9d20e9e31b24 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Mon, 27 Jul 2015 20:52:58 +0200 Subject: [PATCH 10/11] Fix for not invoking files with same content twice when reading from cache --- .gitignore | 1 + index.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7a57e7cf..7e100029 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules test/cache test/fixtures/*.actual.css test/cache +.idea \ No newline at end of file diff --git a/index.js b/index.js index 83965da8..8dafa1c3 100755 --- a/index.js +++ b/index.js @@ -127,7 +127,8 @@ function AtImport(pluginOptions) { options.from && !media && cache && - isCached(options.from, options) + isCached(options.from) && + shouldInvokeCache(options.from) ) { return readFromCache(options.from).then(function(newStyles) { newStyles.nodes.forEach(function(node) { @@ -224,6 +225,11 @@ function AtImport(pluginOptions) { }) } + function shouldInvokeCache(filename) { + var cacheHash = path.basename(cache[filename].cache, ".css") + return !hashFiles[cacheHash] + } + /** * parse @import rules & inline appropriate rules * From 00cac1b36404b1d7ce3a368a02e0e92d431361c1 Mon Sep 17 00:00:00 2001 From: andyjansson Date: Mon, 27 Jul 2015 23:34:25 +0200 Subject: [PATCH 11/11] Fix for putting file read from cache into hashFiles --- index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 8dafa1c3..7ed8e744 100755 --- a/index.js +++ b/index.js @@ -180,7 +180,13 @@ function AtImport(pluginOptions) { function readFromCache(filename) { return new Promise(function(resolvePromise, rejectPromise) { - fs.readFile(cache[filename].cache, function(err, contents) { + var fileCache = cache[filename].cache + var cacheHash = path.basename(fileCache, ".css") + if (!hashFiles[cacheHash]) { + hashFiles[cacheHash] = {} + } + hashFiles[cacheHash][media] = true + fs.readFile(fileCache, function(err, contents) { if (err) { rejectPromise(err) return