Skip to content

reduce synchronous disk IO operations #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jul 5, 2014
Merged

reduce synchronous disk IO operations #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
46f5e37
reduce sync operations
peecky Mar 28, 2014
f36bb8a
reduce more sync operations
peecky Mar 28, 2014
cc105a2
binding this to some functions
peecky Mar 28, 2014
499475e
fix missing parentheses
peecky Mar 28, 2014
44087de
increasing counter for each file before the end event fired
peecky Mar 28, 2014
336598a
add the async module to the package.json
peecky Mar 28, 2014
e36b88a
indentation
peecky Mar 28, 2014
b97e921
fix duplicated call of callback
peecky Mar 28, 2014
f2e62f8
fix missing binding this
peecky Mar 28, 2014
3c8590a
remove unnecessary spaces
peecky Mar 28, 2014
1965c8b
indent
peecky Mar 28, 2014
f28f5e9
fix missing var. remove unnecessary code
peecky Mar 28, 2014
80be7ef
change decrementing counter to calling finish
peecky Mar 28, 2014
3135dec
indent
peecky Mar 28, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 81 additions & 67 deletions lib/uploadhandler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ var EventEmitter = require('events').EventEmitter,
formidable = require('formidable'),
imageMagick = require('imagemagick'),
mkdirp = require('mkdirp'),
_ = require('lodash');
_ = require('lodash'),
async = require('async');

module.exports = function (options) {

Expand Down Expand Up @@ -34,19 +35,24 @@ module.exports = function (options) {
this.noCache();
var files = [];
fs.readdir(options.uploadDir(), _.bind(function (err, list) {
_.each(list, function (name) {
var stats = fs.statSync(options.uploadDir() + '/' + name),
fileInfo;
if (stats.isFile()) {
fileInfo = new FileInfo({
name: name,
size: stats.size
});
this.initUrls(fileInfo);
files.push(fileInfo);
}
}, this);
this.callback({files: files});
async.each(list, _.bind(function(name, cb) {
fs.stat(options.uploadDir() + '/' + name, _.bind(function(err, stats) {
if (!err && stats.isFile()) {
var fileInfo = new FileInfo({
name: name,
size: stats.size
});
this.initUrls(fileInfo, function(err) {
files.push(fileInfo);
cb(err);
});
}
else cb(err);
}, this));
}, this),
_.bind(function(err) {
this.callback({files: files});
}, this));
}, this));
};

Expand All @@ -60,11 +66,15 @@ module.exports = function (options) {
redirect,
finish = _.bind(function () {
if (!--counter) {
_.each(files, function (fileInfo) {
this.initUrls(fileInfo);
this.emit('end', fileInfo);
}, this);
this.callback({files: files}, redirect);
async.each(files, _.bind(function(fileInfo, cb) {
this.initUrls(fileInfo, _.bind(function(err) {
this.emit('end', fileInfo);
cb(err);
}, this));
}, this),
_.bind(function(err) {
this.callback({files: files}, redirect);
}, this));
}
}, this);

Expand All @@ -86,56 +96,58 @@ module.exports = function (options) {
}
})
.on('file', function (name, file) {
counter++;
var fileInfo = map[path.basename(file.path)];
if (fs.existsSync(file.path)) {
fileInfo.size = file.size;
if (!fileInfo.validate()) {
fs.unlink(file.path);
return;
}

var generatePreviews = function () {
if (options.imageTypes.test(fileInfo.name)) {
_.each(options.imageVersions, function (value, version) {
// creating directory recursive
if (!fs.existsSync(options.uploadDir() + '/' + version + '/'))
mkdirp.sync(options.uploadDir() + '/' + version + '/');

counter++;
var opts = options.imageVersions[version];
imageMagick.resize({
width: opts.width,
height: opts.height,
srcPath: options.uploadDir() + '/' + fileInfo.name,
dstPath: options.uploadDir() + '/' + version + '/' + fileInfo.name,
customArgs: opts.imageArgs || ['-auto-orient']
}, finish);
});
fs.exists(file.path, function(exists) {
if (exists) {
fileInfo.size = file.size;
if (!fileInfo.validate()) {
fs.unlink(file.path);
finish();
return;
}
}

if (!fs.existsSync(options.uploadDir() + '/'))
mkdirp.sync(options.uploadDir() + '/');
var generatePreviews = function () {
if (options.imageTypes.test(fileInfo.name)) {
_.each(options.imageVersions, function (value, version) {
counter++;
// creating directory recursive
mkdirp(options.uploadDir() + '/' + version + '/', function (err, made) {
var opts = options.imageVersions[version];
imageMagick.resize({
width: opts.width,
height: opts.height,
srcPath: options.uploadDir() + '/' + fileInfo.name,
dstPath: options.uploadDir() + '/' + version + '/' + fileInfo.name,
customArgs: opts.imageArgs || ['-auto-orient']
}, finish);
});
});
}
}

counter++;
fs.rename(file.path, options.uploadDir() + '/' + fileInfo.name, function (err) {
if (!err) {
generatePreviews();
finish();
} else {
var is = fs.createReadStream(file.path);
var os = fs.createWriteStream(options.uploadDir() + '/' + fileInfo.name);
is.on('end', function (err) {
mkdirp(options.uploadDir() + '/', function(err, made) {
fs.rename(file.path, options.uploadDir() + '/' + fileInfo.name, function (err) {
if (!err) {
fs.unlinkSync(file.path);
generatePreviews();
finish();
} else {
var is = fs.createReadStream(file.path);
var os = fs.createWriteStream(options.uploadDir() + '/' + fileInfo.name);
is.on('end', function (err) {
if (!err) {
fs.unlink(file.path);
generatePreviews();
}
finish();
});
is.pipe(os);
}
finish();
});
is.pipe(os);
}
});
}
});
}
else finish();
});
})
.on('aborted', function () {
_.each(tmpFiles, function (file) {
Expand Down Expand Up @@ -168,15 +180,17 @@ module.exports = function (options) {
});
};

UploadHandler.prototype.initUrls = function (fileInfo) {
UploadHandler.prototype.initUrls = function (fileInfo, cb) {
var baseUrl = (options.ssl ? 'https:' : 'http:') + '//' + (options.hostname || this.req.get('Host'));
fileInfo.setUrl(null, baseUrl + options.uploadUrl());
fileInfo.setUrl('delete', baseUrl + this.req.originalUrl);
_.each(options.imageVersions, function (value, version) {
if (fs.existsSync(options.uploadDir() + '/' + version + '/' + fileInfo.name)) {
fileInfo.setUrl(version, baseUrl + options.uploadUrl() + '/' + version);
}
}, this);
async.each(Object.keys(options.imageVersions), function(version, cb) {
fs.exists(options.uploadDir() + '/' + version + '/' + fileInfo.name, function(exists) {
if (exists) fileInfo.setUrl(version, baseUrl + options.uploadUrl() + '/' + version);
cb(null);
})
},
cb);
};

return UploadHandler;
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
"formidable": ">=1.0.11",
"imagemagick": ">=0.1.2",
"lodash": ">= 0.9.2",
"mkdirp": ">= 0.3.4"
"mkdirp": ">= 0.3.4",
"async": "*"
},
"engines": {
"node": ">= 0.8.8"
Expand Down