plugin.js
module.exports = function(holder) {
return through.obj(function(file, encoding, callback) {
console.log('2')
return callback(null, file);
}
}
task
gulp.task('t', function() {
console.log('1');
var tmp = gulp.src(...).pipe(require('plugin')())
console.log('3');
return tmp;
});
The console result is:
1
3
2
But what I want is:
1
2
3
How could I do that?
You don't make the stream synchronous. Instead you listen for the end event :
gulp.task('t', function() {
console.log('1');
return gulp.src(...)
.pipe(require('plugin')())
.on('end', function() {
console.log('3');
});
});
Related
I have 3 function has to call in series one after the other. but first function is exeucting and in between second and third are executing.
var tasklist=[api_hit,delay,mysql_check];
if(task_list.length>0){
async.series(
tasklist,
function(err, response) {
console.log(err);
console.log(response);
results.data=response;
results.message="Completed";
console.log(results);
}
);
}
Internal functions:
function api_hit(callback){
console.log("Inside api");
var ele=task_list[0];
var apidata=[];
var msg={'data':[]};
apiinfo.forEach((item,key)=>{
if(item.Method_name==ele.Parameters){
//Here checking random Int value
if(item.Value=="{{$randomInt}}"){
item.Value = generate(25);
}
apidata.push(item);
}
});
var data=[];
data['api']=apidata;
apiModel.validateAPI(data,function(res){
console.log("result api");
msg.data=res;
msg.case='api_hit';
callback(msg);
});
}
function delay(callback){
console.log("Inside delay");
var msg={'data':[]};
global_vars.sleep(1000);
msg.data='success';
msg.case='task';
console.log("after delay");
callback(msg);
}
function mysql_check(callback){
console.log("inside mysql");
var ele=task_list[2];
var dbdata=[];
var msg={'data':[]};
dbchecks.forEach((item,key)=>{
if(item.query_id==ele.Parameters){
console.log(item.query+" ::: "+ele.Parameters);
dbdata.push(item);
}
});
data['dbdata']=dbdata;
apiModel.checkmysql(data,function(err,res){
if(err) throw err;
console.log("inside mysql res");
msg.data=res;
msg.case='task2';
callback(msg);
});
}
My intention is to call these function after completing of others and all the results has to process in a single variable. but in api_hit method when it is executing another function inside of it then delay()(second function of async) is executing. how to stop this and make it in sequence. thanks in advance.
The first argument to the callback function is the error, pass null in case of success.
'use strict'
const async = require('async')
function api_hit(callback) {
setTimeout(() => {
console.log('Completed api_hit')
callback(null, 'api_hit')
}, 1000)
}
function delay(callback) {
setTimeout(() => {
console.log('Completed delay')
callback(null, 'delay')
}, 100)
}
function mysql_check(callback) {
setTimeout(() => {
console.log('Completed mysql_check')
callback(null, 'mysql_check')
}, 500)
}
var tasklist = [api_hit, delay, mysql_check];
if (tasklist.length > 0) {
async.series(
tasklist,
function (err, response) {
console.log(err);
console.log(response);
}
);
}
Doc link: https://caolan.github.io/async/docs.html#series
I'm using Async library for asynchronous programming on NodeJS. I want to make a helper method (that also use async method), rather than put all code in same place.
Here is my sample code:
var solve = function() {
async.waterfall([
// a long working task. huh
function (callback) {
setTimeout(function() {
console.log('hello world!');
callback();
}, 2000);
},
function (callback) {
authorService.helperAsync();
},
function (callback) {
setTimeout(function() {
console.log('bye bye!');
}, 2000);
}
]);
};
solve();
In other file, I make an async function:
var helperAsync = function() {
async.waterfall([
function(callback) {
console.log('task 1');
callback();
},
function (callback) {
console.log('task 2');
callback();
}
]);
}
I want the result should be:
Hello World
Task 1
Task 2
Bye Bye
But the output is only
Hello World
Task 1
Task 2
. How can I fix my code ?
Thanks :)
You need to setup each file to be used as a module, which involves exporting the function that you'd like to return, like so:
solve.js
var async = require('async');
var helperAsync = require('./helperAsync.js');
module.exports = function solve() {
async.waterfall([
function helloOne(next) {
setTimeout(function() {
console.log('hello world!');
return next();
}, 2000);
},
function helperCall(next) {
helperAsync(params, function(){
return next();
});
},
function byeBye(next) {
setTimeout(function() {
console.log('bye bye!');
return next();
}, 2000);
}
], function(result){
return result;
});
};
helperAsync.js
var async = require('async');
module.exports = function helperAsync (params, callback) {
async.waterfall([
function(next) {
console.log('task 1');
return next();
},
function (next) {
console.log('task 2');
return next();
}
], function(result){
return callback(result);
});
};
I've renamed the callback from your async.waterfall to next in order to prevent confusion with the main callback of the module.
Users upload files into my express app. I need to calc hash of the uploaded file and then write file to disk using calculated hash as a filename. I try to do it using the following code:
function storeFileStream(file, next) {
createFileHash(file, function(err, hash) {
if (err) {
return next(err);
}
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
return next(err);
});
stream.on('finish', function() {
return next();
});
file.pipe(stream);
});
}
function createFileHash(file, next) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
return next(err);
});
file.on('end', function(data) {
hash.end();
return next(null, hash.read());
});
file.pipe(hash);
}
The problem is that after I calc file hash the writed file size is 0. What is the best way do solve this task?
Update
According #poke suggestion I try to duplicate my stream. Now my code is:
function storeFileStream(file, next) {
var s1 = new pass;
var s2 = new pass;
file.pipe(s1);
file.pipe(s2);
createFileHash(s1, function(err, hash) {
if (err) {
return next(err);
}
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
return next(err);
});
stream.on('finish', function() {
return next();
});
s2.pipe(stream);
});
}
function createFileHash(file, next) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
return next(err);
});
file.on('end', function(data) {
hash.end();
return next(null, hash.read());
});
file.pipe(hash);
}
The problem of this code is that events end and finish are not emited. If I comment file.pipe(s2); events are emited, but I again get my origin problem.
This code fix the problem:
var s1 = new passThrough,
s2 = new passThrough;
file.on('data', function(data) {
s1.write(data);
s2.write(data);
});
file.on('end', function() {
s1.end();
s2.end();
});
The correct and simple way should be as follow:
we should resume the passthroughed stream
function storeFileStream(file, directory, version, reject, resolve) {
const fileHashSource = new PassThrough();
const writeSource = new PassThrough();
file.pipe(fileHashSource);
file.pipe(writeSource);
// this is the key point, see https://nodejs.org/api/stream.html#stream_three_states
fileHashSource.resume();
writeSource.resume();
createFileHash(fileHashSource, function(err, hash) {
if (err) {
return reject(err);
}
const fileName = path.join(directory, version + '_' + hash.slice(0, 8) + '.zip');
const writeStream = fs.createWriteStream(fileName);
writeStream.on('error', function(err) {
return reject(err);
});
writeStream.on('finish', function() {
return resolve();
});
writeSource.pipe(writeStream);
});
}
function createFileHash(readStream, next) {
const hash = crypto.createHash('sha1');
hash.setEncoding('hex');
hash.on('error', function(err) {
return next(err);
});
hash.on('finish', function(data) {
return next(null, hash.read());
});
readStream.pipe(hash);
}
You could use the async module (not tested but should work):
async.waterfall([
function(done) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
done(err);
});
file.on('end', function(data) {
done(null, hash.read);
});
file.pipe(hash);
},
function(hash, done) {
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
done(err);
});
stream.on('finish', function() {
done(null);
});
file.pipe(stream);
}
], function (err) {
console.log("Everything is done!");
});
I have a Node.js application where multiple funcions might be called, depending on several factors, but only one last function is called after the last callback.
This is a simplified version of what I got:
if(foo === bar){
function1(arg1, function(val1){
doWhatever(val1, function(){
res.end("Finished");
});
});
}else if(foo === baz){
function2(arg2, function(val2){
doWhatever(val2, function(){
res.end("Finished");
});
});
}else{
function3(arg3, function(val3){
doWhatever(val3, function(){
res.end("Finished");
});
});
}
And this is what im doing:
var finished = false;
if(foo === bar){
function1(arg1, function(val1){
result = val1;
finished = true;
});
}else if(foo === baz){
function2(arg2, function(val2){
result = val2;
finished = true;
});
}else{
function3(arg3, function(val3){
result = val3;
finished = true;
});
}
var id = setInterval(function(){
if(finished === true){
clearInterval(id);
doWhatever(result, function(){
res.end("Finished");
});
}
}, 100);
I guess this can be simplified by using promises, however im not sure how should I implement them.
You could also do it using when and promises, which IMHO is the easiest to read.
var promises = [];
if(x) {
var deferred1 = when.defer();
doSomethingAsync({ callback: deferred1.resolve });
promises.push(deferred1.promise);
} else if(y) {
var deferred2 = when.defer();
doSomethingAsync({ callback: deferred2.resolve });
promises.push(deferred2.promise);
} else if(z) {
var deferred3 = when.defer();
doSomethingAsync({ callback: deferred3.resolve });
promises.push(deferred3.promise);
}
when.all(promises).then(function () {
console.log('Finished Promises');
});
Here's one way with async series.
https://github.com/caolan/async#series
async.series([
function(callback){
if(foo === bar){
function1(arg1, function(val1){
callback(null, val1);
});
}else if(foo === baz){
function2(arg2, function(val2){
callback(null, val2);
});
}else{
function3(arg3, function(val3){
callback(null, val3);
});
}
}
], function(error, valArray){
doWhatever(valArray[0], function(){
res.end("Finished");
});
});
Here's using wait.for
https://github.com/luciotato/waitfor
//in a fiber
var result;
if(foo === bar){
result = wait.for(function1,arg1);
}else if(foo === baz){
result = wait.for(function2,arg2);
}else{
result = wait.for(function3,arg3);
};
doWhatever(result, function(){
res.end("Finished");
});
You need to be in a fiber (or generator) to use wait.for,
but, if you have a lot of callback hell,
wait.for is a good approach.
I have some synchronous code that looks like this:
function bulk_upload(files, callback) {
for (file in files) {
sync_upload(file); // blocks till file uploads
}
print('Done uploading files');
callback();
}
I now have to use an asynchronous API async_upload(file, callback) instead of sync_upload(file) to do the same. I have various options but not sure what is the best:
1) Use a sleep after the for-loop - that's a hack as I have to hope my timing is correct
2) Recursively chain my array:
function bulk_upload(files, callback) {
if (files.length == 0) {
print('Done uploading files');
callback();
} else {
async_upload(files.removeLast(), function() { bulk_upload(files, callback); });
}
}
This is not only hacky but sub-optimal as I could have uploaded my files in parallel using the new async_upload API but I ended up uploading sequentially.
3) Use a global counter:
function bulk_upload(files, callback) {
uploads = 0
for(file in files) {
async_upload(file, function() { uploads++; });
}
while(uploads < files.length) ; // Empty spin - this is stupid
print('Done uploading files');
callback();
}
4) Slightly better counter (but still awful):
function bulk_upload(files, callback) {
uploads = 0
for(file in files) {
async_upload(file, function() {
if (++uploads == files.length) { // this becomes uglier as I want to await on more stuff
print('Done uploading files');
callback();
};
});
}
}
You can use the async module's forEach method to do this:
function bulk_upload(files, callback) {
async.forEach(files, async_upload(file, callback), function (err) {
if (err) {
console.error('Failed: %s', err);
} else {
console.log('Done uploading files');
}
callback(err);
});
}
Further to my comment, it looks like this code will suffice using futures (untested).
function aync_upload_promise(file) {
// create a promise.
var promise = Futures.promise();
async_upload( file, function(err, data) {
if (err) {
// break it
promise.smash(err);
} else {
// fulfill it
promise.fulfill(data);
}
});
return promise;
}
var promises = [];
for(var i=0; i<files.length; ++i )
{
promises.push( aync_upload_promise( files[i] ) );
}
Futures
.join( promises )
.when( function() {
print('Done uploading files');
callback();
} )
.fail( function(err) { print('Failed :(', err); } )