Node.js streams => Delete contents of folder before transpiling to it - node.js

I am using Gulp and I would like to delete/"clean" all contents of a directory before writing transpiled source to it.
Is there a way to connect the stream that runs the delete operation to the stream that does the read/transform/write operation?
Right now I have this:
function transpileJSX() {
var d = domain.create();
d.run(function () {
gulp.src('./public/static/app/js/views/**/*.js').pipe(react({harmony: true}))
.pipe(gulp.dest('./public/static/app/js/jsx')).on('end', function () {
d.emit('end');
d.exit();
});
});
return d;
}
function deleteJSX() {
return gulp.src('./public/static/app/js/jsx/**/*.js')
.pipe(clean({force: true}));
}
gulp.task('transpile-jsx', function () {
return transpileJSX().on('error', function (err) {
console.error(err);
});
});
gulp.task('delete-jsx', function (cb) {
return deleteJSX();
});
the problem is that this doesn't seem to work:
deleteJSX().pipe(transpileJSX());
the transpileJSX operation seems to start before the deleteJSX operation begins. And that might be expected when it comes to streams. But I think I want to make sure the whole thing works properly by ensuring the entire delete operation completes first?
For example, if I do this:
gulp.task('some-task', ['delete-jsx','transpile-jsx'], function (done) {
someTask(done);
});
the gulp logs show that the transpile-jsx task starts before the delete-jsx task ends.
If someone has a pattern to use when it comes to cleaning out a folder before transpiling to it, I'd like to copy it thanks!

You're almost there already. You just have to rearrange task dependencies, since gulp runs tasks for maximum concurrency:
gulp.task('transpile-jsx', ['delete-jsx'], function () {
//...
});
gulp.task('some-task', ['transpile-jsx'], function () {
//...
});
This will be easier once Gulp 4 hits. There you will be able to write stuff like:
gulp.task('some-task', gulp.series('delete-jsx', 'transpile-jsx', function () {
//...
}));

Related

How to make gulp not finish task before it's done?

I have this gulp task, and it works fine, but when looking at the log, it always says Starting Task, then immediately Finished Task, then it starts logging everything. How do I make it wait until everything is complete until it says it's finished? I think it's due to the function being async so done() gets called right away, but I'm not sure what to do.
gulp function
gulp.task("img-projects", function(done) {
gulp.src('projects/*/images/**/*.*')
.pipe(imagemin())
.pipe(rename(function (path) {
path.dirname = path.dirname.replace('/images','');
console.log('path');
return path;
}))
.pipe(gulp.dest('public'));
done();
});
output:
[19:11:55] Starting 'js-projects'...
[19:11:55] Finished 'js-projects' after 34 ms
[19:11:55] path
[19:11:55] path
[19:11:55] path
All you need to add is an on('end', ...) listener to wait until the gulp stream is complete before calling done():
gulp.task("img-projects", function(done) {
gulp.src('projects/*/images/**/*.*')
.pipe(imagemin())
.pipe(rename(function (path) {
path.dirname = path.dirname.replace('/images','');
console.log('path');
return path;
}))
.pipe(gulp.dest('public'))
.on('end', done);
});
Sources:
How do you run a gulp "on end" task but only at the end of the current task?
gulp API (docs for src()/dest()/etc): https://github.com/gulpjs/gulp/tree/master/docs/api
node.js streams API which provides on(...): https://nodejs.org/api/stream.html
similar issue: https://github.com/gulpjs/gulp/issues/1181#issuecomment-126694791
Try,
The gulp functions are async
gulp.task("img-projects", function() {
return gulp.src('projects/*/images/**/*.*')
.pipe(imagemin())
.pipe(rename(function (path) {
path.dirname = path.dirname.replace('/images','');
console.log('path');
return path;
}))
.pipe(gulp.dest('public'));
});

mocha/chai return undefined

I have a working node script and am now trying to learn mocha/chai to add some unit tests.
//Module Code
var path = require('path');
var recursive = require('recursive-readdir');
function findData(folderPath) {
recursive(folderPath, function (err, files) {
return files;
});
};
module.exports.findData = findData;
My mocha test code:
var expect = require('chai').expect;
describe('checkData', function () {
var findData = require('../custom_modules/findData').findData;
var path;
before (function () {
path = '/Data'
});
it('should have 53 files in array', function () {
expect(findData(path)).to.have.lengthOf(53);
})
});
However, it always fails because the return seems to be undefined. So i stripped my module code back to test a return true and that worked.
So it must the the asynchronous nature of the recursive module so I then tried add in callbacks to my module code:
var path = require('path');
var recursive = require('recursive-readdir');
function findData(folderPath, cb) {
recursive(folderPath, function (err, files) {
cb(null, files);
});
};
module.exports.findData = findData;
But that didn't work either.
The weird thing is if I run node index.js i get the list of files.
Can anyone explain to me how this code can work normally but when I try to use mocha/chai to test I get undefined?
Thanks
EDITED:
So based on what #Louis said in the comments I have added a callback to the function.
describe('checkData', function () {
var findData = require('../custom_modules/findData').findData;
var path;
var files;
before (function () {
path = '/Users/tjmartin/Documents/OSData/OpenNames/Data'
});
it('should have 53 files in array', function () {
expect(findData(path, function(results) {
files = results;
})).to.have.lengthOf(53);
})
});
But this still returns an undefined.
First up, I would log the err result in your findData implementation. Even if only during development, so you can easily see if any errors are reported (you may be doing this already, just wanted to mention it).
As you have spotted, one of the main causes of problems for you is that the callback is asynchronous. So you can't simply return it from your findData method as in your original example.
Next, I wouldn't hard-code the path as you have in the before function. Instead use a local path, so that the test can be run as part of your CI (if you ever have one) or even so you can grab it on another machine and have it work there.
before(function() {
path = './tests/TestData';
});
In your revised example, although you are using a callback you are testing the return result still. You need to alter your test to use the result of the callback.
it('should have 53 files in array', function(done) {
findData(path, function(results) {
expect(results).to.have.lengthOf(53);
done();
});
});

How to create parameterized and reusable gulp tasks

Is there a way to make this generic to the point where I can have one copy of it and pass the config item, and file list into it rather than duplicating it for every file/config combination?
I'd love to have something like
gulp.task('foo_test', function (cb) {
run_tests(files.foo_list, config.fooCoverage);
cb();
}
Note on potential oddities in the code
I'm using lazypipe and gulp-load-plugins full file here
// test the server functions and collect coverage data
gulp.task('api_test', function (cb) {
gulp.src(files.api_files)
.pipe(istanbulPre())
.on('end', function () {
gulp.src(files.api_test_files)
.pipe(mochaTask())
.pipe(istanbulAPI())
.on('end', cb);
});
});
var istanbulAPI = lazypipe()
.pipe(plugins.istanbul.writeReports, config.apiCoverage);
config = {
apiCoverage: {
reporters: ['json'],
reportOpts: {
json: {
dir: 'coverage',
file: 'coverage-api.json'
}
}
},
Gulp is just JavaScript.
You can write plain old regular functions, just as you normally would:
function run_tests(srcFiles, srcTestFiles, coverageConfig, cb) {
var istanbul = lazypipe()
.pipe(plugins.istanbul.writeReports, coverageConfig);
gulp.src(srcFiles)
.pipe(istanbulPre())
.on('end', function () {
gulp.src(srcTestFiles)
.pipe(mochaTask())
.pipe(istanbul())
.on('end', cb);
});
}
gulp.task('unit_test', function (cb) {
run_tests(files.lib_files, files.unit_test_files, config.unitCoverage, cb);
});
gulp.task('api_test', function (cb) {
run_tests(files.api_files, files.api_test_files, config.apiCoverage, cb);
});
Note that the callback cb is just another parameter that is passed to the run_tests function. If it was called immediately after calling run_tests that would signal task completion to gulp before the asynchronous code in run_tests has actually finished.
Lazypipe was a solution (and there are other alternatives) but since Gulp 4 these do not seem to work anymore. Gulp 4 does not pass the stream to pipe functions. Yet the gulp.src(...) function returns a stream.
Also a nice feature of gulp 4 is that functions with Promises can also be a task.
So in the end I came up with this solution that worked for me. With my gulpfile.js looking something like this:
const {
src,
dest,
series,
parallel
} = require('gulp');
// other gulp packages omitted in this example...
const myReusableJsParser = (sources, destination) => {
return src(sources)
.pipe(stripComments({...}))
.pipe(obfuscator({compact:true}))
.pipe(...) //etc
.pipe(dest(destination));
};
const parseScriptsX = () => {
return myReusableJsParser('./js/x/**/*.js', './dist/js/x/');
}
const parseScriptsY = () => {
return myReusableJsParser('./js/y/**/*.js', './dist/js/y/');
}
// more
...
const cleanUp = () => new Promise((resolve, reject) => {
try {
deleteFiles('./dist/').then(resolve).catch(reject);
} catch(err) {
reject(err);
}
});
// export
module.exports = {
default: series(cleanUp, paralell(parseScriptsX, parseScriptsY), paralell(...)),
...,
clean: cleanUp
};

Yeoman generator: how to run async command after all files copied

I'm writing a yeoman generator.
I need to run some shell script after all files copied.
The generator is being called as a child generator so it should wait till script finished.
The script is some command file being run via spawn:
that.spawnCommand('createdb.cmd');
As the script depends on files created by the generator it cannot run right inside generator's methods as all copy/template action are async and have not executed yet:
MyGenerator.prototype.serverApp = function serverApp() {
if (this.useLocalDb) {
this.copy('App_Data/createdb.cmd', 'App_Data/createdb.cmd');
// here I cannot run spawn with createdb.cmd as it doesn't exist
}
}
So the only place I found where I can run spawn is the 'end' event handler:
var MyGenerator = module.exports = function MyGenerator (args, options, config) {
this.on('end', function () {
if (that.useLocalDb) {
that.spawnCommand('createdb.cmd')
}
}
}
The script runs successfully but the generator finishes earlier than the child process. I need to tell Yeoman to wait for my child process.
Something like this:
this.on('end', function (done) {
this.spawnCommand('createdb.cmd')
.on('close', function () {
done();
});
}.bind(this));
But 'end' handler doesn't have the argument with 'dine' callback.
How to do this?
UPDATE:
thanks to #SimonBoudrias I got it working.
The full working code is below.
BTW: end method is described in the docs
var MyGenerator = module.exports = yeoman.generators.Base.extend({
constructor: function (args, options, config) {
yeoman.generators.Base.apply(this, arguments);
this.appName = this.options.appName;
},
prompting : function () {
// asking user
},
writing : function () {
// copying files
},
end: function () {
var that = this;
if (this.useLocalDb) {
var done = this.async();
process.chdir('App_Data');
this.spawnCommand('createdb.cmd').on('close', function () {
that._sayGoodbay();
done();
});
process.chdir('..');
} else {
this._sayGoodbay();
}
},
_sayGoodbay: funciton () {
// final words to user
}
});
Never trigger any action in the end event. This event is to be used by implementors, not generator themselves.
In your case:
module.exports = generators.Base({
end: function () {
var done = this.async();
this.spawnCommand('createdb.cmd').on('close', done);
}
});

Using a node module within a Grunt Task fails

I'm trying to extract meta data from files read within a Grunt task.
executing: node test.js on this file:
var exif = require('exif2');
exif('fixtures/forest.png', function (err, o) {
console.log(arguments);
});
Produces the expected output
However, executing the grunt process: grunt projectJSON
module.exports = function (grunt) {
var exif = require('exif2');
return grunt.registerMultiTask("projectJSON", "Creates project JSON file.", function () {
exif('fixtures/forest.png', function (err, o) {
console.log(arguments);
});
});
}
** note that I am just testing with the fixtures/forest.png file
Produces no output whatsoever. The callback isn't even fired.
When I console.log exif, I get: [Function]
What am I missing? I think that the doesn't work is because of the grunt task, but I have no idea how to fix it. Wrapping it in a try-catch block produces nothing.
You need to make your projectJSON task asynchronous - Grunt is exiting before your exif callback is being invoked.
Have a look at the Grunt documentation on asynchronous tasks.
This is how you can make your task asynchronous:
module.exports = function (grunt) {
var exif = require('exif2');
grunt.registerMultiTask("projectJSON", "Creates project JSON file.", function () {
// Make task asynchronous.
var done = this.async();
exif('fixtures/forest.png', function (err, o) {
console.log(arguments);
// Invoke the task callback to continue with
// other Grunt tasks.
done();
});
});
}

Resources