I am trying to configure my application to use browserify with watchify and factor-bundle. However, I seem to either end up with a memory leak or break watchify triggered builds.
My build.js
var browserify = require('browserify'),
watchify = require('watchify'),
fromArgs = require('watchify/bin/args'),
fs = require('fs');
var inputs = [
'./client/src/application.js'
];
var outputs = [
'app/assets/javascripts/bundles/application.js'
];
var commonOutput = 'app/assets/javascripts/bundles/common.js';
var bundle = function(b){
b.plugin('factor-bundle', { outputs: outputs });
b.transform('browserify-shim');
b.bundle().
on('error', function(err) {
console.log(err.toString());
this.emit('end');
}).
pipe(fs.createWriteStream(commonOutput));
};
var watcher = watchify(browserify(watchify.args));
watcher.add(inputs);
watcher.on('update', function(ids) {
bundle(watcher);
console.log('updated with ids:', ids);
});
watcher.on('log', function(msg) {
console.log(msg);
});
bundle(watcher);
With this configuration I get eventually get:
(node) warning: possible EventEmitter memory leak detected. 11 listeners added. Use emitter.setMaxListeners() to increase limit.
Trace
at WriteStream.EventEmitter.addListener (events.js:160:15)
at Labeled.Readable.pipe (/Users/bmcmullen/dev/whitepages/fouroneone_line/node_modules/factor-bundle/node_modules/labeled-stream-splicer/node_modules/stream-splicer/node_modules/readable-stream/lib/_stream_readable.js:527:8)
at /Users/bmcmullen/dev/whitepages/fouroneone_line/node_modules/factor-bundle/index.js:73:43
at Array.reduce (native)
at Transform._flush (/Users/bmcmullen/dev/whitepages/fouroneone_line/node_modules/factor-bundle/index.js:65:35)
at Transform.<anonymous> (/Users/bmcmullen/dev/whitepages/fouroneone_line/node_modules/factor-bundle/node_modules/through2/node_modules/readable-stream/lib/_stream_transform.js:135:12)
at Transform.g (events.js:180:16)
...
I assume this is because I'm creating that write stream each time and the old ones are not being collected? But I can't figure out the right way to do it.
Related
I am trying to programm an converter which can take any video source and convert it to mp3. The mp3 should be saved on my hard drive, or in an buffer to send it via telegram.
It works good so far, the only problem I am facing is that it can only take one video at a time, and I don't know why.
// IMPORTS
var fs = require('fs');
var https = require('https');
var child_process = require('child_process');
// EVENTEMITER (Not used so far)
var util = require('util');
var EventEmitter = require('events').EventEmitter;
// STREAMHANDLER
var StreamHandler = function(url, name){
// VARIABLES
self = this;
this.url = url;
this.name = name;
// CREATE FFMPEG PROCESS
var spawn = child_process.spawn;
var args = ['-i', 'pipe:0', '-f', 'mp3', '-ac', '2', '-ab', '128k', '-acodec', 'libmp3lame', 'pipe:1'];
this.ffmpeg = spawn('ffmpeg', args);
// GRAB STREAM
https.get(url, function(res) {
res.pipe(self.ffmpeg.stdin);
});
// WRITE TO FILE
this.ffmpeg.stdout.pipe(fs.createWriteStream(name));
//DEBUG
this.ffmpeg.stdout.on("data", function (data) {
console.error(self.name);
});
}
util.inherits(StreamHandler, EventEmitter);
// TESTING
var test1 = new StreamHandler(vidUrl, "test1.mp3");
test1.ffmpeg.on("exit", function (code, name, signal) {
console.log("Finished: " + test1.name);
});
var test2 = new StreamHandler(vidUrl, "test2.mp3");
test2.ffmpeg.on("exit", function (code, name, signal) {
console.log("Finished: " + test2.name);
});
It skips test1.mp3 and only converts test2.mp3, but 2 ffmpeg processes were created:
After test2.mp3 is converted the other ffmpeg thread stays open, but does nothing, and the node program gets stuck waiting (i guess so) for it to send something.
I hope someone can help me :)
Using your code, I had the same problem. It would hang at the end and only output data for the test2.mp3 file. I'm not exactly sure what caused the problem, but I changed it a bit and this works for me:
// IMPORTS
var fs = require('fs');
//var https = require('https');
var http = require('http');
var child_process = require('child_process');
// EVENTEMITER (Not used so far)
var util = require('util');
var EventEmitter = require('events').EventEmitter;
// These never change...
var spawn = child_process.spawn;
var args = ['-i', 'pipe:0', '-f', 'mp3', '-ac', '2', '-ab', '128k', '-acodec', 'libmp3lame', 'pipe:1'];
// STREAMHANDLER
var StreamHandler = function(url, name){
// CREATE FFMPEG PROCESS
var ffmpeg = spawn('ffmpeg', args);
// GRAB STREAM
http.get(url, function(res) {
res.pipe(ffmpeg.stdin);
});
// WRITE TO FILE
ffmpeg.stdout.pipe(fs.createWriteStream(name));
ffmpeg.on("exit", function() {
console.log("Finished:", name);
});
//DEBUG
ffmpeg.stdout.on("data", function(data) {
console.error(name, "received data");
});
}
util.inherits(StreamHandler, EventEmitter);
// TESTING
var vidUrl = 'http://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4';
var test1 = new StreamHandler(vidUrl, "test1.mp3");
var test2 = new StreamHandler(vidUrl, "test2.mp3");
I am using http instead of https, because I didn't have a sample video at an https url available. It shouldn't make a difference.
I moved the spawn and args variables out of the object, because they don't change. I also do not use this to store the local variables. I just use a normal closure instead. Finally, I moved the exit event handling code inside the object. I just think it's better to group all that stuff together -- plus, it's only declared once rather than for each new process you create.
Running this gives me the following output (I saved the script as ffmpeg.js):
$ node ffmpeg.js
test2.mp3 received data
Finished: test2.mp3
test1.mp3 received data
Finished: test1.mp3
Also, just a tip. If you want to use the this object inside StreamHandler, I would recommend using arrow functions if your version of Node supports them. This code also works:
var StreamHandler = function(url, name){
// CREATE FFMPEG PROCESS
this.ffmpeg = spawn('ffmpeg', args);
// GRAB STREAM
http.get(url, (res) => {
res.pipe(this.ffmpeg.stdin);
});
// WRITE TO FILE
this.ffmpeg.stdout.pipe(fs.createWriteStream(name));
this.ffmpeg.on("exit", () => {
console.log("Finished:", name);
});
//DEBUG
this.ffmpeg.stdout.on("data", (data) => {
console.error(name, "received data");
});
}
Notice that with arrow functions, I don't have to use var self = this; Avoiding that is pretty much the reason arrow functions were added to javascript.
Hope this helps!
-- EDIT --
Ok, I figured it out. The problem in your code is this line:
self = this;
It should be:
var self = this;
Without the var specifier, you are creating a global variable. So, the second time you are calling new StreamHandler, you are overwriting the self variable. That's why the test1.mp3 file hangs and the test2.mp3 file is the only one finishing. By adding var, your original script now works for me.
I would like to use gulp to run a custom browserify command whenever a js file (function.js) is modified.
The browserify command that I want to run is;
$ browserify function.js --standalone function > bundle.js
I am using this gulpfile.js as sample.
https://github.com/gulpjs/gulp/blob/master/docs/recipes/fast-browserify-builds-with-watchify.md
How do I modify this gulpfile to run the customized browserify command?
'use strict';
var watchify = require('watchify');
var browserify = require('browserify');
var gulp = require('gulp');
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
var gutil = require('gulp-util');
var sourcemaps = require('gulp-sourcemaps');
var assign = require('lodash.assign');
// add custom browserify options here
var customOpts = {
entries: ['./src/index.js'],
debug: true
};
var opts = assign({}, watchify.args, customOpts);
var b = watchify(browserify(opts));
// add transformations here
// i.e. b.transform(coffeeify);
gulp.task('js', bundle); // so you can run `gulp js` to build the file
b.on('update', bundle); // on any dep update, runs the bundler
b.on('log', gutil.log); // output build logs to terminal
function bundle() {
return b.bundle()
// log errors if they happen
.on('error', gutil.log.bind(gutil, 'Browserify Error'))
.pipe(source('bundle.js'))
// optional, remove if you don't need to buffer file contents
.pipe(buffer())
// optional, remove if you dont want sourcemaps
.pipe(sourcemaps.init({loadMaps: true})) // loads map from browserify file
// Add transformation tasks to the pipeline here.
.pipe(sourcemaps.write('./')) // writes .map file
.pipe(gulp.dest('./dist'));
}
I am using node.js v6.9 on webstorm.
The command you want to run is;
$ browserify function.js --standalone function > bundle.js
Based on this, the modified code is;
// add custom browserify options here
var customOpts = {
entries: ['./function.js'],
standalone: 'function',
};
Simply add one more property to customOpts for the --standalone parameter. The rest of the code remains the same.
I'm using gulp with browserify and tsify. This has been working quite well. Then I decided to add ng-annotate using browserify-ngannotate.
I've added the ng-annotate browserify transform but it seems that if tsify is added as a plugin the ng-annotate transform is never called.
If I remove the tsify plugin then ng-annote gets called. I've played around and switched around the plugin/transform registration. Am I missing something here, or should I go and log an issue at browserify/tsify?
var browserify = require('browserify');
var browserSyncConfig = require('../config').browserSync;
var browserSync = require('browser-sync').get(browserSyncConfig.instance);
var watchify = require('watchify');
var tsify = require('tsify');
var ngAnnotate = require('browserify-ngannotate');
var mergeStream = require('merge-stream');
var bundleLogger = require('../util/bundleLogger');
var gulp = require('gulp');
var handleErrors = require('../util/handleErrors');
var source = require('vinyl-source-stream');
var config = require('../config').browserify;
var _ = require('lodash');
var browserifyTask = function (devMode) {
var browserifyThis = function (bundleConfig) {
if (devMode) {
// Add watchify args and debug (sourcemaps) option
_.extend(bundleConfig, watchify.args, {debug: true});
// A watchify require/external bug that prevents proper recompiling,
// so (for now) we'll ignore these options during development. Running
// `gulp browserify` directly will properly require and externalize.
bundleConfig = _.omit(bundleConfig, ['external', 'require']);
}
var b = browserify(bundleConfig);
if (bundleConfig.tsify) {
b = b.plugin(tsify, {
noImplicitAny: false,
target: 'ES5',
noExternalResolve: false,
module: 'commonjs',
removeComments: false
});
}
if (bundleConfig.ngAnnotate) {
b = b.transform(ngAnnotate);
}
var bundle = function () {
// Log when bundling starts
bundleLogger.start(bundleConfig.outputName);
return b
.bundle()
// Report compile errors
.on('error', handleErrors)
// Use vinyl-source-stream to make the
// stream gulp compatible. Specify the
// desired output filename here.
.pipe(source(bundleConfig.outputName))
// Specify the output destination
.pipe(gulp.dest(bundleConfig.dest))
.pipe(browserSync.stream());
};
if (devMode) {
// Wrap with watchify and rebundle on changes
b = watchify(b, {
poll: true
});
// Rebundle on update
b.on('update', bundle);
bundleLogger.watch(bundleConfig.outputName);
} else {
// Sort out shared dependencies.
// b.require exposes modules externally
if (bundleConfig.require) b.require(bundleConfig.require);
// b.external excludes modules from the bundle, and expects
// they'll be available externally
if (bundleConfig.external) b.external(bundleConfig.external);
}
return bundle();
};
// Start bundling with Browserify for each bundleConfig specified
return mergeStream.apply(gulp, _.map(config.bundleConfigs, browserifyThis));
};
gulp.task('browserify', function () {
return browserifyTask()
});
// Exporting the task so we can call it directly in our watch task, with the 'devMode' option
module.exports = browserifyTask;
You can solve it by specify extensions in ng-annotate options.
bundler.transform(ngAnnotate, { ext: ['.ts', '.js'] });
I realized I had this problem too, when I added uglifyify to the bundle transforms to produce minified builds.
An important aspect of my solution is that the missing, explicit $inject statements, that ng-annotate should have inserted, doesn't matter until the code is actually minified. Luckily, UglifyJS2, which does the actual minification in uglifyify, got support for handling ng-annotate's ngInject comments in version 2.4.9 (in January, 2014).
So, the solution that worked for me was to install uglifyify:
npm install --save-dev uglifyify
and add the following uglifyify transform to the Browserify bundle:
b.transform({
global: true,
mangle: false,
comments: true,
compress: {
angular: true
}
}, 'uglifyify');
This will make UglifyJS2 insert the appropriate $inject statements into your code before it is minified.
So, to summarize, I did not have a solution for only using ng-annotate, but my solution will add the necessary $inject statements before the code is minified, which is what matters in most cases.
I have Gulpfile with jshint configured to use jshint-stylish reporter. I need to pass option verbose to reporter in order to display warning codes. Is it possible to do it using Gulp?
Current my gulpfile.js looks like below:
var gulp = require('gulp');
var jshint = require('gulp-jshint');
var compass = require('gulp-compass');
var path = require('path');
require('shelljs/global');
var jsFiles = ['www/js/**/*.js', '!www/js/libraries/**/*.js', 'www/spec/**/*.js', '!www/spec/lib/**/*.js'];
var sassFiles = 'www/sass/*.scss';
gulp.task('lint', function () {
return gulp
.src(jsFiles)
.pipe(jshint())
.pipe(jshint.reporter('jshint-stylish'));
});
gulp.task('compass', function () {
gulp.src(sassFiles)
.pipe(compass({
project: path.join(__dirname, 'www'),
css: 'css',
sass: 'sass',
image: 'img',
font: 'fonts'
})).on('error', function() {});
});
var phonegapBuild = function (platform) {
if (!which('phonegap')) {
console.log('phonegap command not found')
return 1;
}
exec('phonegap local build ' + platform);
};
gulp.task('build:android', ['lint', 'compass'], function () {
phonegapBuild('android');
});
gulp.task('build:ios', ['lint', 'compass'], function () {
phonegapBuild('ios');
});
gulp.task('watch', function() {
gulp.watch(jsFiles, ['lint']);
gulp.watch(sassFiles, ['compass']);
});
gulp.task('default', ['lint', 'compass']);
Well, this, plus the fact that the output of the stylish reporter is hardly readable on Windows due to the darkness of the blue text, so I have to keep going in an manually changing the colour after installing it, has made me do something about it. So you should hopefully have more luck with this reporter I've just written:
https://github.com/spiralx/jshint-summary
You basically use it like this;
var summary = require('jshint-summary');
// ...
.pipe(jshint.reporter(summary({
verbose: true,
reasonCol: 'cyan,bold',
codeCol: 'green'
})
and the summary function will initialise the function passed to JSHint with those settings - see the page on Github for a bit more documentation.
It's got some very basic tests, and the library's gulpfile.js uses it to show its own JSHint output :)
How about using similar technique, as you already did with phonegap?
var jshint = function (parameter) {
// todo: define paths with js files, or pass them as parameter too
exec('jshint ' + paths + ' ' + parameter);
};
Based on https://github.com/wearefractal/gulp-jshint/blob/master/index.js#L99 it appears that gulp-jshint doesn't facilitate passing more than the name to the reporter if you load it with a string. It seems a simple thing to extend though. I'll race you to a pull request. :D
Alternatively, try something like this:
var stylish = require('jshint-stylish');
// ...
.pipe(jshint.reporter(stylish(opt)));
I'm pretty sure I have the syntax wrong, but this may get you unstuck.
It's annoying, and makes any decent reporter somewhat tricky to use within the existing framework. I've come up with this hack for the Stylish reporter, it's just currently in my gulpfile.js:
function wrapStylishReporter(reporterOptions) {
var reporter = require(stylish).reporter,
reporterOptions = reporterOptions || {};
var wrapped = function(results, data, config) {
var opts = [config, reporterOptions].reduce(function(dest, src) {
if (src) {
for (var k in src) {
dest[k] = src[k];
}
}
return dest;
}, {});
reporter(results, data, opts);
};
return jshint.reporter(wrapped);
}
And then for the task definition itself:
gulp.task('lint', function() {
return gulp.src('+(bin|lib)/**/*.js')
.pipe(jshint())
.pipe(wrapStylishReporter({ verbose: true }))
.pipe(jshint.reporter('fail'));
});
Ideally reporters would either be a function that takes an options parameter and returns the reporter function, or a fairly basic class so you could have options as well as state.
I have been try out many suggestions I found googling for circular dependency in node and requirejs. Unfortunately, I'm not getting it to work. The try which is closed to a solution (I think) is below:
// run.js
var requirejs = require('requirejs');
requirejs.config({
baseUrl: __dirname,
nodeRequire: require
});
requirejs(['A'], function(A) {
var a = new A.Go();
console.log(a.toon())
});
// A.js
define(['B', 'exports'], function(B, exports) {
exports.Go = function() {
var b = new require('B').Ho();
var toon = function() {
return 'me tarzan';
};
return {
b: b,
toon: toon
}
};
});
// B.js
define(['A', 'exports'], function(A, exports) {
exports.Ho = function() {
var a = new require('A').Go();
var show = function() {
return 'you jane';
}
return {
a: a,
show: show
}
};
});
Running this code in node results in a RangeError: Maximum call stack size exceeded
We the dependency of B is removed from A.js, 'me tarzan' is returned
Any suggestion is appreciated!
Circular references are fine and not necessarily a symptom of bad design. You might argue that having many tiny modules could be equally detrimental because code/logic is scattered.
To avoid the dreaded TypeError: Object #<Object> has no method you need to take some care in how you initialize module.exports. I'm sure something similar applies when using requirejs in node, but I haven't used requirejs in node.
The problem is caused by node having an empty reference for the module. It is easily fixed by assigning a value to the exports before you call require.
function ModuleA() {
}
module.exports = ModuleA; // before you call require the export is initialized
var moduleB = require('./b'); //now b.js can safely include ModuleA
ModuleA.hello = function () {
console.log('hello!');
};
This sample is from https://coderwall.com/p/myzvmg where more info available.