Generate SourceMaps for a single js file which includes modules with npm require and gulp - require

I'll begin straight off with an example of my code structure. Assume the following three trivial files reside inside the same directory called /path/from/root/js/src
module1.js:
console.log(1);
module2.js:
console.log(2);
app.js:
require('./module1');
require('./module2');
Then, I am using the following gulp task to compile javascript into one file with gulp:
var gulp = require('gulp');
var sourcemaps = require('gulp-sourcemaps');
var path = require('path');
var browserify = require('gulp-browserify');
gulp.task('default', function() {
gulp.src(['./js/src/app.js'])
.pipe(sourcemaps.init())
.pipe(browserify()).on('error', function(err){
console.log(err);
})
.pipe(sourcemaps.write('.'))
.pipe(gulp.dest(function( file ) {
file.base = path.dirname(file.path);
return path.join(path.dirname(file.path), '/../');
}))
});
After running gulp I am getting the compiled javascript app.js which just logs 1 2 and an app.js.map as expected, but there is no reference to the original file in the browser.
You can check that by looking at the console lines 1 and 2, they are referenced by app.js, not by module1|2.js. If I needed to fix a bug I'd have no idea which file is generating the console notations in the future as the files grow bigger in my project.
What am I doing wrong? Am I not using the sourcemaps correctly?
The app.js.map file doesn't reference the modules, it looks like this:
{
"version":3,
"names":[],
"mappings":"",
"sources":["app.js"],
"sourcesContent":["require('./module1');\r\nrequire('./module2');"],
"file":"app.js"
}

Using rollup
After trying various stuff and build packages online, I have found a solution to my problem with the use of rollup instead of browserify. In case someone is interested, I will add here my trivial build in basic usage with rollup :
module1.js and module2.js remain as are.
app.js becomes
import {m1} from './module1';
import {m2} from './module2';
And my gulpfile becomes
var gulp = require('gulp'),
rollup = require('rollup')
;
gulp.task('default', function () {
return rollup.rollup({
entry: "./js/src/app.js",
})
.then(function (bundle) {
bundle.write({
format: "umd",
moduleName: "library",
dest: "./js/app.js",
sourceMap: true
});
})
});
Now open your html file which includes /js/app.js and you'll see in your console 1 referenced by module1.js and 2 referenced by module2.js
Looks like rollup doesn't support require by default, but the import {...} from ... syntax is more minimalistic and part of ES6, so it might be better for me to start using it instead.
Source from official documentation: https://rollupjs.org/#using-rollup-with-gulp
Further reading for a more complex build (I haven't walked through these steps yet, but looks promising): https://github.com/rollup/rollup-starter-project
All hail mighty messy malicious Javascript for the troubles it leads upon us !

I have found a more relative solution to the problem, so I'm creating a new answer for it. I have noticed that the gulp-browserify package I was using was deprecated, so I checked for an updated usage.
My package.json dependencies are:
"devDependencies": {
"babel-preset-es2015": "^6.24.1",
"babelify": "^7.3.0",
"browserify": "^14.4.0",
"gulp": "^3.9.1",
"gulp-sourcemaps": "^2.6.0",
"gulp-uglify": "^3.0.0",
"gulp-util": "^3.0.8",
"vinyl-buffer": "^1.0.0",
"vinyl-source-stream": "^1.1.0"
}
My gulpfile.js for a basic build looks like this:
'use strict';
var browserify = require('browserify');
var gulp = require('gulp');
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
var uglify = require('gulp-uglify');
var sourcemaps = require('gulp-sourcemaps');
var gutil = require('gulp-util');
gulp.task('default', function () {
// set up the browserify instance on a task basis
var b = browserify({
entries: './js/src/app.js',
debug: true
}).transform("babelify", {presets: ["es2015"]});
return b.bundle()
.pipe(source('app.js'))
.pipe(buffer())
.pipe(sourcemaps.init({loadMaps: true}))
// Add transformation tasks to the pipeline here.
//.pipe(uglify())
.on('error', gutil.log)
.pipe(sourcemaps.write('./'))
.pipe(gulp.dest('./js/'));
});
For a production build, you can uncomment the uglify command.
Babelify is there to allow ES5 syntax (didn't test, but probably you can instead use the *2017 package too).
Sources:
1. Gulp setup for Browserify
2. Babel setup for Browserify

Related

gulp can't find module source-map

I'm having an issue with sourcemaps in gulp: when I try to implement gulp-sourcemaps directly through a pipe, but also when I try to feed it in via webpack, which if I've understood correctly, has sourcemaps by default, I keep getting the same confusing error:
Error: Cannot find module 'source-map'
I can see the #gulp-sourcemaps folder in my node_modules, which contains within it an identity-map folder containing a bunch more node_modules including the source-map module in question, so I think everything is hooked up as it should be. That being said, I'm new to gulp, so I might be missing something really obvious.
Can anyone offer me some guidance on how to help gulp find the module?
// package vars
const pkg = require("./package.json");
// gulp
const gulp = require("gulp");
const gulpIf = require("gulp-if");
// webpack
const webpack_config = require("./webpack.config");
// load all plugins in "devDependencies" into the variable $
const $ = require("gulp-load-plugins")({
pattern: ["*"],
scope: ["devDependencies"]
});
// ...
gulp.task("sass", function() {
return gulp
.src(pkg.paths.app.scss + "**/*.scss")
.pipe($.sourcemaps.init())
.pipe($.sass())
.pipe($.autoprefixer())
.pipe($.sourcemaps.write("./maps"))
.pipe(gulp.dest(pkg.paths.app.css))
.pipe(
$.browserSync.reload({
stream: true
})
);
});
// ...
gulp.task("webpack", function() {
return gulp
.src(pkg.paths.app.js + "**/*.js")
.pipe($.webpack(webpack_config))
.pipe(gulp.dest(pkg.paths.public.js));
});
// ...

require is not defined error on gulp-concat

I want to uglify and combine my js files with gulp. Here is my code
var gulp = require('gulp');
var uglify = require('gulp-uglify');
var pump = require('pump');
var gutil = require('gulp-util');
gutil.env.type = 'production';
gulp.task('uglify', function (cb) {
return gulp.src([
'pure/modernizr.js',
'pure/horizon.js'
])
.pipe(gutil.env.type === 'production' ? uglify() : gutil.noop())
.pipe(sourcemaps.write('.'))
.pipe(gulp.dest("ugly"));
});
var sourcemaps = require("gulp-sourcemaps");
var concat = require("gulp-concat-js");
gulp.task("concat", function () {
return gulp.src([
'ugly/modernizr.js',
'ugly/horizon.js'
])
.pipe(sourcemaps.init())
.pipe(concat({
"target": "concatenated.js", // Name to concatenate to
"entry": "./main.js" // Entrypoint for the application, main module
// The `./` part is important! The path is relative to
// whatever gulp decides is the base-path, in this
// example that is `./lib`
}))
.pipe(sourcemaps.write('.'))
.pipe(gulp.dest("pure/"));
});
I end up with some code enclosing my js files from obfuscator stating:
//// THIS FILE IS CONCATENATED WITH gulp-obfuscator-js
When I include this in code, it throws require is not defined, I surf around the web and found one similar question. But that answer is also not clear for me. I believe that I miss some small thing here, since I am new to gulp.
The issue here is I have used gulp-concat-js which obfuscate your js. I should have used gulp-concat. May help someone.

Browserify + Remapify (Gulp workflow)

I've successfully got Browserify to compile my JavaScript entry files, but I want to utilise the Remapify plugin so as to not have to specify the full relative path upon requiring a module every time.
For example:
require('components/tabs.js')
Rather than:
require('../../components/tabs/tabs.js').
But I cannot get the shorter module references to map to the corresponding file... "Error: Cannot find module [specified_ref] from [file]".
Have I misconfigured Remapify, or is there something wrong with my wider Browserify setup? I am new to Broswerify and Gulp having previously used Require.js and Grunt. Any help would be greatly appreciated. Please let me know if you need any more information about my setup.
If alternatively you can recommend an alternative Gulp task file that will do all of this, thereby throwing my current task out the window, by all means. I wasn't able to find many Browserify + Remapify examples.
Directory Structure
I have my modules (components) in the following directory: './src/components', so for example: './src/components/tabs/tabs.js'.
I am requiring these modules in a JS file for a given page of the app, which are in: './src/pages', so for example, './src/pages/portfolio/portfolio.js'.
Gulp Browserify Task
var gulp = require('gulp');
var config = require('../config');
var browserify = require('browserify');
var remapify = require('remapify');
var source = require('vinyl-source-stream');
var glob = require('glob');
var browserSync = require('browser-sync');
gulp.task('browserify', function(){
var entries = glob.sync(config.src.pages + '/**/*.js');
return browserify({
entries: entries,
debug: true
})
// (Remapify:)
.plugin(remapify, [{ src: config.src.components + '/**/*.js', expose: 'components', cwd: config.srcDir }])
.bundle()
.pipe(source('app.js'))
.pipe(gulp.dest(config.build.js))
.pipe(browserSync.reload({ stream: true }));
});
Page.js
'use strict';
var tabs = require('components/tabs.js'); // (Doesn't work, but I want it to)
// var tabs = require('../../components/tabs/tabs.js'); // (Does work)
Remapify has all sorts of problems. I suggest giving my pathmodify plugin a shot.
For your situation usage would look something like:
var pathmod = require('pathmodify');
// ...
.plugin(pathmod(), {mods: [
pathmod.mod.dir('components', '/path/to/src/components'),
]})

Importing Sass through npm

Currently in our Sass files we have something like the following:
#import "../../node_modules/some-module/sass/app";
This is bad, because we're not actually sure of the path: it could be ../node_modules, it could be ../../../../../node_modules, because of how npm installs stuff.
Is there a way in Sass that we can search up until we find node_modules? Or even a proper way of including Sass through npm?
If you are looking for a handy answer in 2017 and are using Webpack, this was the easiest I found.
Suppose your module path is like:
node_modules/some-module/sass/app
Then in your main scss file you can use:
#import "~some-module/sass/app";
Tilde operator shall resolve any import as a module.
As Oncle Tom mentioned, the new version of Sass has this new importer option, where every "import" you do on your Sass file will go first through this method. That means that you can then modify the actual url of this method.
I've used require.resolve to locate the actual module entry file.
Have a look at my gulp task and see if it helps you:
'use strict';
var path = require('path'),
gulp = require('gulp'),
sass = require('gulp-sass');
var aliases = {};
/**
* Will look for .scss|sass files inside the node_modules folder
*/
function npmModule(url, file, done) {
// check if the path was already found and cached
if(aliases[url]) {
return done({ file:aliases[url] });
}
// look for modules installed through npm
try {
var newPath = path.relative('./css', require.resolve(url));
aliases[url] = newPath; // cache this request
return done({ file:newPath });
} catch(e) {
// if your module could not be found, just return the original url
aliases[url] = url;
return done({ file:url });
}
}
gulp.task("style", function() {
return gulp.src('./css/app.scss')
.pipe(sass({ importer:npmModule }))
.pipe(gulp.dest('./css'));
});
Now let's say you installed inuit-normalize using node. You can simply "require" it on your Sass file:
#import "inuit-normalize";
I hope that helps you and others. Because adding relative paths is always a pain in the ass :)
You can add another includePaths to your render options.
Plain example
Snippet based on example from Oncle Tom.
var options = {
file: './sample.scss',
includePaths: [
path.join(__dirname, 'bower_components'), // bower
path.join(__dirname, 'node_modules') // npm
]
};
sass.render(options, function(err, result){
console.log(result.css.toString());
});
That should do. You can include the files from package using #import "my-cool-package/super-grid
Webpack and scss-loader example
{
test: /\.scss$/,
loader: 'style!css!autoprefixer?browsers=last 2 version!sass?outputStyle=expanded&sourceMap=true&sourceMapContents=true&includePaths[]=./node_modules'
},
Notice the last argument, includePaths has to be array. Keep in mind to use right format
You can use a Sass importer function to do so. Cf. https://github.com/sass/node-sass#importer--v200.
The following example illustrates node-sass#3.0.0 with node#0.12.2:
Install the bower dependency:
$ bower install sass-mq
$ npm install sass/node-sass#3.0.0-pre
The Sass file:
#import 'sass-mq/mq';
body {
#include mq($from: mobile) {
color: red;
}
#include mq($until: tablet) {
color: blue;
}
}
The node renderer file:
'use strict';
var sass = require('node-sass');
var path = require('path');
var fs = require('fs');
var options = {
file: './sample.scss',
importer: function bowerModule(url, file, done){
var bowerComponent = url.split(path.sep)[0];
if (bowerComponent !== url) {
fs.access(path.join(__dirname, 'bower_components', bowerComponent), fs.R_OK, function(err){
if (err) {
return done({ file: url });
}
var newUrl = path.join(__dirname, 'bower_components', url);
done({ file: newUrl });
})
}
else {
done({ file: url });
}
}
};
sass.render(options, function(err, result){
if (err) {
console.error(err);
return;
}
console.log(result.css.toString());
});
This one is simple and not recursive. The require.resolve function could help to deal with the tree – or wait until npm#3.0.0 to benefit from the flat dependency tree.
I made the sass-npm module specifically for this.
npm install sass-npm
In your SASS:
// Since node_modules/npm-module-name/style.scss exists, this will be imported.
#import "npm-module-name";
// Since just-a-sass-file isn't an installed npm module, it will be imported as a regular SCSS file.
#import "just-a-sass-file";
I normally use gulp-sass (which has the same 'importer' option as regular SASS)
var gulp = require('gulp'),
sass = require('gulp-sass'),
sassNpm = require('sass-npm')();
Then, in your .pipe(sass()), add the importer as an option:
.pipe(sass({
paths: ['public/scss'],
importer: sassNpm.importer,
}))
For dart-sass and commandline user at 2022, just use the --load-path option:
$ npx sass --load-path=node_modules
Important: the whole node_modules folder contains so much, just set it launch extremely slow in watch mode. Your should only set your package paths, eg:
$npx sass -w --load-path=node_modules/foo --load-path=node_modules/bar/scss
From offical docuumentation of Sass, adding ~ to imports should do the job.
However, for some reason it did'nt work for me, and sass compiler still complains that the module cannot be found.
Hence, I tried another method which worked for me without any issues. Here's the solution:
If you are compiling sass files directly from CLI try this:
sass src/main.scss dist/main.css --load-path=node_modules
If you are using npm and/or webpack for compiling sass files, add something like this to the scripts of package.json:
"scripts": {
...
"build": "sass src/main.scss dist/main.css --load-path=node_modules",
...
}
Then Run:
npm run build
Finally, import your modules like this:
#import "some-module/sass/app";
To wrap it up, adding --load-path=node_modules flag solved the issue permanently. For more information you can check:
sass --help

Using Gulp to build requireJS project - gulp-requirejs

I am trying to use gulp-requirejs to build a demo project. I expect result to be a single file with all js dependencies and template included. Here is my gulpfile.js
var gulp = require('gulp');
var rjs = require('gulp-requirejs');
var paths = {
scripts: ['app/**/*.js'],
images: 'app/img/**/*'
};
gulp.task('requirejsBuild', function() {
rjs({
name: 'main',
baseUrl: './app',
out: 'result.js'
})
.pipe(gulp.dest('app/dist'));
});
// The default task (called when you run `gulp` from cli)
gulp.task('default', ['requirejsBuild']);
The above build file works with no error, but the result.js only contains the content of main.js and config.js. All the view files, jquery, underscore, backbone is not included.
How can I configure gulp-requirejs to put every js template into one js file?
If it is not the right way to go, can you please suggest other method?
Edit
config.js
require.config({
paths: {
"almond": "/bower_components/almond/almond",
"underscore": "/bower_components/lodash/dist/lodash.underscore",
"jquery": "/bower_components/jquery/dist/jquery",
"backbone": "/bower_components/backbone/backbone",
"text":"/bower_components/requirejs-text/text",
"book": "./model-book"
}
});
main.js
// Break out the application running from the configuration definition to
// assist with testing.
require(["config"], function() {
// Kick off the application.
require(["app", "router"], function(app, Router) {
// Define your master router on the application namespace and trigger all
// navigation from this instance.
app.router = new Router();
// Trigger the initial route and enable HTML5 History API support, set the
// root folder to '/' by default. Change in app.js.
Backbone.history.start({ pushState: false, root: '/' });
});
});
The output is just a combination this two files, which is not what I expected.
gulp-requirejs has been blacklisted by the gulp folks. They see the RequireJS optimizer as its own build system, incompatible with gulp. I don't know much about that, but I did find an alternative in amd-optimize that worked for me.
npm install amd-optimize --save-dev
Then in your gulpfile:
var amdOptimize = require('amd-optimize');
var concat = require('gulp-concat');
gulp.task('bundle', function ()
{
return gulp.src('**/*.js')
.pipe(amdOptimize('main'))
.pipe(concat('main-bundle.js'))
.pipe(gulp.dest('dist'));
});
The output of amdOptimize is a stream which contains the dependencies of the primary module (main in the above example) in an order that resolves correctly when loaded. These files are then concatenated together via concat into a single file main-bundle.js before being written into the dist folder.
You could also minify this file and perform other transformations as needed.
As an aside, in my case I was compiling TypeScript into AMD modules for bundling. Thinking this through further I realized that when bundling everything I don't need the asynchronous loading provided by AMD/RequireJS. I am going to experiment with having TypeScript compile CommonJS modules instead, then bundling them using webpack or browserify, both of which seem to have good support within gulp.
UPDATE
My previous answer always reported taskReady even if requirejs reported an error. I reconsidered this approach and added error logging. Also I try to fail the build completely as described here gulp-jshint: How to fail the build? because a silent fail really eats your time.
See updated code below.
Drew's comment about blacklist was very helpfull and gulp folks suggest using requirejs directly. So I post my direct requirejs solution:
var DIST = './dist';
var requirejs = require('requirejs');
var requirejsConfig = require('./requireConfig.js').RJSConfig;
gulp.task('requirejs', function (taskReady) {
requirejsConfig.name = 'index';
requirejsConfig.out = DIST + 'app.js';
requirejsConfig.optimize = 'uglify';
requirejs.optimize(requirejsConfig, function () {
taskReady();
}, function (error) {
console.error('requirejs task failed', JSON.stringify(error))
process.exit(1);
});
});
The file at ./dist/app.js is built and uglified. And this way gulp will know when require has finished building. So the task can be used as a dependency.
My solution works like this:
./client/js/main.js:
require.config({
paths: {
jquery: "../vendor/jquery/dist/jquery",
...
},
shim: {
...
}
});
define(["jquery"], function($) {
console.log($);
});
./gulpfile.js:
var gulp = require('gulp'),
....
amdOptimize = require("amd-optimize"),
concat = require('gulp-concat'),
...
gulp.task('scripts', function(cb) {
var js = gulp.src(path.scripts + '.js')
.pipe(cached('scripts'))
.pipe(jshint())
.pipe(jshint.reporter('default'))
.pipe(remember('scripts'))
.pipe(amdOptimize("main",
{
name: "main",
configFile: "./client/js/main.js",
baseUrl: './client/js'
}
))
.pipe(concat('main.js'));
.pipe(gulp.dest(path.destScripts));
}
...
This part was important:
configFile: "./client/js/main.js",
baseUrl: './client/js'
This allowed me to keep my configuration in one place. Otherwise I was having to duplicate my paths and shims into gulpfile.js.
This works for me. I seems that one ought to add in uglification etc via gulp if desired. .pipe(uglify()) ...
Currently I have to duplicate the config in main.js to run asynchronously.
....
var amdOptimize = require("amd-optimize");
...
var js = gulp.src(path.scripts + '.js')
.pipe(cached('scripts'))
.pipe(jshint())
.pipe(jshint.reporter('default'))
.pipe(remember('scripts'))
.pipe(amdOptimize("main",
{
name: "main",
paths: {
jquery: "client/vendor/jquery/dist/jquery",
jqueryColor: "client/vendor/jquery-color/jquery.color",
bootstrap: "client/vendor/bootstrap/dist/js/bootstrap",
underscore: "client/vendor/underscore-amd/underscore"
},
shim: {
jqueryColor : {
deps: ["jquery"]
},
bootstrap: {
deps: ["jquery"]
},
app: {
deps: ["bootstrap", "jqueryColor", "jquery"]
}
}
}
))
.pipe(concat('main.js'));
Try this code in your gulpfile:
// Node modules
var
fs = require('fs'),
vm = require('vm'),
merge = require('deeply');
// Gulp and plugins
var
gulp = require('gulp'),
gulprjs= require('gulp-requirejs-bundler');
// Config
var
requireJsRuntimeConfig = vm.runInNewContext(fs.readFileSync('app/config.js') + '; require;'),
requireJsOptimizerConfig = merge(requireJsRuntimeConfig, {
name: 'main',
baseUrl: './app',
out: 'result.js',
paths: {
requireLib: 'bower_modules/requirejs/require'
},
insertRequire: ['main'],
// aliases from config.js - libs will be included to result.js
include: [
'requireLib',
"almond",
"underscore",
"jquery",
"backbone",
"text",
"book"
]
});
gulp.task('requirejsBuild', ['component-scripts', 'external-scripts'], function (cb) {
return gulprjs(requireJsOptimizerConfig)
.pipe(gulp.dest('app/dist'));
});
Sorry for my english. This solution works for me. (I used gulp-requirejs at my job)
I think you've forgotten to set mainConfigFile in your gulpfile.js. So, this code will be work
gulp.task('requirejsBuild', function() {
rjs({
name: 'main',
mainConfigFile: 'path_to_config/config.js',
baseUrl: './app',
out: 'result.js'
})
.pipe(gulp.dest('app/dist'));
});
In addition, I think when you run that task in gulp, require can not find its config file and
This is not gulp-requirejs fault.
The reason why only main.js and config.js is in the output is because you're not requiring/defining any other files. Without doing so, the require optimizer wont understand which files to add, the paths in your config-file isn't a way to require them!
For example you could load a main.js file from your config file and in main define all your files (not optimal but just a an example).
In the bottom of your config-file:
// Load the main app module to start the app
requirejs(["main"]);
The main.js-file: (just adding jquery to show the technique.
define(["jquery"], function($) {});
I might also recommend gulp-requirejs-optimize instead, mainly because it adds the minification/obfuscation functions gulp-requirejs lacks: https://github.com/jlouns/gulp-requirejs-optimize
How to implement it:
var requirejsOptimize = require('gulp-requirejs-optimize');
gulp.task('requirejsoptimize', function () {
return gulp.src('src/js/require.config.js')
.pipe(requirejsOptimize(function(file) {
return {
baseUrl: "src/js",
mainConfigFile: 'src/js/require.config.js',
paths: {
requireLib: "vendor/require/require"
},
include: "requireLib",
name: "require.config",
out: "dist/js/bundle2.js"
};
})).pipe(gulp.dest(''));
});

Resources