I am trying to write the results of 3 Mongoose find query into a single file test.txt using event-stream nodejs.
My program is something like this…
var es = require('event-stream');
var fs = require('fs');
var wstream = fs.createWriteStream('TestFile.csv');
mongoose.connection.on('connected', function () {
//First Find Query
Model.find({"FirstId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
callback(null, csv)
}))
.pipe(wstream);
//Second find query
Model.find({"SecondId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
if(data.Email)
{
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
}
else{
callback();
}
callback(null, csv)
}))
.pipe(wstream);
//Third find query
Model.find({"ThirdId":{$exist:true}).cursor()
.pipe(es.map(function (data, callback) {
if(data.Email)
{
var csv = json2csv({data:formated, fields:fields, hasCSVColumnTitle:false});
}
else{
callback();
}
callback(null, csv)
}))
.pipe(wstream);
});
In this program I am able to write each find query result separately into the file. But when I combine 3-find query in a program, it throws “write after end” error.
Could anyone help me to resolve this error?
Thank You all for your time!
Related
I'm learning how to use the csv-parse module for nodejs. I wrote this code and it works perfectly:
var fs = require('fs');
const fileName = './spreadsheet.csv';
const assert = require('assert');
const { parse } = require('csv-parse');
const records = [];
// Initialize the parser
const parser = parse({
delimiter: ','
});
// Use the readable stream api to consume records
parser.on('readable', function(){
let record;
while ((record = parser.read()) !== null) {
records.push(record);
}
});
// Catch any error
parser.on('error', function(err){
console.error(err.message);
});
fs.readFile(fileName, 'utf8', function (err, f) {
if (err) {
return console.error(err);
}
const rows = f.split("\r\n");
for(let x in rows) {
parser.write(rows[x]+"\n");
}
parser.end();
console.log(records);
});
But right now, I depend on the fs module and fs.readFile to consume my csv file. Does the csv-parse have an option to read ffrom file? I ask because as you can see in my code, I ahve to specify my own line-break characters, which could differ between csv files. I thought maybe the csv-parse module would have something that can more readily address such a situation?
The parser object will do most of the work for you. It is expecting the data to arrive on its stream interface and it will do everything else. All you have to do is open a stream and the pipe it to the parser like this:
fs.createReadStream(fileName).pipe(parser);
And, here it is combined with your code:
const fs = require('fs');
const fileName = './spreadsheet.csv';
const { parse } = require('csv-parse');
const records = [];
// Initialize the parser
const parser = parse({
delimiter: ','
});
// Use the readable stream api to consume records
parser.on('readable', function(){
let record;
while ((record = parser.read()) !== null) {
records.push(record);
}
});
// Catch any error
parser.on('error', function(err){
console.error(err.message);
});
parser.on('end', function() {
console.log(records);
});
// open the file and pipe it into the parser
fs.createReadStream(fileName).pipe(parser);
P.S. It's amazing that such a simple example of getting the CSV data from a file is not shown in the documentation (at least not anywhere I could find it). I'm also surprised, they don't offer an option where they will automatically read the data from the stream, instead requiring you to implement the readable event handler. Odd, for such an otherwise complete package.
I am using the EJS compile to create notification templates and I would like to know how to write the file to the file system in parallel and send the notification once all the files are saved.
Please see the below code snippet which I used
var fs = require('fs');
var ejs = require('ejs');
var arrayOfData = [someData]; //Prepare data from database
//Iterate through the data
for (var i = 0; i < arrayOfData.length; i++) {
generateFileFromTemplate(arrayOfData[i],function(){});
}
function generateFileFromTemplate(templateData,callback)
{
var outputFile = fileData.Id + ".html";
var compiled = ejs.compile(fs.readFileSync('email-template.ejs', 'utf8'));
var html = compiled(templateData);
fs.writeFile(outputFile, html, callback);
}
Please help.
Use async.each for your use case
async.each(arrayOfData,
function(ele, next){
generateFileFromTemplate(ele,function(){});
},
function(err){
if(err) console.log('err', err);
sendNotification();
}
);
You can use a great utility library called Async, particularly its parallel method: https://github.com/caolan/async#parallel.
Here's an example:
var async = require('async');
/*-------------*/
var tasks = arrayOfData.map(function(data) {
return function(cb) {
generateFileFromTemplate(data,function(){});
cb(null);
}
});
async.parallel(tasks, function(err) {
console.log('My job is done');
})
I need to download ~26k images. The images list and urls are stored in csv file. Im reading the csv file and trying to download the images while looping through the list.
If im using small set ~1-2k it works fine but when i switch to the full set im getting EMFILE error.
Error: EMFILE, open 'S:\images_download\Images\189900008.jpg'
I've noticed that node tries to create all the files at once and this might be the issue but i'm unable to force it to create it one by one. My understanding is the code below should work like this but obviously is not.
(Just to mention that this code is executed on Windows)
Code:
var csv = require("fast-csv");
var fs = require('fs');
var request = require('request');
var async = require('async');
fs.writeFile('errors.txt', '', function(){})
var downloaded = 0;
var totalImages = 0;
var files = [];
csv
.fromPath("Device_Images_List.csv")
.on("data", function(data){
files.push({device: data[0], url: data[1]})
})
.on("end", function(){
totalImages = files.length;
async.each(files, function(file, callback) {
var deviceId = file.device;
var deviceUrl = file.url;
if ( deviceId != 'DEVICE_TYPE_KEY' ) {
try {
writeStream = fs.createWriteStream('./Images/' + deviceId + '.jpg');
proxiedRequest = request.defaults({proxy: "http://proxy:8080"});
proxiedRequest(deviceUrl).pipe(writeStream);
writeStream.on('open', function(fd) {
var rem = proxiedRequest.get(deviceUrl);
rem.on('data', function(chunk) {
writeStream.write(chunk);
});
rem.on('end', function() {
downloaded++;
console.log('Downloaded: ' + deviceId + '; ' + (downloaded + 1) + ' of ' + totalImages);
writeStream.end();
});
});
writeStream.on('close', function(){
callback();
});
} catch (ex) {
fs.appendFile('errors.txt', deviceId + ' failed to download', function (err) {
callback();
});
}
}
}, function(err){
if( err ) {
console.log(err);
} else {
}
});
});
As #slebetman commented the issue can be solved by using async.eachSeries to process the files one by one or async.eachLimit to limit the parallel nodes:
async.eachLimit(files, 5, function(file, callback) {
// ... Process 5 files at the same time
}, function(err){
});
I would like to read a csv file and upload each row to a couchdb using a grunt task. At this point I am not yet doing any database validation such as checking if the record already exists but will have to do that at some point also.
Currently this is what I am doing and the problem is only the first 65 rows, of the first sub task named people is being uploaded to couchdb.
I know this has something to do with asynchronous execution but just can't work out how to do this
Gruntils.js
csv2couch: {
people: {
db: 'http://localhost:5984/db',
collectionName: 'person',
src:['./data/schema3/people.csv']
},
organisms: {
db: '<%= qmconfig.COUCHDBURL %>',
collectionName: 'organism',
src:['./data/schema3/organisms.csv']
}
}
csv2couch.js
'use strict';
var nanolib = require('nano'),
csv = require('csv'),
urls = require('url'),
fs = require('fs');
module.exports = function(grunt) {
grunt.registerMultiTask('csv2couch', 'Parse csv file and upload data to couchdb.', function() {
var done, parts, dbname, _this, collectionName;
_this = this;
done = this.async();
parts = urls.parse(this.data.db);
dbname = parts.pathname.replace(/^\//, '');
collectionName = this.data.collectionName;
// Merge task-specific and/or target-specific options with these defaults.
var options = this.options({});
// couchdb connection
try {
var nano = nanolib(parts.protocol + '//' + parts.host);
} catch (e) {
grunt.warn(e);
done(e, null);
}
// database connection
var db = nano.use(dbname);
// process each source csv file
this.filesSrc.forEach(function(f) {
console.log('source file:', f);
csv()
.from.path(f, {
columns:true,
delimeter:',',
quote:'"'
})
.on('record', function(row,index){
console.log('#'+index, row);
save(row, collectionName);
})
.on('end', function(count){
console.log('Number of lines: '+count);
done();
})
.on('error', function(error){
console.log(error.message);
done(error);
});
});
function save (data, collectionName) {
// document ID is concatenation of collectionName and ID
var docID = collectionName[0]+'_'+data.ID;
// add some additional data
data.type = collectionName;
// insert data into couchdb
db.insert(data, docID, function(err, body, header) {
if (err) {
console.log('[db.insert] ', err.message);
return;
}
});
}
});
};
You're right, the async code is incorrect. The CSV file is being read to the end before all your records are saved. You need to call done only when your last record has been saved.
Your save method needs to take a callback
var rowsRead = 0, // the number of rows read from the csv file
rowsWritten = 0; // the number of rows written to CouchdDb
caller:
.on('record', function(row,index){
rowsRead++;
save(row, collectionName, function(err){
if(err){
return done(err);
}
rowsWritten++;
if(rowsRead===rowsWritten){ // check if we've written all records to CouchDb
done();
}
});
})
save method:
function save (data, collectionName, callback) {
// document ID is concatenation of collectionName and ID
var docID = collectionName[0]+'_'+data.ID;
// add some additional data
data.type = collectionName;
// insert data into couchdb
db.insert(data, docID, callback);
}
I have a problem in getting a .json file in express and displaying in a view. Kindly share your examples.
var fs = require("fs"),
json;
function readJsonFileSync(filepath, encoding){
if (typeof (encoding) == 'undefined'){
encoding = 'utf8';
}
var file = fs.readFileSync(filepath, encoding);
return JSON.parse(file);
}
function getConfig(file){
var filepath = __dirname + '/' + file;
return readJsonFileSync(filepath);
}
//assume that config.json is in application root
json = getConfig('config.json');
Do something like this in your controller.
To get the json file's content :
ES5
var foo = require('./path/to/your/file.json');
ES6
import foo from './path/to/your/file.json';
To send the json to your view:
function getJson(req, res, next){
res.send(foo);
}
This should send the json content to your view via a request.
NOTE
According to BTMPL
While this will work, do take note that require calls are cached and will return the same object on each subsequent call. Any change you make to the .json file when the server is running will not be reflected in subsequent responses from the server.
This one worked for me. Using fs module:
var fs = require('fs');
function readJSONFile(filename, callback) {
fs.readFile(filename, function (err, data) {
if(err) {
callback(err);
return;
}
try {
callback(null, JSON.parse(data));
} catch(exception) {
callback(exception);
}
});
}
Usage:
readJSONFile('../../data.json', function (err, json) {
if(err) { throw err; }
console.log(json);
});
Source