I have a functionality on a webpage, that uploads an excel file to Node and parse it to JSON, then pass the data to mongodb. It only sends one document to mongodb, each document is organized inside of an array and iterate through each one using a for loop, so the only one that is being sent is the first document. I tried also to use the model.create(docs) function which sends each document to the database but is the same issue. This is the code(the model.create(docs) is inside the //////////////):
app.post('/upload', function(req, res){
var exceltojson;
upload(req, res, function(err){
if (err) {
res.json({error_code:1,err_desc:err})
return;
}
if(!req.file){
res.json({error_code:1, err_desc:"No file passed"});
return;
}
if(req.file.originalname.split('.')[req.file.originalname.split('.').length-1] === 'xlsx'){
exceltojson = xlsxtojson;
} else {
exceltojson = xlstojson;
}
try {
exceltojson({
input: req.file.path,
output: "./outPutJSON/output.json",
lowerCaseHeaders: true
}, function(err, result){
if(err){
return res.json({error_code:1, err_desc:err, data: null});
}
res.json({datos:"Los datos fueron agregados exitosamente"});
//res.json({error_code:0, err_desc:null, data: result});
let resultDos = fs.readFile("./outPutJSON/output.json", 'utf8', (err, fileContents) => {
if (err) {
console.error(err)
return;
}
try {
const data = JSON.parse(fileContents)
console.log(data.length);
//////////////////////////////////////////////////
model.create(data, function (err) {
if(err){
console.log(err);
}
});
///////////////////////////////////////////////////
//for(var cantidad = 0; cantidad < data.length;cantidad++{
//let documento = data[cantidad];
//let mod = new model(documento);
//console.log(documento);
// mod.save(function(err){
// if(err){
// console.log(err);
// }
// });
//}
//////////////////////////////////////////////////////
} catch(err) {
console.error(err);
}
})
console.log(resultDos);
});
var fs = require('fs');
try {
fs.unlinkSync(req.file.path)
}catch(e){
}
} catch (e) {
res.json({error_code:1, err_desc:"Corrupted excel file"});
}
});
});
This is the JSON file:
Only this one is sent -->{"nombre":"Wilson Junior Toribio","cedula":"4022589632","direccion":"Calle 7 #33 Buenos Aires"},
{"nombre":"Jose Luis Toribio","cedula":"4023495023","direccion":"Calle 11 # 69 Buenos Aires"},
{"nombre":"Joel de Jesus Toribio","cedula":"4023548902","direccion":"Calle 1 # 3 Buenos Aires"},
{"nombre":"Corazon Roa","cedula":"4026984452","direccion":"Calle 3 # 19 Buenos Aires"}
I even output each document to verify if the documents are being store in the variable, this is the output:
The problem has been solve, I had to edit the callback to be synchronous using async and await and also use let to declare the variable cantidad inside the for:
let resultDos = fs.readFile("./outPutJSON/output.json", 'utf8', -> async (err, fileContents) => {
if (err) {
console.error(err)
return;
}
try {
let data = JSON.parse(fileContents)
console.log(data.length);
console.log(data);
// model.create(data, function (err) {
// if(err){
// console.log(err);
// }
// });
for(let cantidad = 0; cantidad < data.length; cantidad++){
var documento = data[cantidad];
var mod = new model(documento);
console.log(documento);
-> await mod.save(documento);
// model.create(documento).save();
}
Related
I am using node-async-loop for asyncronous programming
var array = ['item0', 'item1', 'item2'];
asyncLoop(array, function (item, next)
{
do.some.action(item, function (err)
{
if (err)
{
next(err);
return;
}
next();
});
}, function (err)
{
if (err)
{
console.error('Error: ' + err.message);
return;
}
console.log('Finished!');
});
Like this I am using three async loops one under one.
I want to send the response only after the third inner loop ends. How can I do so?.
Here is the link for node-async-loop (https://www.npmjs.com/package/node-async-loop)
here is my code which i writing but whnever i want to response when the last loop completes it say can set header after send to cliend.
also in console log i am getting data every time when data coming from query.
const id = req.params.id;
finalData = [];
tb_user.findOne({ where: { id: id } }).then((userRiverSys, err) => {
if (userRiverSys) {
// console.log(userRiverSys.regionJson)
asyncLoop(userRiverSys.regionJson, function (item, next) {
// console.log("item", item);
tb_riverSystems.findAll(
{
where: { regionId: item.id }
}).then((findriverSys, err) => {
if (err) {
next(err);
return;
}
// console.log("findriverSys", findriverSys);
if (findriverSys) {
asyncLoop(findriverSys, function (item1, next1) {
if (err) {
next(err);
return;
}
// console.log("item1", item1.dataValues);
tb_facilities.findAll(
{
where: { riverSystemId: item1.dataValues.id }
}).then((findFacilities) => {
if (findFacilities) {
// console.log("findFacilities", findFacilities[0].dataValues.name);
asyncLoop(findFacilities, function (item2, next2) {
if (err) {
next(err);
return;
}
tb_userAccess.findAll(
{
where: { facilityId: item2.dataValues.id }
}).then((userAccessFacilities, err) => {
// console.log("userAccessFacilities", userAccessFacilities[0].dataValues);
// var i = 0;
asyncLoop(userAccessFacilities, function (item3, next3) {
finalData.push({
UserId: item3.userid,
facilityId: item3.facilityId,
})
next3();
},
function (err) {
if (err) {
console.error('Error: ' + err.message);
return;
}
// i++;
// console.log('Finished!!!!');
// if (userAccessFacilities.length === i) {
// console.log("finalData", i);
// // res.json({"status":"true", "message":"update OrgChallenge"})
// }
})
return res.json({"status":"true", "message":"update OrgChallenge"})
// console.log("finalData", finalData);
})
next2();
}, function (err) {
if (err) {
console.error('Error: ' + err.message);
return;
}
console.log('Finished!!!');
});
}
});
next1();
}, function (err) {
if (err) {
console.error('Error: ' + err.message);
return;
}
console.log('Finished!!');
});
}
});
next();
}, function (err) {
if (err) {
console.error('Error: ' + err.message);
return;
}
console.log('Finished!');
});
} else {
console.log("err3", err)
}
})
If you promisify your asynchronous action (so it returns a promise), then you can just use a regular for loop and async/await and there is no need for a 3rd party library to sequence your asynchronous loop. This is modern Javascript:
const { promisify } = require('util');
do.some.actionP = promisify(do.some.action);
async function someFunction() {
const array = ['item0', 'item1', 'item2'];
for (let item of array) {
let result = await do.some.actionP(item);
// do something with result here
}
return someFinalResult;
}
someFunction().then(result => {
console.log(result);
}).catch(err => {
console.log(err);
});
FYI, in real code, many (or even most) asynchronous operations now offer promisified versions of their API already so usually you don't even need to do the promisify step any more. For example, pretty much all databases already offer a promise interface that you can just use directly.
const loop = async (arr, results = []) => {
const item = arr.shift()
if (!item) {
console.log("DONE");
return results;
}
// as async function
await new Promise(resolve => {
resolve(results.push(`asynced-${item}`))
})
return loop(arr, results);
}
(async () => {
const result = await loop(["item0", "item1", "item2"])
console.log(result);
})();
I'd be happy if I can help you.
but this script uses a recursive function instead of node-async-loop.
so this might not be suitable for you.
I am trying to call method get_radar_life_cycle from app.get("/api/radar_cloning and it throws the error shown below,
error is coming from line return done(null,documents) how do I return documents back to my API call?
METHODS:-
let get_cloned_radars = function(documents, done) {
let complete_radar_list=[]
for (let i = 0; i < documents.length; i++) {
complete_radar_list.push(documents[i]['orgRadar']);
for (let j = 0; j < documents[i]['clonedRadarsdetailslist'].length; j++) {
complete_radar_list.push(documents[i]['clonedRadarsdetailslist'][j]['clonedRadar']);
}
}
data = complete_radar_list
return done(null, data)
}
let get_radar_life_cycle = function(data,done) {
console.log("data after get_radar_life_cycle")
console.log(data)
Radar_life_cycle.find({orgRadar: {$in:data}})
.then(documents => {
console.log(documents) --> shows correct data
});
return done(null,documents) --> Error is coming from this line
};
API call:
app.get("/api/radar_cloning", (req, res, next) => {
Radar_cloning.find({orgRadar: {$in:req.query.params.split(',')}})
.then(documents => {
get_cloned_radars(documents, function(err,data) {
if (err) {
res.json(err);
if (data!=null){
console.log(data)
}//end for data
}//end of (Err)
});//get_cloned_radars
get_radar_life_cycle(data, function(err,radar_life_cycle_data) {
if (err) {
res.json(err);
console.log(radar_life_cycle_data)
}//end for radar_life_cycle_data
}//end of (Err)
});//end of get_radar_life_cycle
});
});
ERROR:-
(node:10065) UnhandledPromiseRejectionWarning: ReferenceError: documents is not defined
You are trying to access documents outside of its scope, the scope being everything between the { and }. So you cannot access it below the .then(() => {}) scope.
Luckily you are are providing a callback function called done(err, radar_life_cycle_data), which you can use anywhere in the scope of the get_radar_life_cycle(documents, done) function. Even the scopes inside of its scope. When you are calling the done function, what you are basically doing is calling this function (well it has some syntax errors, so I cleaned it up)
function(err,radar_life_cycle_data) {
if (err) {
res.json(err);
console.log(radar_life_cycle_data)
}
}//end for radar_life_cycle_data
which then gets executed
So the solution:
Move your done in your .then(() => {}) scope like this:
let get_radar_life_cycle = function(data,done) {
console.log("data after get_radar_life_cycle")
console.log(data)
Radar_life_cycle.find({orgRadar: {$in:data}})
.then(documents => {
console.log(documents) // --> shows correct data
done(null,documents) // --> No error coming from this line
});
};
Same goes for the data it is not in the scope of the get_cloned_radars
app.get("/api/radar_cloning", (req, res, next) => {
Radar_cloning.find({orgRadar: {$in:req.query.params.split(',')}})
.then(documents => {
get_cloned_radars(documents, function(err,data) {
if (err) {
res.json(err);
if (data!=null) {
console.log(data)
get_radar_life_cycle(data, function(err,radar_life_cycle_data) {
if (err) {
res.json(err);
console.log(radar_life_cycle_data)
} //end of (Err)
}); //end of get_radar_life_cycle
} //end for data
} //end of (Err)
}); //get_cloned_radars
});
But since your code is unreadable, here is a cleaned up version:
app.get("/api/radar_cloning", (req, res, next) => {
const radar_life_cycle_cb = function (err, data) {
if (err) {
res.json(err);
return;
}
console.log(data);
}
const cloned_radar_cb = function (err, data) {
if (err) {
res.json(err);
return;
}
if (data != null) {
get_radar_life_cycle(data, radar_life_cycle_cb);
}
};
Radar_cloning.find({orgRadar: {$in:req.query.params.split(',')}})
.then(documents => get_cloned_radars(documents, cloned_radar_cb));
}
I am learning node.js and database. I am trying to stream heavy data about 7,700,000 rows and 96 columns from oracle to client. Later i use that data for virtual table. But in client it is showing only one row and then in node command error is displaying "Cannot set headers after they are sent to the client". How to stream data in client. Please help
var oracledb = require('oracledb');
const cors = require('cors');
var express = require('express');
var app = express();
app.use(cors());
oracledb.outFormat = oracledb.ARRAY;
oracledb.getConnection({
user: 'user',
password: 'password',
connectString: 'some string'
},
(err, connection) => {
if (err) {
console.error(err.message);
return;
}
var rowsProcessed = 0;
var startTime = Date.now();
var dataSize = 0;
var stream = connection.queryStream(
'SELECT * FROM table',
);
// stream.on('data', function (data) {
// rowsProcessed++;
// // console.log(JSON.stringify(data));
// // console.log(data);
// dataSize = dataSize + data.length;
// // oracleData.push(data);
// // console.log("pushing");
// // console.log(oracleData);
// // app.get('/data', (req, res) => {
// // res.send(data);
// // })
// // console.log(data);
// });
app.get('/data', (req, res) => {
stream.on('data', (data) => {
rowsProcessed++;
dataSize = dataSize + data.length;
res.send(JSON.stringify(data));
})
})
stream.on('end', function () {
var t = ((Date.now() - startTime) / 1000);
console.log('queryStream(): rows: ' + rowsProcessed +
', seconds: ' + t);
// console.log(dataSize + ' bytes');
connection.close(
function (err) {
if (err) {
console.error(err.message);
} else {
console.log("connection closed")
}
}
)
})
}
);
app.listen(5000, () => {
console.log('Listening at 5000')
})
I tried using above approach. But it is failing. How can I achieve the output?
The browser is freezing if I output entire data at single time that's why I am trying to use streaming and in the node command prompt it is displaying out of memory if I load entire data at single time.
Thank you.
The first thing you'll want to do is organize your app a little better. Separation of concerns is important, you should have a connection pool, etc. Have a look at this series for some ideas: https://jsao.io/2018/03/creating-a-rest-api-with-node-js-and-oracle-database/
Once you get the organization figured out, incorporate this example of streaming a large result set out.
const oracledb = require('oracledb');
async function get(req, res, next) {
try {
const conn = await oracledb.getConnection();
const stream = await conn.queryStream('select * from employees', [], {outFormat: oracledb.OBJECT});
res.writeHead(200, {'Content-Type': 'application/json'});
res.write('[');
stream.on('data', (row) => {
res.write(JSON.stringify(row));
res.write(',');
});
stream.on('end', () => {
res.end(']');
});
stream.on('close', async () => {
try {
await conn.close();
} catch (err) {
console.log(err);
}
});
stream.on('error', async (err) => {
next(err);
try {
await conn.close();
} catch (err) {
console.log(err);
}
});
} catch (err) {
next(err);
}
}
module.exports.get = get;
If you find you're doing this a lot, simplify things by creating a reusable transform stream:
const oracledb = require('oracledb');
const { Transform } = require('stream');
class ToJSONArray extends Transform {
constructor() {
super({objectMode: true});
this.push('[');
}
_transform (row, encoding, callback) {
if (this._prevRow) {
this.push(JSON.stringify(this._prevRow));
this.push(',');
}
this._prevRow = row;
callback(null);
}
_flush (done) {
if (this._prevRow) {
this.push(JSON.stringify(this._prevRow));
}
this.push(']');
delete this._prevRow;
done();
}
}
async function get(req, res, next) {
try {
const toJSONArray = new ToJSONArray();
const conn = await oracledb.getConnection();
const stream = await conn.queryStream('select * from employees', [], {outFormat: oracledb.OBJECT});
res.writeHead(200, {'Content-Type': 'application/json'});
stream.pipe(toJSONArray).pipe(res);
stream.on('close', async () => {
try {
await conn.close();
} catch (err) {
console.log(err);
}
});
stream.on('error', async (err) => {
next(err);
try {
await conn.close();
} catch (err) {
console.log(err);
}
});
} catch (err) {
next(err);
}
}
module.exports.get = get;
Here I am trying to create a new train and adding seat available information for next 5 days to that train. Here I am doing things repetitively. I wanted to know how to simplify this code.
here I have used express, mongoose and moment.js to generate date.
app.post("/admin",[
hasAccess('admin'),
function(req,res,next){
var today = moment();
Train.create(req.body.train,function(err,train){
if(err){
console.log(err);
}else{
Availability.create({date:moment(today).add(0, 'days').format("DD-MM-YYYY"),available:10},function(err,avail){
if(err){
console.log(err);
}else{
train.availabilities.push(avail);
Availability.create({date:moment(today).add(1, 'days').format("DD-MM-YYYY"),available:10},function(err,avail){
if(err){
console.log(err);
}else{
train.availabilities.push(avail);
Availability.create({date:moment(today).add(2, 'days').format("DD-MM-YYYY"),available:10},function(err,avail){
if(err){
console.log(err);
}else{
train.availabilities.push(avail);
Availability.create({date:moment(today).add(3, 'days').format("DD-MM-YYYY"),available:10},function(err,avail){
if(err){
console.log(err);
}else{
train.availabilities.push(avail);
Availability.create({date:moment(today).add(4, 'days').format("DD-MM-YYYY"),available:10},function(err,avail){
if(err){
console.log(err);
}else{
train.availabilities.push(avail);
train.save();
res.redirect("/trains/"+train._id);
}
})
}
})
}
})
}
})
}
})
}
});
}
]);
1 - Using promises
app.post("/admin", [
hasAccess('admin'),
function (req, res, next) {
var today = moment();
const days = [0, 1, 2, 3, 4];
let createdTrain;
Train.create(req.body.train)
.then(train => {
createdTrain = train;
return Promise.all(days.map(day => {
return Availability.create({
date: moment(today).add(day, 'days').format("DD-MM-YYYY"),
available: 10
});
}));
})
.then(result => {
createdTrain.availabilities = result;
createdTrain.save();
})
.then(saved => {
res.redirect("/trains/" + saved._id);
})
.catch(err => {
console.log(err);
});
}
]);
2 - Using async await [node 7.6 +]
app.post("/admin", [
hasAccess("admin"),
async (req, res, next) => {
try {
const today = moment();
const DAYS = [0, 1, 2, 3, 4];
const train = await Train.create(req.body.train);
const result = await Promise.all(
DAYS.map(day => {
return Availability.create({
date: moment(today)
.add(day, "days")
.format("DD-MM-YYYY"),
available: 10
});
})
);
train.availabilities = result;
const saved = await train.save();
res.redirect("/trains/" + saved._id);
} catch (err) {
console.log(err);
}
}
]);
You can also use async package for creating availability object in parallel by using async.parallel, this make your execution fast and create one function that can create availability object.
app.post("/admin", [
hasAccess('admin'),
function (req, res, next) {
var today = moment();
Train.create(req.body.train, function (err, train) {
if (err) {
console.log(err);
} else {
var functionArr = [];
for (var i = 0; i < 5; i++) {
var availFunction = function (i, callback) {
availability(moment(today), 0, 10, function (err, avail) {
if (err) {
return callback(err);
}
return callback(null, avail);
});
}
functionArr.push(availFunction);
}
async.parallel([
functionArr
], function (err, results) {
if (err) {
console.log(err);
}
train.availabilities.push(results);
train.save();
res.redirect("/trains/" + train._id);
});
}
});
}
]);
function availability(date, days, seats, callback) {
Availability.create({
date: moment(date).add(days, 'days').format("DD-MM-YYYY"),
available: seats
}, function (err, avail) {
if (err) {
return callback(err);
} else {
return callback(null, avail);
}
});
}
I am trying to combine multiple textfiles,convert them in a single zip file using zip archiver.
exports.downloadFilesInZip = function(req, res, next) {
var respObj = {};
var file_names = [];
var projectId = 111;
var file_ids = 11111;
console.log(projectId);
db.getConnection(function (err, connection) {
if (err) {
debug(err);
next(err);
}
else {
var updateQuery = "select data from file_data where file_id IN (?)";
console.log(updateQuery);
connection.query(updateQuery,[file_ids], function (err, results) {
console.log("inside" + updateQuery);
if (err) {
connection.release();
console.log("error" + JSON.stringify(err));
debug(err);
next(err);
}
else {
async.eachSeries(results,function(item,loopCallBack){
var text = "";
console.log("hllllllll");
console.log(item.data);
console.log(JSON.parse(item.data));
document_text = JSON.parse(item.data);
console.log("dssddssdsdsdsdsd"+document_text);
for(var j=0; j < document_text.length ;j++)
{
text += document_text[j]['text'];
}
//file_names.push(convertStringToTextFile(text));
convertStringToTextFile(text,function(err,file_name){
if(err){
console.log(err);
loopCallBack(err);
}
else {
file_names.push(file_name);
loopCallBack();
}
})
},function(err){
if(err){
console.log(err);
next(err);
}
else {
var updateQuery = "select name from project where id in (?)";
console.log(updateQuery);
connection.query(updateQuery,[projectId], function (err, results) {
console.log("inside" + updateQuery);
connection.release();
if (err) {
console.log("error" + JSON.stringify(err));
debug(err);
next(err);
}
else {
var fileName_link = JSON.stringify(results[0].name);
console.log("projectname"+fileName_link);
convertTextFilesToZip(file_names,fileName_link, function (err, filename) {
if (err) {
console.log(err);
next(err);
}
else {
console.log("filename link" + filename);
res.json({
status: 0,
file_link: filename
});
}
});
}
});
}
});
}
});
}
});
}
}
convertStringToTextFile = function(text,cb){
var json_filename = 'tmp/file_'+uuid.v4().replace('-','')+'.txt';
fs.writeFile(json_filename, text , function (err) {
if (err) {
debug(err);
cb(err);
}
else{
cb(null,json_filename);
}
});
};
convertTextFilesToZip = function(textFiles,file_link,cb){
console.log("textfiles"+textFiles);
var filename = 'reports/'+JSON.parse(file_link)+'_extractedText.zip';
var output = fs.createWriteStream(filename);
output.on('close', function() {
console.log(zipArchive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
zipArchive.on('error', function(err) {
cb(err);
});
zipArchive.pipe(output);
zipArchive.bulk([
{ expand: true, src: textFiles }
]);
zipArchive.finalize();
cb(null,filename);
}
It works okay the first time and after that it throws this error.I have checked other posts in which res is returned twice but i couldn't find it.It says that can't set headers after they are sent.I think the problem is in the convertTextFilesToZip function but i cant seem to pinpoint the exact location which is generating the error.ANy help is appreciated.
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:350:11)
at ServerResponse.header (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:700:10)
at ServerResponse.send (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:154:12)
at fn (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:934:10)
at View.exports.renderFile [as engine] (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/jade/lib/index.js:374:12)
at View.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/view.js:93:8)
at EventEmitter.app.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/application.js:566:10)
at ServerResponse.res.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:938:7)
at /Users/zeeshandar/Desktop/Agreements_info/agreements_info/app.js:207:13
at Layer.handle_error (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/li b/router/layer.js:58:5)
Making my comment into an answer since it appears to have led to the solution.
The variable zipArchive is not initialized in convertTextFilesToZip() therefore you are reusing that variable from one function call to the next and that seems unlikely to be the right implementation.
Also, I would expect your method calls to zipArchive to be asynchronous and it doesn't look like your are coding for that since the callback is called before you have any sort of completion notification.