CSV File downloads from Node Express server - node.js

I have an API backend with Node and Express. I am trying to take some filtered data from the frontend and create a CSV file and download it for the user. I have been using json2csv. I am able to create the data file correctly and when I use that file in my express route I download a file that just says undefined. At first, I thought it was an asynchronous issue, but after using a setTimeout as a test to see if that was an issue I still get the undefined data file. Console logging the "csvData" shows the correct data.
Express route to download the file.
app.post('/api/downloads/filtered', (req, res) => {
let fields = [];
fields = Object.keys(req.body[0])
const filteredData = req.body;
const json2csvParser = new json2csv({fields: fields});
const csvData = json2csvParser.parse(filteredData);
console.log(csvData)
fs.writeFile('./report.csv', csvData, (err) => {
if (err) {
console.log(err);
}
else {
console.log('created report.csv');
res.download('./report.csv');
}
})
})
I'm using Vue on the frontend, I get the file when clicking a button not sure if that is something I should include.

I ended up figuring out my issue. I found that downloading in a post request didn't seem to be possible. I needed a get request. Since the data for the file came in the request body I ended up keeping the post request to create the file and creating a separate get request to download the file this seemed to work fine but didn't find it documented anywhere so I wasn't sure if a better way exists.
app.post('/api/downloads/filtered', (req, res) => {
console.log(req.body)
let fields = [];
fields = Object.keys(req.body[0])
const filteredData = req.body;
const json2csvParser = new json2csv({fields: fields});
const csvData = json2csvParser.parse(filteredData);
console.log(csvData)
fs.writeFile('./report.csv', csvData, (err) => {
if (err) {
console.log(err);
}
else {
console.log('created report.csv');
}
})
})
app.get('/api/downloads/filtered', (req, res) => {
setTimeout(() => {res.download('./report.csv')}, 1000)
})

Related

Uploading image with MongoDB, Express.js and Node.js

The image is being saved in the database and in a folder for the uploads. However, when i try to display the image on another page, it can't seem to find it. This is my controllerimage on database.
exports.create = async (req, res, next) => {
try {
// eslint-disable-next-line prefer-const
let entity = new Product(req.body);
entity.image = `uploads/${req.file.filename}`;
entity.userId = req.user._id;
const saved = await entity.save();
res.status(httpStatus.CREATED);
res.json(saved);
} catch (error) {
next(error);
}
};
I tried to change the route to "localhost:3000/uploads/${req.file.filename}", "localhost:3000/v1/${req.file.filename}" and other variations of it, but it doesn't seem to work anyway. It's just not finding the image.

How to download an .xlsx file using nodejs

I'm trying to download a .xlsx file from a website through web scraping, I've done the entire process until I access the temporary url that the website generates for the download.
When I open the file url in the browser, the download starts automatically (as shown in the image).
excel file download
The problem is that I need to parse this file to send later to my front-end. When I try to create the file using fs.createWriteStream('result.xlsx') and later populate it with res.pipe(fileStream); the file is always generated empty.
Here's my full code:
const https = require("https");
const fs = require("fs");
const path = require("path");
const xlsx = require("node-xlsx");
function download(url, callback) {
const filename = path.basename(url);
const req = https.get(url, function (res) {
const fileStream = fs.createWriteStream("result.xlsx");
res.pipe(fileStream);
const obj = xlsx.parse('result.xlsx');
callback(obj[0]);
fileStream.on("error", function (err) {
console.log("Error writting to the stream.");
console.log(err);
});
fileStream.on("close", function () {
callback(filename);
});
fileStream.on("finish", function () {
fileStream.close();
});
});
req.on("error", function (err) {
console.log("error downloading the file");
console.log(err);
});
}
module.exports.download = download;
My questions are:
Is it possible to parse this data into an array WITHOUT needing to save to a physical file? If yes, how?
If I can't parse the data without needing to populate a physical file, how can I download the spreadsheet and then read and parse the data later.
NOTE: I have already tested the rest of my download function with a valid file entered manually, everything is working perfectly. The only thing that isn't working is the data downloading and reading part of the spreadsheet.
Is it possible to parse this data into an array WITHOUT needing to save to a physical file? Basically No (file from remote server). Except the server allowed you to do it live.
Your code is nearly right, except the order is wrong. You must callback after the writing is done, it will fix your empty file issue.
Here is how:
const https = require("https");
const fs = require("fs");
const path = require("path");
const xlsx = require("node-xlsx");
function download(url, callback) {
const filename = path.basename(url);
const req = https.get(url, function (res) {
const fileStream = fs.createWriteStream("result.xlsx");
res.pipe(fileStream);
fileStream.on("error", function (err) {
console.log("Error writting to the stream.");
console.log(err);
});
fileStream.on("close", function () {
const obj = xlsx.parse('result.xlsx');// or whatever you named it
callback(obj[0]);
});
fileStream.on("finish", function () {
fileStream.close();
});
});
req.on("error", function (err) {
console.log("error downloading the file");
console.log(err);
});
}
module.exports.download = download;

How to send a NODE.JS post request from an Angular Project?

I have a NODE.JS api using expressjs that connects to an SQL Server, and I want to use it in an angular project. I make use two files, a route file and a controllers file. My route file is as follows:
module.exports = (app) => {
const UsrContrllr = require('../Controllers/users.controllers');
//1. GET ALL USERS
app.get('/api/users', UsrContrllr.func1);
//2. POST NEW USER
app.post('/api/user/new', UsrContrllr.func2);
};
And my controllers file is given below:
const mssql = require('mssql');
exports.func1 = (req, res) =>
{
// Validate request
console.log(`Fetching RESPONSE`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `SELECT * FROM USERS`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() === '') {
res.send('Oops!!! Required data not found...');
}
else {
// send records as a response
res.send(recordset);
}
};
});
};
exports.func2 = (req, res) =>
{
// Validate request
console.log(`INSERTING RECORD ${req}`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `INSERT INTO GDUSERS (USERCODE, PASSWORD, LANGUAGE, USERCLASS, FIRSTNAME, LASTNAME, CONTACTNO) VALUES ('${req.body.usercode}', '${req.body.password}', 'EN', '0', '${req.body.firstname}', '${req.body.lastname}', '${req.body.contactno}');`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() == '') {
res.send('Oops!!! Required data not found...');
}
else {
// Send records as response
res.send(recordset);
}
};
});
};
The GET request works well, but when I try to run the POST request directly from the angular application, I get an error stating
Cannot GET URL/api/user/new
The angular code in my angular project is:
signup() {
let headers = new Headers({ 'Content-Type': 'application/json' });
let options = new RequestOptions({ headers: headers });
console.log(this.user); //User details come from a form
this.http.post(“URL", this.user, options)
.subscribe(
(err) => {
if(err) console.log(err);
console.log("Success");
});
}
I’m not sure whether the angular code I’m using, is right or not, and I don’t know where I’m going wrong. How does one exactly send a http POST request from an Angular project?
this i the way i handled my user signup with http.post calls. my approach is slightly different when signing up user because i am using a promise instead of observable (which i normally use for my servicecalls). but i will show you both ways.
createUser(user: User): Promise < string > {
const promise = new Promise < string > ((resolve, reject) => {
const userForPost = this.createUserForPost(user);
this.http.post(environment.backendUrl + '/api/user/signup', userForPost, this.config).toPromise < HttpConfig > ()
.then(createdUser => {
}).catch(error => {
console.log(error);
});
});
return promise;
}
here another example with an observable
createForumPost(forumPost: ForumPost) {
this.http.post < { message: string, forumPostId: string } > (environment.backendUrl + '/api/forumPosts', forumPost).subscribe((responseData) => {
const id = responseData.forumPostId;
forumPost.id = id;
});
}
i defined my URL somewhere else and then just use the environment.backedUrl + 'path' to define my path (the same as the path in your backend controller)
this is one of my first answers here on SO. i am sry if it is a bit messy
i hope i was able to help with my examples :)

How to efficiently forward request to multiple endpoints using nodejs?

I built a nodejs server to act as an adapter server, which upon receiving a post request containing some data, extracts the data from the request body and then forwards it to a few other external servers. Finally, my server will send a response consisting of the responses from each of the external server (success/fail).
If there's only 1 endpoint to forward to, it seems fairly straightforward. However, when I have to forward to more than one servers, I have to rely on things like Promise.All(), which has a fail-fast behaviour. That means if one promise is rejected (an external server is down), all other promises will also be rejected immediately and the rest the servers will not receive my data.
May be this ain't be the exact solution. But, what I am posting could be the work around of your problem.
Few days back I had the same problem, as I wanted to implement API versioning. Here is the solution I implemented, please have a look.
Architecture Diagram
Let me explain this diagram
Here in the diagram is the initial configuration for the server as we do. all the api request come here will pass to the "index.js" file inside the release directory.
index.js (in release directory)
const express = require('express');
const fid = require('./core/file.helper');
const router = express.Router();
fid.getFiles(__dirname,'./release').then(releases => {
releases.forEach(release => {
// release = release.replace(/.js/g,'');
router.use(`/${release}`,require(`./release/${release}/index`))
})
})
module.exports = router
code snippet for helper.js
//requiring path and fs modules
const path = require('path');
const fs = require('fs');
module.exports = {
getFiles: (presentDirectory, directoryName) => {
return new Promise((resolve, reject) => {
//joining path of directory
const directoryPath = path.join(presentDirectory, directoryName);
//passsing directoryPath and callback function
fs.readdir(directoryPath, function (err, files) {
// console.log(files);
//handling error
if (err) {
console.log('Unable to scan directory: ' + err);
reject(err)
}
//listing all files using forEach
// files.forEach(function (file) {
// // Do whatever you want to do with the file
// console.log(file);
// });
resolve(files)
});
})
}
}
Now, from this index file all the index.js inside each version folder is mapped
Here is the code bellow for "index.js" inside v1 or v2 ...
const express = require('express');
const mongoose = require('mongoose');
const fid = require('../../core/file.helper');
const dbconf = require('./config/datastore');
const router = express.Router();
// const connection_string = `mongodb+srv://${dbconf.atlas.username}:${dbconf.atlas.password}#${dbconf.atlas.host}/${dbconf.atlas.database}`;
const connection_string = `mongodb://${dbconf.default.username}:${dbconf.default.password}#${dbconf.default.host}:${dbconf.default.port}/${dbconf.default.database}`;
mongoose.connect(connection_string,{
useCreateIndex: true,
useNewUrlParser:true
}).then(status => {
console.log(`Database connected to mongodb://${dbconf.atlas.username}#${dbconf.atlas.host}/${dbconf.atlas.database}`);
fid.getFiles(__dirname,'./endpoints').then(files => {
files.forEach(file => {
file = file.replace(/.js/g,'');
router.use(`/${file}`,require(`./endpoints/${file}`))
});
})
}).catch(err => {
console.log(`Error connecting database ${err}`);
})
module.exports = router
In each of this index.js inside version folder is actually mapped to each endpoints inside endpoints folder.
code for one of the endpoints is given bellow
const express = require('express');
const router = express.Router();
const userCtrl = require('../controllers/users');
router.post('/signup', userCtrl.signup);
router.post('/login', userCtrl.login);
module.exports = router;
Here in this file actually we are connecting the endpoints to its controllers.
var config = {'targets':
[
'https://abc.api.xxx',
'https://xyz.abc',
'https://stackoverflow.net'
]};
relay(req, resp, config);
function relay(req, resp, config) {
doRelay(req, resp, config['targets'], relayOne);
}
function doRelay(req, resp, servers, relayOne) {
var finalresponses = [];
if (servers.length > 0) {
var loop = function(servers, index, relayOne, done) {
relayOne(req, servers[index], function(response) {
finalresponses.push[response];
if (++index < servers.length) {
setTimeout(function(){
loop(servers, index, relayOne, done);
}, 0);
} else {
done(resp, finalresponses);
}
});
};
loop(servers, 0, relayOne, done);
} else {
done(resp, finalresponses);
}
}
function relayOne(req, targetserver, relaydone) {
//call the targetserver and return the response data
/*return relaydone(response data);*/
}
function done(resp, finalresponses){
console.log('ended');
resp.writeHead(200, 'OK', {
'Content-Type' : 'text/plain'
});
resp.end(finalresponses);
return;
}
It sounds like you are trying to design a reverse proxy. If you are struggling to get custom code to work, there is a free npm library which is very robust.
I would recommend node-http-proxy
I have posted link below, which will lead you directly to the "modify response", since you mentioned modification of the API format in your question. Be sure to read the entire page though.
https://github.com/http-party/node-http-proxy#modify-a-response-from-a-proxied-server
Note: this library is also very good because it can support SSL, and proxies to both localhost (servers on the same machine) and servers on other machines (remote).
Promise.all() from MDN
It rejects with the reason of the first promise that rejects.
To overcome the problem, you'll need to catch() each request you've made.
e.g.
Promise.all([
request('<url 1>').catch(err => /* .. error handling */),
request('<url 2>').catch(err => /* .. error handling */),
request('<url 3>').catch(err => /* .. error handling */)
])
.then(([result1, result2, result3]) => {
if(result1.err) { }
if(result2.err) { }
if(result3.err) { }
})

How can I parse data from a csv file uploaded by a user in node/express?

I have a front end react app and a backend node/express app. I want to allow a user to upload a csv file, then parse the file and instantiate a model for each row. However, I am somewhat confused about how to do this, since I am used to simply posting to a route in the API, and persisting the thing from the request body. In this case, the thing from the request body is the file, and I don't want to save the file, just the data inside it. How can I parse the file without saving it to the database? I have tried to use multer to process the upload and csv-parse to parse the contents, but I am not sure this makes sense. Nonetheless, here is the code (app/index):
...
const multer = require('multer');
const upload = multer().single();
const parse = require('csv-parse');
...
router.post('/distributor/:id/files', (req,res) => {
upload(req, res, function (err) {
if (err) {
console.error("An error occurred when uploading. Please try again. Note
that you may only upload one file at a time, and we only support .csv
files.")
return
}
console.log("We have received your file")
})
});
...
// router.get('/distributor/:id/files/:id', (req, res) => {
// File
// .forge({id: req.params.id})
// .fetch()
// .then((file) => {
// if (_.isEmpty(file))
// return res.sendStatus(404);
// return parseJson(file)
// })
// .then((jsonData) => {
// for (var i in jsonData) {
// //save instance of model
// }
// })
// .catch((error) => {
// console.error(error);
// return res.sendStatus(500);
// });
// })
// function parseJson(file) {
// var output = [];
// // Create the parser
// var parser = parse({delimiter: ':'});
// // Use the writable stream api
// parser.on('readable', function(){
// while(record = parser.read()){
// output.push(record);
// }
// });
// // Catch any error
// parser.on('error', function(err){
// console.log(err.message);
// });
// parser.end();
// }
I know this doesn't make sense, since I don't actually want to save the file as a model and table in the database, I just want to save each item inside the file, so I know I cannot make a route called '/distributor/:id/files/:id'. But I am lost as to what to do instead. I hope that what I am trying to do is clear! I am fairly new to node, and programming in general, and I have never come across a situation in which I needed to handle file upload.
You can use this node module to parse the csv file. https://www.npmjs.com/package/csvtojson
For example you have file name users in the request object.
const csv=require('csvtojson');
csv()
.fromString(req.files.users.data.toString('utf8'))
.on('json', (user) => {
console.log(user);
})
.on('done', () => {
console.log('done parsing');
});
You will be able to get every row as a json object.

Resources