Send KnexJS error message to the frontend - node.js

I am having trouble sending an error to the front end when a csv file is uploaded and the numbers already exist in the database. The backend is logging an error that the primary key value already exist, but the code I have written tells the front end that the file uploaded just fine.
Code snippet:
router.post('/:program/upload', upload.single('testUpload'), (req, res, next) => {
try {
CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json({
message: 'File Uploaded Just fine :)'
});
} catch (error) {
return res.status(500).json({
message: error
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', () => {
csvData.forEach(item => {
queries.upload(program, item)
.then(() => {
console.log('QR Code Added: ' + item);
}).catch((err) => {
console.log(`oopsie: ${err}`);
});
})
});
stream.pipe(csvStream);
}
Pretty confident the issue is with my poor understanding of promises.

As expected, I wasn't handling my promises correctly. I've updated the code a bit and now it responds with 2 arrays of successful uploads and errored uploads.
router.post('/:program/upload', upload.single('testUpload'), async (req, res, next) => {
try {
const result = await CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json(result);
}
catch (error) {
return res.status(500).json({
message: error,
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
return new Promise((resolve) => {
const results = {
seccess: [],
error: [],
}
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', async () => {
await Promise.all(
csvData.map(async (item) => {
try{
await queries.upload(program, item);
results.success.push(item);
console.log('QR Code Added: ' + item);
}
catch (error) {
console.log(`oopsie: ${error}`)
results.error.push(item);
}
})
)
resolve(results);
});
stream.pipe(csvStream);
})
}

Related

How do I handle errors properly with fast CSV?

In the code below, I want to check if the headers are valid and then stop reading the stream if they are and send an error back to the client (via my ErrorHandler function). I see the following in my console:
UnhandledPromiseRejectionWarning: Error: Please check your column headers.
Why is that?
uploadExercisesViaFile = async (
req: Request,
res: Response,
next: NextFunction
) => {
try {
let checker = (arr, target) => target.every((v) => arr.includes(v));
const columns = ["name", "link", "description"];
const results = [];
let errors: string[] = [];
const parseCSV = parseString(req.file.buffer.toString(), {
headers: true,
maxRows: 5,
})
.on("headers", (header) => {
if (!checker(header, columns)) {
errors.push("Please check your column headers.");
parseCSV.end();
}
})
.on("data", async (row) => {
console.log(row);
results.push(row);
})
.on("error", (error) => {
errors.push(error.message);
})
.on("end", async (rowCount) => {
// Check if errors
if (errors.length > 0) {
console.log(errors);
throw new ErrorHandler(400, errors[0]);
}
// Upload
const uploadedExercises = await Exercises.batchUploadExercise(
results.map((r) => {
return { ...r, access: req.body.access, coach: req.user.id };
})
);
res.status(200).json({ rowCount: rowCount });
return;
});
} catch (e) {
next(e);
}
};

Writing Mocha Chai Test cases for NodeJs Controllers

I am new to unit testing. I am trying to write test cases for controller.js files for nodejs microservices files. I am unable to understand where I am going wrong. Always throws an error "TypeError: Cannot read property 'empId' of undefined" for 2 of these properties.
This is the controller code:
const crmgDetails = db.crmgResource_details;
const employeeProposal = db.employee_Proposal;
const Op = db.Sequelize.Op;
const raDetails = db.crmgRaSheet_entity;
let results = [];
Sequelize = require('sequelize')
exports.findOne = (req, res) => {
console.log(req.body.empId);
crmgDetails.findAll({where: {
resEmployeeNumber: req.body.empId
}
})
.then(data => {
res.send(data);
})
.catch(err => {
res.status(500).send({
message:
err.message || "Some error occurred while retrieving tutorials."
});
});
};
exports.findMatchingDemandsForRmg = (req,res) => {
let proposedDemands = [];
employeeProposal.findAll({
where: {
emp_id: req.body.empId,
demandSbu : req.body.sbu
}
}).then(proposedEmployee => {
console.log('proposedEmployee',proposedEmployee);
if(proposedEmployee.length === 0){
crmgDetails.findAll({
where: {
resEmployeeNumber: req.body.empId,
demandSbu: req.body.sbu
}
}).then(matchingDemands => {
console.log('matchingDemands ',matchingDemands)
proposedDemands = matchingDemands;
})
}
else{
console.log("crmg Employee")
console.log(proposedEmployee)
for(let employee of proposedEmployee){
crmgDetails.findOne({
where: {
demandUid: employee.demandUid,
resEmployeeNumber: req.body.empId,
demandSbu: req.body.sbu
}
}).then( crmgProposed=> {
proposedDemands.push(crmgProposed);
})
}
}
setTimeout(() => {
console.log(proposedDemands)
res.send(proposedDemands);
}, 3000);
}).catch((err)=>{
res.status(500).send({
message:
err.message || "Some error occurred while retrieving tutorials."
});
})
}
exports.getResourceAllocationDetails = (req,res) => {
employeeProposal.findAll({
include: {
model: raDetails
},
where: Sequelize.and(
{activeFlag : true},
Sequelize.or({status:"Accepted By RMG"},
{status:"Rejected"}
))
}).then(employees => {
res.send(employees)
})
}
This is the test file I tried to write without my head:
const CrmgRaSheetModel = require('../controllers/crmgResource_Details.controller')
describe('Check for succcessful fetech API call', () => {
it('property getResourceAllocationDetails should be called', async () => {
CrmgRaSheetModel.getResourceAllocationDetails((res) => {
expect(res).to.be.an('object')
return res.json()
})
});
it('property findMatchingDemandsForRmg should be called', async () => {
CrmgRaSheetModel.findMatchingDemandsForRmg((res) => {
expect(res).to.be.an('object')
return res.json()
})
});
it('property findOne should be called', async () => {
CrmgRaSheetModel.findOne((res) => {
expect(res).to.be.an('object')
return res.json()
})
})
})
from test file you are calling controller method with only res, so no chance to send your input as your body.
So pass req,res both and pass your input value in req

storing file with Large Object Postgres with Nodejs

I have a simple HTML form with a file input. I need to save the file with Postgres Large Object but I'm having issues with the fileStream.pipe. It should be in a Promise itself like so:
return new Promise((resolve, reject) => {
stream.on('finish', resolve(oid));
stream.on('error', reject);
});
But it wouldn't work since it would result into a Promise in a Promise. The complete functions are:
function storeLargeObject(path) {
return new Promise((resolve, reject) => {
pgp.db.tx(tx => {
const man = new LargeObjectManager({pgPromise: tx});
return man.createAndWritableStreamAsync(bufferSize);
}).then(([oid, stream]) => {
const fileStream = createReadStream(path);
fileStream.pipe(stream);
stream.on('finish', resolve(oid));
stream.on('error', reject);
}).catch(err => {
console.log(err);
return err;
});
});
}
const addBinary = (req, res) => {
pgp.db.oneOrNone({
text: insertQuery,
values: values
}).then(f => {
let explFile = req.files.bin;
let uploadPath = __dirname + '/' + f.id;
if (!req.files || Object.keys(req.files).length === 0) {
res.status(400).send('No File was uploaded.');
return;
}
explFile.mv(uploadPath, async function (err) {
if (err)
return res.status(500).send(err);
let newOid = await storeLargeObject(uploadPath);
fs.unlink(uploadPath, err => console.log(err));
coupleIds(exp.id, newOid);
return res.status(200).send({
status: "success",
message: "File successfully uploaded!"
})
});
}).catch(err => {
return res.status(401).send({
status: 'error',
errorCode: 102,
message: 'Error! ' + err
});
});
}

Appending array values to csv

I would like to take some output and append the output to a csv
This is the code I have so far:
async function writeData() {
const csv = require('csv-parser')
const results = [];
fs.createReadStream('C:\\Users\\User\\Documents\\testingclean.csv')
.pipe(csv())
.on('data',(data)=> results.push(data))
.on('end', async () => {
const cookieJar = await getCookieJar();
const promises = [];
results.forEach((data) => {
promises.push(boxrec.getPersonById(cookieJar,data.id));
})
try {
const fighters = await Promise.all(promises);
fighters.forEach((fighter)=>{
boxer = JSON.parse(JSON.stringify(fighter.output));
fs.appendFile('C:\\Users\\User\\Documents\\newtest.csv',boxer, (err) => {
if (err) console.error('Could not append data to csv');
console.log('Data successfully appended');
})
});
} catch (e) {
console.log(e);
}
})
};
try {
writeData();
} catch (error) {
console.log("Error in writeData: " + error);
}
However running this code does not produce the desired csv output.
I am specifically writing to csv because I have read that I cannot append to a json (would ideally want to write data to a json)
If you don't have anything writen to the CSV file, you should close it explicitly at the end of your program :
var readsteam = fs.createReadStream();
...
readStream.destroy();
Not sure but this should be the full code :
async function writeData() {
const csv = require('csv-parser')
const results = [];
var readsteam = fs.createReadStream('C:\\Users\\User\\Documents\\testingclean.csv');
readsteam
.pipe(csv())
.on('data',(data)=> results.push(data))
.on('end', async () => {
const cookieJar = await getCookieJar();
const promises = [];
results.forEach((data) => {
promises.push(boxrec.getPersonById(cookieJar,data.id));
})
try {
const fighters = await Promise.all(promises);
fighters.forEach((fighter)=>{
boxer = JSON.parse(JSON.stringify(fighter.output));
fs.appendFile('C:\\Users\\User\\Documents\\newtest.csv',boxer, (err) => {
if (err) console.error('Could not append data to csv');
console.log('Data successfully appended');
})
});
} catch (e) {
console.log(e);
}
}
readsteam.destroy();
};
try {
writeData();
} catch (error) {
console.log("Error in writeData: " + error);
}

What's wrong with this use of async await?

I am trying to download tracks via the soundcloud API, and then launch a callback once an indeterminant amount of tracks is downloaded. When I run the below code, I see "All done" being console logged before anything else, even though I intend for it to be the last thing... What am I doing wrong?
// Deps
import fs from 'fs'
import SC from 'node-soundcloud'
import request from 'request'
// Write mp3 function
function writeMP3(track) {
return new Promise((resolve, reject) => {
console.log('Starting download: ', track.title)
request.get(track.download_url)
.on('error', err => {
// reject('Download error: ', err)
})
.on('finish', () => {
() => resolve('Download complete')
})
.pipe(fs.createWriteStream(`./data/temp/${track.title}_${track.user.username}.mp3`))
})
}
async function asyncTrackFetch(track) {
return await writeMP3(track)
}
// Array of promises to callback upon
const trackActions = []
SC.init({
id: 'MY_ID',
secret: 'MY_SECRET'
})
SC.get('/tracks', (err, tracks) => {
if (err) {
throw new Error(err)
} else {
console.log('Tracks fetched: ', tracks.length)
tracks.map(track => {
if (track.downloadable) {
console.log('downloadable')
trackActions.push(asyncTrackFetch(track))
}
})
}
})
// Perform requests async
Promise.all(trackActions).then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
Promise.all(trackActions) waits on whatever promises are in trackActions, but trackActions is empty at the time you make the call. You're only adding promises to the array after your SC.get callback gets called.
Try putting your Promise.all... block inside the SC.get callback like this:
SC.get('/tracks', (err, tracks) => {
if (err) {
throw new Error(err)
} else {
console.log('Tracks fetched: ', tracks.length)
tracks.map(track => {
if (track.downloadable) {
console.log('downloadable')
trackActions.push(asyncTrackFetch(track))
}
})
Promise.all(trackActions).then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
}
})
It's worth mentioning as well that your line throw new Error(err) will crash the program since there's nowhere for that error to be caught.
As Antonio Val mentioned, there are better ways to do this. If you promisify the node-soundcloud library then the last part of your code could look like this:
SC.get('/tracks').then(tracks => {
// No need for trackedActions array.
return Promise.all(tracks.filter(track => track.downloadable)
.map(track => asyncTrackFetch(track)))
}).then(fetchedTracks => {
console.log('All done fetching tracks', fetchedTracks)
}).catch(err => {
// Handle error.
})
Or inside an async function,
try {
const tracks = await SC.get('/tracks')
const fetchPromises = tracks
.filter(track => track.downloadable)
.map(track => asyncTrackFetch(track))
const fetchedTracks = await Promise.all(fetchPromises)
console('All done fetching tracks.', fetchedTracks)
} catch (err) {
// Handle error
}
I think the easiest way would be to move Promise.all after tracks.map loop finished.
A more elegant solution would be to promisify SC.get as well and use async await along all your code.
UPDATE:
Couldn't test it so not sure if it works, but it would be something like this:
import fs from 'fs'
import SC from 'node-soundcloud'
import request from 'request'
function writeMP3(track) {
return new Promise((resolve, reject) => {
console.log('Starting download: ', track.title)
request.get(track.download_url)
.on('error', err => {
// reject('Download error: ', err)
})
.on('finish', () => {
() => resolve('Download complete')
})
.pipe(fs.createWriteStream(`./data/temp/${track.title}_${track.user.username}.mp3`))
})
}
function getTracks() {
return new Promise((resolve, reject) => {
SC.get('/tracks', (err, tracks) => {
if (err) {
return reject(err)
}
console.log('Tracks fetched: ', tracks.length)
resolve(tracks)
})
})
}
SC.init({
id: 'MY_ID',
secret: 'MY_SECRET'
})
With async await:
async function start() {
const tracks = await getTracks();
for (let track of tracks) {
await writeMP3(track)
}
}
start()
.then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
.catch((err) => {
// insert error handler here
})
If you just want to use Promises:
getTracks
.then((tracks) => {
const promiseArray = tracks.map((track) => {
return writeMP3(track)
})
return Promise.all(promiseArray)
})
.then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
.catch((err) => {
// insert error handler here
})

Resources