How to validate data from csv file in nodejs? - node.js

Question

My CSV file contains 3 columns: first name,last name,email. I want to add all the valid lines of the CSV file in the users table in the database. For example: if the CSV file has 10 lines where line 2 is missing first name and line 3 has invalid email, I want to see 8 users added to database. After adding the users from the CSV file, it should display how many users were added and how many failed.
My model:
const User = sequelize.define(
"users",
{
first_name: {
type: Sequelize.STRING(60),
allowNull: false,
},
last_name: {
type: Sequelize.STRING(60),
allowNull: false,
},
email: {
type: Sequelize.STRING(255),
allowNull: false,
},
My Controller:
const upload = async (req, res) => {
try {
if (req.file == undefined) {
return res.status(400).send("Please upload a CSV file!");
}
let users = [];
console.log(" __basedir", __basedir);
let path = __basedir + "/uploads/" + req.file.filename;
fs.createReadStream(path)
.pipe(csv.parse({ headers: true }))
.on("error", (error) => {
throw error.message;
})
.on("data", (row) => {
users.push(row);
})
.on("end", () => {
User.bulkCreate(users, {
validate: true,
})
.then(() => {
res.status(200).send({
message:
`Uploaded ${users.length} data successfully from ` + req.file.originalname,
});
})
.catch((error) => {
res.status(500).send({
message: `Fail to import ${users.length} into database!`,
error: error.message,
});
});
});
} catch (error) {
res.status(500).send({
message: "Could not upload the file: " + req.file.originalname,
});
}
};
How can I solve this?

Answer 1

I solve it using fast-csv package
import * as csv from 'fast-csv';
const upload = async (req, res) => {
try {
if (req.file == undefined) {
return res.status(400).send("Please upload a CSV file!");
}
let users = [];
console.log(" __basedir", __basedir);
let path = __basedir + "/uploads/" + req.file.filename;
console.log("path", path);
let emailPattern= /^[A-Z0-9._%+-]+#[A-Z0-9.-]+\.[A-Z]{2,4}$/i;
let successCount = 0;
let errorCount = 0;
fs.createReadStream(path)
.pipe(csv.parse({ headers: [ undefined,'first_name', 'last_name', 'email', undefined ],renameHeaders: true, ignoreEmpty: true }))
.validate(data => data.first_name !== '' && data.last_name !== '' && emailPattern.test(data.email))
.on("error", (error) => {
throw error.message;
})
.on("data", (row) => {
successCount++;
users.push(row);
console.log(`ROW=${JSON.stringify(row)}`)
})
.on('data-invalid', (row, rowNumber) => {
errorCount++;
console.log(`Invalid [rowNumber=${rowNumber}] [row=${JSON.stringify(row)}]`)
})
.on("end", (rowCount) => {
console.log(`Parsed ${rowCount} rows`)
User.bulkCreate(users, {
validate: true,
})
.then(async() => {
res.status(200).json({
error: false,
success:
`Uploaded ${successCount} row successfully from ` + req.file.originalname,
failed:`Uploaded ${errorCount} row failed from ` + req.file.originalname,
});
})
.catch((error) => {
console.log(error);
res.status(500).json({
error: error.message,
failed: `Fail to import ${users.length} row into database!`,
});
});
});
} catch (error) {
console.log(error);
console.log(error.message);
res.status(500).json({
error: error.message,
failed: "Could not upload the file: " + req.file.originalname,
});
}
};