converting excel(.xlsx) file to JSON - node.js

I have excel sheet called sampledata.xlsx which i converted into json and console.log to print this data.
server.js
var xlsx2json = require('xlsx2json')
xlsx2json(
'sampledata.xlsx',
{
dataStartingRow: 2,
mapping: {
'name': 'B',//name
'sku': 'C',//unit price //sku
'quantity': 'D',//quantity
}
}).then(jsonArray => {
// [
// {"col_1": "Barton LCC", "col_2": "30", "col_3": "86.69"}
// ]
//console.log(jsonArray);
});
with the help of this doc.
What i want to do here is,in my sampledata.xlsx file i have more data like flat,address,price,etc here i already don't know which fields are present in my excel sheet but i want all that to be console.log.How could i do this is there any way to do this.

import xlsx2json from 'xlsx2json';
OR
const xlsx2json = require('xlsx2json');
const excel2json = [
(req, res, next) => {
xlsx2json(req.body.file.path)
.then(result => result[0])
.reduce((object, item, index) => {
if (index === 0) {
object.mapper = item; // eslint-disable-line no-param-reassign
return object;
}
const data = {};
Object.keys(item).forEach((key) => {
data[object.mapper[key]] = item[key];
});
object.data.push(data);
return object;
}, { mapper: {}, data: [] })
.then(excel => console.log(excel)) // this gives json as output
.catch(err => next(err));
},
];

npm install xlsx-to-json-lc --save
npm install xls-to-json-lc --save
var exceltojson = require("xls-to-json-lc");
exceltojson({
input: "pass the input excel file here (.xls format)"
output: "if you want output to be stored in a file"
sheet: "sheetname", // specific sheetname inside excel file (if you have multiple sheets)
lowerCaseHeaders:true //to convert all excel headers to lowr case in json
}, function(err, result) {
if(err) {
console.error(err);
} else {
console.log(result);
//result will contain the overted json data
}
});

Related

Cypress is returning an empty array when trying to log sheetnames of an excel file

I am currently trying to get the sheetnames of an excel file but Cypress is returning an empty array. Is there something I missed? I'll be using it to verify data on later steps.
I'm using Cypress 9.6.0 with Cucumber. Below are my scripts and screenshots:
index.js for task
module.exports = (on, config) => {
on('file:preprocessor', cucumber());
on('task', {
checkExcelSheetContents(args){
if (fs.existsSync(args.filePath)) {
const workbook = xlsx.readFile(args.filePath);
return xlsx.utils.sheet_to_json(workbook.SheetNames)
} else {
throw new Error ("File not found")
}
}
})
return Object.assign({}, config, {
fixturesFolder: 'cypress/fixtures',
integrationFolder: 'cypress/integration',
screenshotsFolder: 'cypress/screenshots',
videosFolder: 'cypress/videos',
supportFile: 'cypress/support/index.js'
});
}
.js file
And ('try', () => {
var excelFilePath = "../CreateAutomatedTests/cypress/downloads/courses20220714_09_51_27.xlsx"
cy.wrap(excelFilePath).as('filePath')
cy.get('#filePath').then((filePath) => {
cy.task('checkExcelSheetContents', { filePath }).then((contents) => {
cy.log(contents)
})
})
})
Please see these screenshots as well
I've always used the buffer version of xlsx.read().
From xlsx package
For Node ESM, the readFile helper is not enabled. Instead, fs.readFileSync should be used to read the file data as a Buffer for use with XLSX.read:
import { readFileSync } from "fs";
import { read } from "xlsx/xlsx.mjs";
const buf = readFileSync("test.xlsx");
/* buf is a Buffer */
const workbook = read(buf);
Your task:
on('task', {
checkExcelSheetContents(args){
if (fs.existsSync(args.filePath)) {
const buf = fs.readFileSync(file);
const workbook = xlsx.read(buf, { type: 'buffer' });
return workbook.SheetNames
} else {
throw new Error ("File not found")
}
}
})

How to get a specific data from object of array in angular

I wanna know that how to get the data
HERE IS MY node.js
app.post('/pathname', function (req, res){
fs.readfile('filepath', 'utf8', function(err, data){
if(data){
let valueofdata = JSON.parse(data);
let anothervalue = JSON.stringify(valueofdata[0]);
res.send(anothervalue);
}
My JSON file is
[{
"number":[{
"data":"one",
"data":"two",
"data":"three"
}],
"number1":[{
"data":"four",
"data":"five",
"data":"six"
}],
}]
My ANGULAR file
numberval:any;
ngOnInit(): void {
this.numberval = this.service.numbervalueall; --> the value (number) is stored in service
console.log(this.numberval)
}
numberall(data:any){
this.http.post('http://localhost:4001/pathname', data, {responseType : "text"} )
.subscribe(res => {
console.log(res)
this.numbersname = JSON.parse(res) --> Data from node.js is stored in here
console.log(this.numbersname )
})
}
numbersname!:any;
numberdata(){
this.numberall(this.service.numbervalueall)
}
sampledata(){
console.log(this.service.citydata)
setTimeout(() => {
this.numberall(this.service.citydata)
console.log(this.hotelvalue)
},100);
}
How can I get the specific value from object data stored in res I used res.this.service.numbervalueall but can't get the value. Please Help me with this.
In your ANGULAR file
create new variable as datavalue or your choice
datavalue:any;
numberall(data:any){
this.http.post('http://localhost:4001/pathname', data, {responseType : "text"} )
.subscribe(res => {
console.log(res)
this.numbersname = JSON.parse(res)
this.datavalue = this.numbersname[numbervalueall] --> here u get the specific data in datavalue
})
}

node.js "Usage of deprecated field: 'series'" while using c3 chart maker

I am using c3 chart maker following the instructions of this github project: https://github.com/ashleydavis/nodejs-chart-rendering-example.
I would like to create a chart from a .csv data, however when i run the code below the chart is empty and I have this message: "Usage of deprecated field: 'series' ".
const c3ChartMaker = require('c3-chart-maker')
const inputFilePath = "./test_file.csv"
const chartDefinitionFile = "./chart.json"
const outputFilePath = "./chart.png"
c3ChartMaker(inputFilePath, chartDefinitionFile, outputFilePath)
.then(() => {
console.log('done')
})
.catch(err => {
console.error(err)
})
i test your code with this example: https://github.com/ashleydavis/c3-chart-maker
i use example-chart.json and example-data.csv in example folder and this is my code:
const c3ChartMaker = require('c3-chart-maker')
const inputFilePath = "./example-data.csv"
const chartDefinitionFile = "./example-chart.json"
const outputFilePath = "./example-js.png"
c3ChartMaker(inputFilePath, chartDefinitionFile, outputFilePath)
.then(() => {
console.log('done')
})
.catch(err => {
console.error(err)
})
when i run this code show me this image:
and on my console i have this message:
Usage of deprecated field: 'series'.
done
i read index.js code in node_modules\c3-chart-maker directory and find something:
if (chart.series) { // THIS SECTION IS DEPRECATED.
console.error("Usage of deprecated field: 'series'.");
if (!chart.data.columns) {
chart.data.columns = [];
}
var series = Object.keys(chart.series);
var dataFrame = new dataForge.DataFrame(data);
series.forEach(seriesName => {
var dataSeries = chart.series[seriesName];
if (Sugar.Object.isString(inputData) && seriesName !== "x") {
dataFrame = dataFrame.parseFloats(dataSeries).bake();
}
chart.data.columns.push(
[seriesName].concat(
dataFrame.getSeries(dataSeries)
.select(v => v === undefined ? null : v)
.toArray()
)
)
});
}
you see this error log: Usage of deprecated field: 'series'. because in example-chart.json file we have this line of json:
"series": {
"x": "Date",
"Close": "Close",
"Volume": "Volume"
},
and series is deprecated.

Unable to write item(s) to DynamoDB table utilizing DocumentClient - Nodejs

I'm absolutely brand new to DynamoDb and I'm trying to simply write an object from a NodeJS Lambda. Based on what I've read and researched I should probably be using DocumentClient from the aws-sdk. I also found the following question here regarding issues with DocumentClient, but it doesn't seem to address my specific issue....which I can't really find/pinpoint unfortunately. I've set up a debugger to help with SAM local development, but it appears to be only providing some of the errors.
The code's implementation is shown here.
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": {"S": randstring.generate(9)},
"School":{"S": team_name},
"Seed": {"S": seed},
"ESPN_Id": {"S": espn_id}
}
}
console.log(JSON.stringify(params))
dynamodb.put(params, (error,data) => {
if (error) {
console.log("Error ", error)
} else {
console.log("Success! ", data)
}
})
Basically I'm scrubbing a website utilizing cheerio library and cherry picking values from the DOM and saving them into the json object shown below.
{
"TableName": "March-Madness-Teams",
"Item": {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
When I attempt to push this json object to Dynamo, I get errors says
Error MultipleValidationErrors: There were 2 validation errors:
* MissingRequiredParameter: Missing required key 'TableName' in params
* MissingRequiredParameter: Missing required key 'Item' in params
The above error is all good in well....I assume it didn't like the fact that I had wrapped those to keys in strings, so I removed the quotes and sent the following
{
TableName: "March-Madness-Teams",
Item: {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
However, when I do that...I kind of get nothing.
Here is a larger code snippet.
return new Promise((resolve,reject) => {
axios.get('http://www.espn.com/mens-college-basketball/bracketology')
.then(html => {
const dynamodb = new aws.DynamoDB.DocumentClient()
let $ = cheerio.load(html.data)
$('.region').each(async function(index, element){
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(async function(index2, element2){
var seed = $(element2).siblings('span.rank').text()
if (seed.length > 2){
seed = $(element2).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element2).attr('href').split('/').slice(-2)[0]
var team_name = $(element2).text()
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School":team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
}
console.log(JSON.stringify(params))
// dynamodb.put(params)
// .then(function(data) {
// console.log(`Success`, data)
// })
})
})
})
})
Can you try without the type?
Instead of
"School":{"S": team_name},
for example, use
"School": team_name,
From your code, I can see the mis promise on the dynamodb request. Try to change your lines :
dynamodb.put(params).then(function(data) {
console.log(`Success`, data)
})
to be :
dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
you can combine with await too :
await dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
exports.lambdaHandler = async (event, context) => {
const html = await axios.get('http://www.espn.com/mens-college-basketball/bracketology')
let $ = cheerio.load(html.data)
const schools = buildCompleteSchoolObject(html, $)
try {
await writeSchoolsToDynamo(schools)
return { statusCode: 200 }
} catch (error) {
return { statusCode: 400, message: error.message }
}
}
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(async school => {
await dynamodb.put(school).promise()
})
await Promise.all(promises)
}
const buildCompleteSchoolObject = (html, $) => {
const schools = []
$('.region').each(loopThroughSubRegions(schools, $))
return schools
}
const loopThroughSubRegions = (schools, $) => {
return (index, element) => {
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(populateSchoolObjects(schools, $))
}
}
const populateSchoolObjects = (schools, $) => {
return (index, element) => {
var seed = $(element).siblings('span.rank').text()
if (seed.length > 2) {
seed = $(element).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element).attr('href').split('/').slice(-2)[0]
var team_name = $(element).text()
schools.push({
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School": team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
})
}
}
I know this is drastically different from what I started with but I did some more digging and kind of kind of worked to this...I'm not sure if this is the best way, but I seemed to get it to work...Let me know if something should change!
Oh I understand what you want.
Maybe you can see the code above works, but there is one concept you have to improve here about async - await and promise especially on lambda function.
I have some notes here from your code above, maybe can be your consideration to improve your lambda :
Using await for every promise in lambda is not the best approach because we know the lambda time limitation. But sometimes we can do that for other case.
Maybe you can change the dynamodb.put method to be dynamodb.batchWriteItem :
The BatchWriteItem operation puts or deletes multiple items in one or more tables.
Or If you have to use dynamodb.put instead, try to get improve the code to be like so :
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(school => {
dynamodb.put(school).promise()
})
return Promise.all(promises)
}

Export mongodb collection data to csv file in node js

I have created a mongodb collection in mongolab and want to print all documents in that collecton. I have mongolab and the url of mongolab to get connected with the collection is -
mongodb://user:1234#xyz.mongolab.com:41248/new
The sample document structure is -
{
"_id": "9759572745-Sing",
"details": {
"Gender": "M",
"PreTrainingStatus": "Fresher",
"Religion": "Hindu",
"attendanceInPercentage": "",
"batchHolders": {
"AssessmentDate": "Thu Jul 16 2015",
"CourseFee": "7500",
"isEditable": false
},
"batchID": "282726",
"eid": "",
"whereDidYouHearAboutStar": "---Select---",
"skillInstructorOrTrainerName": "282726",
"specificGovtInstitutetieups": "---Select---",
"isSelected": false,
"isEditable": false
},
"addedOnMs": 1439455766000,
"submittedOnMs": 1439454813000,
"latitude": "27.409566879272",
"longitude": "77.69295501709",
"locationName": "Uttar Pradesh 281006,null"
}
I want to Print it and all nested properties to be displayed in a column.
But I am not able to do so, Please help.
Thank you (In Advance),
Dia
Implementing json2csv library for exporting data to csv file on nodejs
Example
const json2csv = require('json2csv').parse;
//For unique file name
const dateTime = new Date().toISOString().slice(-24).replace(/\D/g,
'').slice(0, 14);
const filePath = path.join(__dirname, "../../../", "public", "exports", "csv-" + dateTime + ".csv");
let csv;
const student = await req.db.collection('Student').find({}).toArray();
// Logging student
// [{id:1,name:"John",country:"USA"},{id:1,name:"Ronny",country:"Germany"}]
const fields = ['id','name','country'];
try {
csv = json2csv(booking_info, {fields});
} catch (err) {
return res.status(500).json({err});
}
fs.writeFile(filePath, csv, function (err) {
if (err) {
return res.json(err).status(500);
}
else {
setTimeout(function () {
fs.unlink(filePath, function (err) { // delete this file after 30 seconds
if (err) {
console.error(err);
}
console.log('File has been Deleted');
});
}, 30000);
res.download(filePath);
}
})
You can use https://www.npmjs.com/package/json2csv
Set nested option value to true
And Also Specify fields that you want from JSON. For nested document you can specify like this details.Gender

Resources