I have created a mongodb collection in mongolab and want to print all documents in that collecton. I have mongolab and the url of mongolab to get connected with the collection is -
mongodb://user:1234#xyz.mongolab.com:41248/new
The sample document structure is -
{
"_id": "9759572745-Sing",
"details": {
"Gender": "M",
"PreTrainingStatus": "Fresher",
"Religion": "Hindu",
"attendanceInPercentage": "",
"batchHolders": {
"AssessmentDate": "Thu Jul 16 2015",
"CourseFee": "7500",
"isEditable": false
},
"batchID": "282726",
"eid": "",
"whereDidYouHearAboutStar": "---Select---",
"skillInstructorOrTrainerName": "282726",
"specificGovtInstitutetieups": "---Select---",
"isSelected": false,
"isEditable": false
},
"addedOnMs": 1439455766000,
"submittedOnMs": 1439454813000,
"latitude": "27.409566879272",
"longitude": "77.69295501709",
"locationName": "Uttar Pradesh 281006,null"
}
I want to Print it and all nested properties to be displayed in a column.
But I am not able to do so, Please help.
Thank you (In Advance),
Dia
Implementing json2csv library for exporting data to csv file on nodejs
Example
const json2csv = require('json2csv').parse;
//For unique file name
const dateTime = new Date().toISOString().slice(-24).replace(/\D/g,
'').slice(0, 14);
const filePath = path.join(__dirname, "../../../", "public", "exports", "csv-" + dateTime + ".csv");
let csv;
const student = await req.db.collection('Student').find({}).toArray();
// Logging student
// [{id:1,name:"John",country:"USA"},{id:1,name:"Ronny",country:"Germany"}]
const fields = ['id','name','country'];
try {
csv = json2csv(booking_info, {fields});
} catch (err) {
return res.status(500).json({err});
}
fs.writeFile(filePath, csv, function (err) {
if (err) {
return res.json(err).status(500);
}
else {
setTimeout(function () {
fs.unlink(filePath, function (err) { // delete this file after 30 seconds
if (err) {
console.error(err);
}
console.log('File has been Deleted');
});
}, 30000);
res.download(filePath);
}
})
You can use https://www.npmjs.com/package/json2csv
Set nested option value to true
And Also Specify fields that you want from JSON. For nested document you can specify like this details.Gender
Related
I have a google sheet with a list (dropdown, or data validation - list from ranges, as Google Sheets call it), like so:
Image of sheet
Imagine that in the list I have 4 values to select from. My goal is not only to get all table values, but also all values that constitute the list ["Beer","Wine","Rum","Martini"].
I've tried 2 different ways to retrieve the info and the list:
a) With sheets.spreadsheets.values.get, I get the table values in a digestible way, but not the content of the dropdown. Instead, the cell comes in as blank ("") [Comment: on Apps Script, you would get this information]
b) With sheets.spreadsheets.getByDataFilter, I get much more than I need and in a horrible format. However, I do not get the dropdown content as an array (as I'd want), but rather as a refence: (userEnteredValue: "=Input!$F$5:$F$7")
The question is, how do I get only the table, including the dropdown content as an array? I know it is possible and easy to do in Google Apps Script (I have it implemented), but not on Node.
Below the code as a reference for other programmers.
var {google} = require("googleapis");
let privatekey = require("./client_secret.json");
// configure a JWT auth client
let jwtClient = new google.auth.JWT(
privatekey.client_email,
null,
privatekey.private_key,
['https://www.googleapis.com/auth/spreadsheets',
]);
//authenticate request
jwtClient.authorize(function (err, tokens) {
if (err) {
console.log(err);
return;
} else {
console.log("Successfully connected!");
}
});
//Google Sheets API
let spreadsheetId = '<SPREADSHEET ID>';
let sheetName = 'Input!A1:B4'
let sheets = google.sheets('v4');
exports.fetch = (req, res) => {
sheets.spreadsheets.values.get({
auth: jwtClient,
spreadsheetId: spreadsheetId,
range: sheetName,
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err);
} else {
res.json(response);
}
});
// OR
sheets.spreadsheets.getByDataFilter({
auth: jwtClient,
spreadsheetId: spreadsheetId,
"includeGridData": true,
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err);
} else {
res.json(response);
}
});
}
What you are interested in is dataValidation
In order to retrieve it, you can use the method spreadsheets.get, setting the parameter fields to sheets/data/rowData/values/dataValidation
If your data validation is set as List of items, the response will look like:
{
"sheets": [
{
"data": [
{
"rowData": [
{
"values": [
{
"dataValidation": {
"condition": {
"type": "ONE_OF_LIST",
"values": [
{
"userEnteredValue": "Beer"
},
{
"userEnteredValue": "Wine"
},
{
"userEnteredValue": "Martini"
}
]
},
"showCustomUi": true
}
}
]
}
]
}
]
}
]
}
If your data validation is set as List form a range the response will look like:
{
"sheets": [
{
"data": [
{
"rowData": [
{
"values": [
{
{
"dataValidation": {
"condition": {
"type": "ONE_OF_RANGE",
"values": [
{
"userEnteredValue": "=Sheet1!$B$1:$B$3"
}
]
},
"showCustomUi": true
}
}
]
}
]
}
]
}
]
}
In the latter case you would need to call subsequently the method spreadsheets.values.get on =Sheet1!$B$1:$B$3 - that is the range returned as userEnteredValue within the object values nested within the object dataValidation.
I am working on AWS Lambda and creating method by using node.js.
I need an object like this:
[
{
"TeamName" : "Sales",
"2020-01-01": "90",
"2020-01-02": "92",
"2020-01-03": "95",
"2020-01-04": "90",
"2020-01-05": "56",
"2020-01-06": "70",
"2020-01-07": "73"
},
]
but my current response is this:
[
{
"TeamName": "Billing",
"DateTime": "2020-06-13T00:00:00.000Z",
"Score": 9
},
{
"TeamName": "Billing",
"DateTime": "2020-06-13T00:00:00.000Z",
"Score": 9
},
{
"TeamName": "Billing",
"DateTime": "2020-06-11T00:00:00.000Z",
"Score": 5
},
]
Here is my Lambda method. I am not good at creating javascript object so please help me to make a response like this, Thanks.
exports.handler = (event, context, callback) => {
console.log('Events:',event);
let UserHierarchyGroupID = event['hierarchyGroupId'];
let team = [];
// allows for using callbacks as finish/error-handlers
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) throw err;
let sql = `SELECT date(Feedback.DateTime) as datetime,Feedback.Score,UserHierarchy.Layer5
FROM ctrData2.Feedback
LEFT OUTER JOIN ctrData2.CallDetail ON CallDetail.ContactId = Feedback.FeedbackID
LEFT OUTER JOIN ctrData2.UserTable ON UserTable.UserID = CallDetail.UserID
LEFT OUTER JOIN ctrData2.UserHierarchy ON UserTable.UserID = UserHierarchy.UserID
WHERE UserTable.UserHierarchyGroupID=?`;
let field = [UserHierarchyGroupID];
connection.query(sql,field, function (err, result, fields) {
if (err) throw err;
// console.log(result);
connection.release();
var date;
var score;
if(result.length>0){
result.forEach((item)=>{
team.push({
"TeamName": item.Layer5,
"DateTime": item.datetime,
"Score": item.Score
});
});
}else{
callback(null,{
status: 404,
Body: "Not found"
});
}
callback(null,team);
// FomratObjects(result,(formattedResponse)=>{
// // console.log(formattedResponse);
// callback(formattedResponse);
// });
});
});
};
Its doesn't look possible to create an object exactly like you mentioned but you can do this to assign value to every single date.
Hope it will be helpful.
function formatData(data){
var nObject = {};
data.forEach(d=>{
nObject[moment(d.datetime).format('MM-DD-YYYY')]=d.Score;
});
return nObject;
}
I'm absolutely brand new to DynamoDb and I'm trying to simply write an object from a NodeJS Lambda. Based on what I've read and researched I should probably be using DocumentClient from the aws-sdk. I also found the following question here regarding issues with DocumentClient, but it doesn't seem to address my specific issue....which I can't really find/pinpoint unfortunately. I've set up a debugger to help with SAM local development, but it appears to be only providing some of the errors.
The code's implementation is shown here.
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": {"S": randstring.generate(9)},
"School":{"S": team_name},
"Seed": {"S": seed},
"ESPN_Id": {"S": espn_id}
}
}
console.log(JSON.stringify(params))
dynamodb.put(params, (error,data) => {
if (error) {
console.log("Error ", error)
} else {
console.log("Success! ", data)
}
})
Basically I'm scrubbing a website utilizing cheerio library and cherry picking values from the DOM and saving them into the json object shown below.
{
"TableName": "March-Madness-Teams",
"Item": {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
When I attempt to push this json object to Dynamo, I get errors says
Error MultipleValidationErrors: There were 2 validation errors:
* MissingRequiredParameter: Missing required key 'TableName' in params
* MissingRequiredParameter: Missing required key 'Item' in params
The above error is all good in well....I assume it didn't like the fact that I had wrapped those to keys in strings, so I removed the quotes and sent the following
{
TableName: "March-Madness-Teams",
Item: {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
However, when I do that...I kind of get nothing.
Here is a larger code snippet.
return new Promise((resolve,reject) => {
axios.get('http://www.espn.com/mens-college-basketball/bracketology')
.then(html => {
const dynamodb = new aws.DynamoDB.DocumentClient()
let $ = cheerio.load(html.data)
$('.region').each(async function(index, element){
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(async function(index2, element2){
var seed = $(element2).siblings('span.rank').text()
if (seed.length > 2){
seed = $(element2).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element2).attr('href').split('/').slice(-2)[0]
var team_name = $(element2).text()
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School":team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
}
console.log(JSON.stringify(params))
// dynamodb.put(params)
// .then(function(data) {
// console.log(`Success`, data)
// })
})
})
})
})
Can you try without the type?
Instead of
"School":{"S": team_name},
for example, use
"School": team_name,
From your code, I can see the mis promise on the dynamodb request. Try to change your lines :
dynamodb.put(params).then(function(data) {
console.log(`Success`, data)
})
to be :
dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
you can combine with await too :
await dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
exports.lambdaHandler = async (event, context) => {
const html = await axios.get('http://www.espn.com/mens-college-basketball/bracketology')
let $ = cheerio.load(html.data)
const schools = buildCompleteSchoolObject(html, $)
try {
await writeSchoolsToDynamo(schools)
return { statusCode: 200 }
} catch (error) {
return { statusCode: 400, message: error.message }
}
}
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(async school => {
await dynamodb.put(school).promise()
})
await Promise.all(promises)
}
const buildCompleteSchoolObject = (html, $) => {
const schools = []
$('.region').each(loopThroughSubRegions(schools, $))
return schools
}
const loopThroughSubRegions = (schools, $) => {
return (index, element) => {
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(populateSchoolObjects(schools, $))
}
}
const populateSchoolObjects = (schools, $) => {
return (index, element) => {
var seed = $(element).siblings('span.rank').text()
if (seed.length > 2) {
seed = $(element).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element).attr('href').split('/').slice(-2)[0]
var team_name = $(element).text()
schools.push({
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School": team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
})
}
}
I know this is drastically different from what I started with but I did some more digging and kind of kind of worked to this...I'm not sure if this is the best way, but I seemed to get it to work...Let me know if something should change!
Oh I understand what you want.
Maybe you can see the code above works, but there is one concept you have to improve here about async - await and promise especially on lambda function.
I have some notes here from your code above, maybe can be your consideration to improve your lambda :
Using await for every promise in lambda is not the best approach because we know the lambda time limitation. But sometimes we can do that for other case.
Maybe you can change the dynamodb.put method to be dynamodb.batchWriteItem :
The BatchWriteItem operation puts or deletes multiple items in one or more tables.
Or If you have to use dynamodb.put instead, try to get improve the code to be like so :
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(school => {
dynamodb.put(school).promise()
})
return Promise.all(promises)
}
I want to convert the result from aggregate to a csv. In the code (or pseudocode) below the results of the aggregate are stored in variable cursor
I want to return a csv as a response ( res.send(csv) ). Do I have to use res.set('Content-Type', 'content-type: text/csv') ?
The code below is a mix of NodeJs and pseudocode. Function covert_to_csv indicates my ignorance about the subject.
const express = require('express')
const app = express();
app.get('/', (req, res) => {
let agg=[ /* a query in MongoDB */]
MongoClient.connect(URL,(err, client) => {
res.set('Content-Type', 'content-type: text/csv');
let collection = client.db('db').collection('col')
let cursor = collection.aggregate(agg)
let csv_file = covert_to_csv(cursor)
csv_file.toArray((error, result) => {
res.send(result);
});
});
app.listen(port, () => console.log(`listening on port ${port}!`));
Edit:
In my original code i am getting a response in json.I run the following code:
let agg=[ /* a query in MongoDB */]
let cursor = collection.aggregate(agg)
cursor.toArray((error, result) => {
if(error) {
return res.status(500).send(error);
res.send(result);
I am getting this response from Postman in JSON :
[
{
"Source": "entso-e",
"Dataset": "ActualTotalLoad",
"AreaName": "Germany",
"AreaTypeCode": "CTY",
"MapCode": "DE",
"ResolutionCode": "PT15M",
"Year": "2018",
"Month": "1",
"Day": "2",
"DateTimeUTC": "2018-01-02 00:00:00.0000000",
"ActualTotalLoadValue": "41412.38",
"UpdateTimeUTC": "2018-01-02 13:16:19.0000000"
},
{
"Source": "entso-e",
"Dataset": "ActualTotalLoad",
"AreaName": "Germany",
"AreaTypeCode": "CTY",
"MapCode": "DE",
"ResolutionCode": "PT15M",
"Year": "2018",
"Month": "1",
"Day": "2",
"DateTimeUTC": "2018-01-02 00:45:00.0000000",
"ActualTotalLoadValue": "40785.17",
"UpdateTimeUTC": "2018-01-02 13:16:19.0000000"
},
..... (more documents)
]
You can use a module like json2csv to do the job for you. Your example will look something like that
const { Parser } = require('json2csv');
app.get('/', function(req, res) {
let agg = [/* mongo aggregation */]
res.setHeader('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', 'attachment; filename=\"' + 'download-' + Date.now() + '.csv\"');
let cursor = collection.aggregate(agg)
const fields = ['Source', 'Dataset', 'Areaname'];//all your field names
const json2csvParser = new Parser({ fields });
cursor.toArray((error, result) => {
res.send(json2csvParser.parse(result))
})
})
Try referring to https://www.npmjs.com/package/fast-csv and set keys as header option to true
I have excel sheet called sampledata.xlsx which i converted into json and console.log to print this data.
server.js
var xlsx2json = require('xlsx2json')
xlsx2json(
'sampledata.xlsx',
{
dataStartingRow: 2,
mapping: {
'name': 'B',//name
'sku': 'C',//unit price //sku
'quantity': 'D',//quantity
}
}).then(jsonArray => {
// [
// {"col_1": "Barton LCC", "col_2": "30", "col_3": "86.69"}
// ]
//console.log(jsonArray);
});
with the help of this doc.
What i want to do here is,in my sampledata.xlsx file i have more data like flat,address,price,etc here i already don't know which fields are present in my excel sheet but i want all that to be console.log.How could i do this is there any way to do this.
import xlsx2json from 'xlsx2json';
OR
const xlsx2json = require('xlsx2json');
const excel2json = [
(req, res, next) => {
xlsx2json(req.body.file.path)
.then(result => result[0])
.reduce((object, item, index) => {
if (index === 0) {
object.mapper = item; // eslint-disable-line no-param-reassign
return object;
}
const data = {};
Object.keys(item).forEach((key) => {
data[object.mapper[key]] = item[key];
});
object.data.push(data);
return object;
}, { mapper: {}, data: [] })
.then(excel => console.log(excel)) // this gives json as output
.catch(err => next(err));
},
];
npm install xlsx-to-json-lc --save
npm install xls-to-json-lc --save
var exceltojson = require("xls-to-json-lc");
exceltojson({
input: "pass the input excel file here (.xls format)"
output: "if you want output to be stored in a file"
sheet: "sheetname", // specific sheetname inside excel file (if you have multiple sheets)
lowerCaseHeaders:true //to convert all excel headers to lowr case in json
}, function(err, result) {
if(err) {
console.error(err);
} else {
console.log(result);
//result will contain the overted json data
}
});