I am using firebase hosting for the purpose of getting a google spreadsheet data using node js API. It works well if i fetch 1000 rows but when i fetch 10000, it give me error Error: could not handle the request.
Below is the url with 1000 request
https://us-central1-valuemyplace-1525346661329.cloudfunctions.net/app/spreadsheet
Below is the url with 10000 request
https://us-central1-valuemyplace-1525346661329.cloudfunctions.net/app/test
Here is my code
function listMajors(auth) {
const sheets = google.sheets({version: 'v4', auth});
sheets.spreadsheets.values.get({
spreadsheetId: '1Lpbx-LQQTh-lF2E1lQmPnqZhvT2bWfQ9gxBX-qGBz_M',
range: 'A1:P120933',
//valueRenderOption: 'UNFORMATTED_VALUES'
//majorDimension: 'COLUMNS'
}, (err, {data}) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = data.values;
if (rows.length) {
rows.map((row) => {
const zipObj = (arr1, arr2, obj) => {
if (arr1.length === 0 || arr2.length === 0) { return 0; }
obj[arr1[0]] = arr2[0];
return zipObj(arr1.slice(1), arr2.slice(1), obj);
}
const ress = [];
for (let i = 1; i < rows.length; i++) {
let obj = {};
zipObj(data.values[0], data.values[i], obj);
ress.push(obj);
}
//console.log(res);
firebase.database().ref().set(ress);
res.send(ress);
// console.log(row.length);
});
} else {
console.log('No data found.');
}
});
}
Please help me to solve this problem. Thanks in advance.
I solved by myself using
function listMajors(auth) {
const sheets = google.sheets({version: 'v4', auth});
sheets.spreadsheets.values.get({
spreadsheetId: '1Lpbx-LQQTh-lF2E1lQmPnqZhvT2bWfQ9gxBX-qGBz_M',
range: 'A1:P120933',
//valueRenderOption: 'UNFORMATTED_VALUES'
//majorDimension: 'COLUMNS'
}, (err, {data}) => {
if (err) return console.log('The API returned an error: ' + err);
const rows = data.values;
if (rows.length) {
** //rows.map((row) => {** *Do comment here*
const zipObj = (arr1, arr2, obj) => {
if (arr1.length === 0 || arr2.length === 0) { return 0; }
obj[arr1[0]] = arr2[0];
return zipObj(arr1.slice(1), arr2.slice(1), obj);
}
const ress = [];
for (let i = 1; i < rows.length; i++) {
let obj = {};
zipObj(data.values[0], data.values[i], obj);
ress.push(obj);
}
//console.log(res);
firebase.database().ref().set(ress);
res.send(ress);
// console.log(row.length);
**//});** *Comment Here*
} else {
console.log('No data found.');
}
});
}
Related
I've been trying to understand async/await and Task in NodeJS. I have implemented I need to update db when all process executed.
Below is code samples
const download = require('image-downloader');
const imageDownload = (imgurl) =>{
console.log("inside imageDownload ", new Date())
return new Promise(async(resolve) => {
let imgpath = '/mount/imageDownload/';
if (!fs.existsSync(imgpath)) {
fs.mkdirSync(imgpath, {
recursive: true
});
}
const options = {
url: imgurl,
dest: imgpath
};
console.log("before download image " + assetId + " ", new Date())
download.image(options)
.then(({ filename }) => {
resolve({"error":false, "url":filename});
})
.catch((err) => {
console.log(err);
resolve({"error":true,"message":err});
});
});
}
(async () => {
let downloadUrl1 = "";
let downloadUrl2 = "";
let downloadUrlFirst = await imageDownload("https://dummyimage.com/300.png/09f/fff");
if (typeof downloadUrlFirst.url != 'undefined' && downloadUrlFirst.url != null) {
downloadUrl1 = downloadUrlFirst.url;
}
let downloadUrlSecond = await imageDownload("https://dummyimage.com/300.png/09f/fff");
if (typeof downloadUrlSecond.url != 'undefined' && downloadUrlSecond.url != null) {
downloadUrl2 = downloadUrlSecond.url;
}
if (downloadUrl1 != '' || downloadUrl2 !='' ) {
let updateImagePayload = {}
if (portrait_mounturl != '') {
updateImagePayload.downloadUrl1 = downloadUrl1;
}
if (landscape_mounturl != '') {
updateImagePayload.downloadUrl2 = downloadUrl2;
}
updateImagePayload.modified_date_time = new Date().toISOString();
db.collection("images").findOneAndUpdate({_id:ObjectId(assetId)}, { $set: updateImagePayload }, (err, update) => {
if (err) {
resolve({"error":true,"message": err});
} else {
let resolveStatus = false;
let resolveMessage = "Successfully updated image";
resolve({"error":resolveStatus,"message":resolveMessage});
}
});
}
})();
I need to update if both has async proccess completed.
Note: Image will be optional. Means either 1 image or both
Getting High Memory and CPU utilization
Any help understanding this would be much appreciated.
I have a NodeJS+Express REST API method executing reverse geocoding (using Google's Maps API).
I'm trying to solve it with Promises but the 'then' is getting executed before my function returns with the answers from Google.
When testing the same code just calling a setTimeout, it works as expected. Please see comments in the code (simplify version).
app.get('/api/v1/events', verifyToken, async (req, res) => {
await db.poolPromise.then(pool => {
return pool.request()
.input('UserId', db.sql.UniqueIdentifier, res.authData.userId)
.input('DateFrom', db.sql.DateTime2(7), req.query.dateFrom)
.input('DateTill', db.sql.DateTime2(7), req.query.dateTo)
.output('UserIdAuthorized', db.sql.Bit)
.execute('sp')
}).then(result => {
let output = (result.output || {})
if (!output.UserIdAuthorized) {
res.sendStatus(403)
}
else if (result.recordset.length > 0) {
(new Promise( (resolve) => {
//resolve(123) // this one works as expected
//setTimeout(resolve, 3000, 'temp success') // this one works as expected
// *** this one get passed and the following then is being executed before it answers ***
resolve( getAddress_TEST(result.recordset) )
// **************************************************************************************
})).then(function (value) {
res.json(
{
meta: { count: 10 }, //this is just a sample
result: value // *** this one fails with undefined ***
})
})
} else {
res.sendStatus(404)
}
}).catch(err => {
res.sendStatus(500)
console.error(err)
})
});
const nodeGeocoder_options = {
provider: 'google',
apiKey: process.env.GOOGLE_API_KEY
}
async function getAddress_TEST(recordset) {
//sample recordset for debugging - as you dont have my database
recordset = [{'eventId':14205556,'Lat':54.57767,'Lon':-2.4920483},{'eventId':14205558,'Lat':54.57767,'Lon':-2.492048},{'eventId':14205579,'Lat':53.416908,'Lon':-2.952071},{'eventId':14205588,'Lat':52.644448,'Lon':-1.153185},{'eventId':14205601,'Lat':52.29174,'Lon':-1.532283},{'eventId':14205645,'Lat':52.644448,'Lon':-1.153185},{'eventId':14205801,'Lat':53.68687,'Lon':-1.498708},{'eventId':14206041,'Lat':51.471521,'Lon':-0.2038033},{'eventId':14206049,'Lat':51.471521,'Lon':-0.2038033},{'eventId':14206072,'Lat':51.471521,'Lon':-0.2038033}]
let geocoder = nodeGeocoder(nodeGeocoder_options)
let ps = []
for (var i = 0, length = recordset.length; i < length; i++) {
if (i == 0 || !(i > 0
&& recordset[i - 1].Lat == recordset[i].Lat
&& recordset[i - 1].Lon == recordset[i].Lon)) {
ps.push(new Promise(function (resolve) {
resolve(reverseGeocode(geocoder, recordset[i].Lat, recordset[i].Lon))
}))
} else {
ps.push('-')
}
}
await Promise.all(ps)
.then(function (values) {
for (var i = 0, length = values.length; i < length; i++) {
if (values[i] != '-') {
recordset[i].locationAddress = values[i]
} else {
recordset[i].locationAddress = recordset[i - 1].locationAddress
}
}
}).then(function () {
recordset.forEach(function (v) {
delete v.Lat
delete v.Lon
});
console.log(recordset)
return recordset
})
};
async function reverseGeocode(geocoder, lat, lon) {
let address = '+'
if (lat != 0 && lon != 0) {
await geocoder.reverse({ lat: lat, lon: lon })
.then(res => {
address = res[0].formattedAddress
})
.catch(err => {
console.error(err)
});
}
return address
};
I'm sure it is something simple that I'm missing here...
The basic problem is that your getAddress_TEST function returns a promise that fulfills with nothing (undefined), because it does not contain a return statement. The return recordset is in a then() callback, from where it affects the promise resolution of the awaited promise, but that result is thrown away.
If you want to use async/await, you should get rid of any new Promise and then calls:
app.get('/api/v1/events', verifyToken, async (req, res) => {
try {
const pool = await db.poolPromise
const result = await pool.request()
.input('UserId', db.sql.UniqueIdentifier, res.authData.userId)
.input('DateFrom', db.sql.DateTime2(7), req.query.dateFrom)
.input('DateTill', db.sql.DateTime2(7), req.query.dateTo)
.output('UserIdAuthorized', db.sql.Bit)
.execute('sp')
let output = (result.output || {})
if (!output.UserIdAuthorized) {
res.sendStatus(403)
} else if (result.recordset.length > 0) {
const value = await getAddress_TEST(result.recordset)
res.json({
meta: { count: 10 }, //this is just a sample
result: value
})
} else {
res.sendStatus(404)
}
} catch(err) {
res.sendStatus(500)
console.error(err)
}
});
const nodeGeocoder_options = {
provider: 'google',
apiKey: process.env.GOOGLE_API_KEY
}
async function getAddress_TEST(recordset) {
const geocoder = nodeGeocoder(nodeGeocoder_options)
const ps = recordset.map((record, i) => {
if (i == 0 || !(i > 0
&& recordset[i - 1].Lat == record.Lat
&& recordset[i - 1].Lon == recordLon)) {
return reverseGeocode(geocoder, recordset[i].Lat, recordset[i].Lon))
} else {
return '-'
}
});
const values = await Promise.all(ps)
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
for (var i = 0, length = values.length; i < length; i++) {
if (values[i] != '-') {
recordset[i].locationAddress = values[i]
} else {
recordset[i].locationAddress = recordset[i - 1].locationAddress
}
}
recordset.forEach(function (v) {
delete v.Lat
delete v.Lon
});
console.log(recordset)
return recordset
// ^^^^^^^^^^^^^^^^
}
async function reverseGeocode(geocoder, lat, lon) {
if (lat != 0 && lon != 0) {
const res = await geocoder.reverse({ lat: lat, lon: lon })
return res[0].formattedAddress
}
return '+'
}
I'm new to Node.js and I'm creating a simple pagination page. The REST API works fine, but consuming it has left me in limbo.
Here is the REST API (other parts have been taken out for brevity)
const data = req.query.pageNo;
const pageNo =
(typeof data === 'undefined' || data < 1) ? 1 : parseInt(req.query.pageNo);
let query = {};
const total = 10;
query.skip = (total * pageNo) - total;
query.limit = total;
try {
const totalCount = await Users.countDocuments();
const pageTotal = Math.ceil(totalCount / total);
const users = await Users.find({}, {}, query);
return res.status(200).json(users);
} catch (error) {
console.log('Error ', error);
return res.status(400).send(error)
};
};
When I return the json with just the 'users' object, like so return res.status(200).json(users); the page renders correctly, but when I pass in other objects like what I have in the code, it fails. This is how I'm consuming the API:
const renderHomepage = (req, res, responseBody) => {
let message = null;
if (!(responseBody instanceof Array)) {
message = 'API lookup error';
responseBody = [];
} else {
if (!responseBody.length) {
message = 'No users found nearby';
}
}
res.render('users-list', {
title: 'Home Page',
users: responseBody,
message: message
});
}
const homelist = (req, res) => {
const path = '/api/users';
const requestOptions = {
url: `${apiOptions.server}${path}`,
method: 'GET',
json: true,
};
request(
requestOptions,
(err, {statusCode}, body) => {
if (err) {
console.log('Ther was an error ', err);
} else if (statusCode === 200 && body.length) {
renderHomepage(req, res, body);
} else if (statusCode !== 200 && !body.length) {
console.log('error ',statusCode);
}
}
);
}
I've searched extensively on both here and other resources but none of the solutions quite answers my question. I hope someone could be of help
I need help with code below. I get an array of items from the client then the goal is to save them in mongodb and return the list classified as 'saved' and 'failed' items. sample of failed items are those that are duplicate on a unique attribute.
I know the code below will not work because of variable scope. how do i get around it? the code below returns an empty array for both savedItems and failedItems. Thanks!
router.post('/addItems', async (req, res, next) => {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
function saveData() {
for (i = 0; i < items.length; i++) {
item = items[i];
Model.create({ ...item }, (err, data) => {
if (err) {
result.failedItems.push(item);
} else {
result.savedItems.push(item);
}
});
}
return result;
}
saveData().then(result => {
res.send({
results: result
});
});
});
router.post('/addItems', async (req, res, next) => {
// use try catch when use async
try {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
for (let i = 0; i < items.length; i++) {
const item = items[i];
// use the returned promise instead of callback for Model.create
const data = await Model.create({ ...item });
result.savedItems.push(item);
// if also need to handle failed item in result use anathor try catch inside
/*try {
const data = await Model.create({ ...item });
result.savedItems.push(item);
} catch( err ) {
result.failedItems.push(item);
}*/
}
res.send({
results: result
});
} catch( err ) {
// To all the errors unexpected errors + thrown rejected promises
res.send({
error: err
});
}
});
Your saveData method didn't return a promise, try this
function saveData() {
return new Promise(resolve => {
let items = req.body;
let result = {
savedItems: [],
failedItems: []
};
let promises = [];
for (i = 0; i < items.length; i++) {
item = items[i];
let promise = new Promise(resolve => {
Model.create({ ...item }, (err, data) => {
if (err) {
result.failedItems.push(item);
} else {
result.savedItems.push(item);
}
resolve();
});
});
promises.push(promise);
}
Promise.all(promises).then(() => resolve(result));
})
}
The request body sometimes (less than 1% of the time) is null when pulled into lambda. I am processing on the order of 14,000 request bodies one at a time. Any request bodies erring out have to be handled manually. Why is the body randomly coming in null?
Sample code ran from prompt (node index):
const async = require('async');
const _ = require('lodash');
const moment = require('moment');
const Client = require('node-rest-client').Client;
const fs = require('fs');
const input = require('./TestFull.json');
module.exports = () => {
const filename = `./eventfulFails-${new moment().format("YYYY-MM-DD-HHmmss")}.json`;
console.log('Start Time: ', new moment().format("HH:mm:ss"));
let failedObjects = {
events: [],
venues: [],
performers: []
};
async.parallel([
async.apply(processVenues, input.venues, failedObjects),
async.apply(processPerformers, input.performers, failedObjects)
], (lookupErr) => {
if (lookupErr) {
return console.error('Error processing venues and performers.', lookupErr);
}
console.log('Start Events: ', new moment().format("HH:mm:ss"));
async.waterfall([
async.apply(processEvents, input.events, failedObjects)
], (eventErr) => {
if (eventErr) {
console.log('Time of Failure: ', new moment().format("HH:mm:ss"));
return console.error('Error processing events.', eventErr);
}
console.log('End Time: ', new moment().format("HH:mm:ss"));
if (failedObjects.events.length || failedObjects.venues.length || failedObjects.performers.length) {
const stream = fs.createWriteStream(filename);
stream.once('open', function(fd) {
stream.write(JSON.stringify(failedObjects));
stream.end();
});
}
});
});
};
function processVenues(venues, failedObjects, callback) {
const calls = [];
for (let i = 0; i < venues.length; i++) {
const v = venues[i];
calls.push(async.apply((venue, postCallback) => {
const client = new Client();
const args = {
data: venue,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful-venue', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.venues.push({
venue,
response
});
console.log('venue status code: ', response);
console.log('venue data: ', venue);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
function processPerformers(performers, failedObjects, callback) {
const calls = [];
for (let i = 0; i < performers.length; i++) {
const v = performers[i];
calls.push(async.apply((performer, postCallback) => {
const client = new Client();
const args = {
data: performer,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful-performer', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.performers.push({
performer,
response
});
console.log('performer status code: ', response);
console.log('performer data: ', performer);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
function processEvents(events, failedObjects, callback) {
const calls = [];
for (let i = 0; i < events.length; i++) {
const v = events[i];
calls.push(async.apply((event, postCallback) => {
const client = new Client();
const args = {
data: event,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.events.push({
event,
response
});
console.log('event status code: ', response);
console.log('event data: ', event);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
if (!module.parent) {
module.exports();
}
Code of function processVenues (eventful-venue-load) that is called:
const _ = require('lodash');
const AWS = require('aws-sdk');
const async = require('async');
const sdk = require('#consultwithmikellc/withify-sdk');
const host = process.env.aurora_host;
const user = process.env.aurora_user;
const database = process.env.aurora_database;
let decryptedPassword;
const lambda = new AWS.Lambda({
region: 'us-west-2' //your region
});
class WithifyEventCreate extends sdk.Lambda {
constructor(event, context, keysToDecrypt) {
super(event, context, keysToDecrypt);
this.getLocation = this.getLocation.bind(this);
this.insertLocations = this.insertLocations.bind(this);
this.insertLocationImages = this.insertLocationImages.bind(this);
}
decryptedKey(key, value) {
switch (key) {
case 'aurora_password':
decryptedPassword = value;
break;
}
}
initializeComplete() {
this.connect(host, user, decryptedPassword, database, true);
}
connectComplete() {
async.waterfall(
[
this.getLocation,
this.insertLocations,
this.insertLocationImages
]
);
}
getLocation(callback) {
const {id: eventfulLocationID} = this.body;
this.connection.query('SELECT * FROM `Location` WHERE `eventfulLocationID` = ?',
[eventfulLocationID],
(err, results) => {
if (err) {
// error call block
return this.sendResponse(err, this.createResponse(500));
} else if (results.length === 1) {
console.log('Invoking withify-eventful-venue-update...');
lambda.invoke({
FunctionName: 'withify-eventful-venue-update',
Payload: JSON.stringify(this.event)
}, (error, data) => {
return this.sendResponse(null, JSON.parse(data.Payload));
});
} else if (results.length > 1) {
return this.sendResponse(`The location lookup produced multiple results. event:${JSON.stringify(this.body)}`, this.createResponse(500));
} else {
return callback(null);
}
}
);
}
insertLocations(callback) {
const {name: locationName, address: street, city, region_abbr: state, postal_code,
description, id: eventfulLocationID, latitude: lat, longitude: lng, withdrawn: locationWithdrawn} = this.body;
let addresses = street.concat(', ', city, ', ', state, ', ', postal_code);
if (!description.length){
var phones = "";
}else{
var re = /(([\(][0-9]{3}[\)][\s][0-9]{3}[-][0-9]{4})|([0-9]{3}[-][0-9]{3}[-][0-9]{4})|([0-9]{3}[\.][0-9]{3}[\.][0-9]{4}))/i;
this.body.found = description.match(re);
if (!this.body.found){
var phone = "";
}else{
if (!this.body.found.length){
var phone = "";
}else{
var phone = this.body.found[0];
}
}
}
this.connection.query('INSERT IGNORE INTO `Location` (`locationName`, `address`, ' +
'`phone`, `lat`, `lng`, `eventfulLocationID`, `locationWithdrawn`) VALUES (?, ?, ?, ?, ?, ?, ?)',
[locationName, addresses, phone, lat, lng, eventfulLocationID, locationWithdrawn],
(err, results) => {
if (err) {
return this.sendResponse(err, this.createResponse(500));
}
this.body.locationID = results.insertId;
return callback(null);
}
);
}
insertLocationImages(callback) {
var altText = "";
const images = _.flatten(this.body.images.map(im => {
return _.map(im.sizes, (ims, idx) => {
const title = `Image ${idx}`;
return [
this.body.locationID,
this.body.name,
ims.url,
null,
null,
this.body.id,
ims.width,
ims.height
];
});
}));
if(!images[0]){
return this.sendResponse(null, this.createResponse(201, this.body));
}
this.connection.query('INSERT IGNORE INTO `LocationImage` (`locationID`, `imageTitle`, `imageUrl`, ' +
'`imageName`, `altText`, `eventfulLocationID`, `width`, `height`) VALUES ?',
[images],
(err, results) => {
if (err) {
return this.sendResponse(err, this.createResponse(500));
} else if (results.affectedRows !== images.length) {
return this.sendResponse('The image inserts did not affect the right number' +
' of rows.', this.createResponse(500));
}
return this.sendResponse(null, this.createResponse(201, this.body));
}
);
}
}
exports.handler = (event, context) => {
const withifyEventCreate = new WithifyEventCreate(event, context, ['aurora_password']);
withifyEventCreate.initialize([decryptedPassword]);
};