I have a fairly straightforward script that reads summary data from an api and the loops through the records to save the detail to a database.
The code runs without problems when I launch it from VS Code but when I move it into a Lambda function it only runs halfway through.
There are two api calls using axios. The first gets the summary and the second pulls the detail.
The first call works in Lambda. The second, which uses the same method, does not. I can tell through logging statements that the correct data is getting to the second method. The only real differences are that the second is in a loop and it also uses Bottleneck to prevent overloading a touchy api.
I have put logging statements all over the place but once the routine enters the second api call I get no response at all. The logging statement directly inside the routine shows that it is getting there but I don't get anything back from axios. No success or error.
Here is the code.
var Bottleneck = require("bottleneck");
const axios = require('axios');
const Sequelize = require('sequelize');
let apiKey = process.env.APIKEY;
var timeDelay = 1000;
const instance = axios.create({
baseURL: 'https://anapi.com/api/v1/',
headers: {
'Content-Type': "application/json",
'X-Auth-Key': apiKey,
}
});
const limiter = new Bottleneck({
maxConcurrent: 1,
minTime: timeDelay
});
const sequelize = new Sequelize(
"postgres://postgres:reallystrongpassword#awsrdsdb.cluster-vms39sknjssk1.us-west-2.rds.amazonaws.com/targetdatabase"
);
const notes = sequelize.define(
"notes",
{
appointmentid: {
type: Sequelize.STRING,
}, ...
questions: {
type: Sequelize.JSONB,
},
},
{
tableName: "notes",
timestamps: false
}
);
async function notesInject(detailData) {
log.info("inside notesInject");
const injector = await notes.create({
appointmentid: detailData.AppointmentId,
...
questions: detailData.Questions,
}).then(function(){
log.info("created note ", detailData.Id)
}).catch(function(error){
log.info(error)
})
}
function getDetail(detailId) {
log.info(detailId)
try {
instance.get('notes/' + detailId)
.then ((resp) => {
try {
var detailData = (resp.data)
} catch {
log.info("detailData success", resp.status)
}
try {
notesInject(detailData)
} catch (error) {
log.info("notesInject catch", resp.status);
}
})
} catch (error) {
log.info("error in the detail instance")
}
}
function procDetail(apiData) {
for (let i = 0; i < apiData.length; i++) {
const element = apiData[i];
let detailId = element.Id;
getDetail(detailId)
}
}
function getTodayData() {
const pullDate = new Date();
const dateY = pullDate.getFullYear();
const dateM = pullDate.getMonth()+1;
const dateD = pullDate.getDate()-1;
const apiDate = (dateY+'-'+dateM+'-'+dateD)
try {
instance.get('notes/summary?startDate=' + apiDate)
.then ((resp) => {
try {
var apiData = (resp.data)
} catch {
log.info("set apiData", resp.status)
}
try {
procDetail(apiData)
} catch (error) {
log.info("saveDetail", resp.status);
}
})
} catch (error) {
log.info("in the summary instance")
}
}
exports.handler = async (event) => {
getTodayData();
};
I was thinking that the problem was with Bottleneck because that is the most significant difference between the first and second axios calls. When I isolated the database write code after the api pull, it had the same behavior. No error or response.
I'm stumped. I've logged everything I can think of. Lambda doesn't display console.log messages for this so I've been using Lambda-Log.
I'm sure it's something dumb but it works just fine when I run it from Code.
If anyone has any idea what I'm doing wrong, please let me know.
Sorry if I posted too much code but I really don't know where the problem is.
Many thanks
Related
I am using express with MongoDB as a database and while I am sending requests using Axios library to my backend, I have found that two documents were created at the time. I figured that out by checking the createdAt field and found the two documents have the exact timestamp. I am creating createdAt & updatedAt fields using mongoose-timestamp2
Here is the code responsible for sending the documents which have isSynchronized flag set to false
const syncExams = async (token) => {
let examSyncError;
await Exam.find({ isSynchronized: false })
.cursor()
.eachAsync(async (exam) => {
// use try/catch inside the loop
// so if problem occured in one document, others will be posted normally
try {
const formData = new FormData();
formData.append("exam", JSON.stringify(exam));
Object.keys(exam.images.toJSON()).forEach((key) => {
formData.append("images", fs.createReadStream(exam.images[key]));
});
const res = await axios.post(`${BACKEND_URL}/exams/`, formData, {
maxContentLength: Infinity,
maxBodyLength: Infinity,
headers: {
...formData.getHeaders(),
Authorization: `Bearer ${token}`,
},
});
console.log(res);
exam.isSynchronized = true;
await exam.save();
} catch (err) {
examSyncError = err;
}
});
if (examSyncError) {
throw examSyncError;
}
};
And here is the controller responsible for handling the coming request and creating these documents in the database.
router.post(
"/exams",
fileUpload.array("images"),
trimmer,
syncControllers.postExam
);
const postExam = async (req, res, next) => {
let createdExam;
let exam;
let patient;
try {
const reqBody = JSON.parse(req.body.exam);
reqBody.operator = reqBody.creator;
delete reqBody._id;
delete reqBody.creator;
const { nationalID } = reqBody;
patient = await Patient.findOne({
"personalData.nationalID": nationalID,
}).exec();
if (!patient) {
return next(errorEmitter("exams.get-patient.not-found", 404));
}
createdExam = new Exam(reqBody);
// insert the new paths of the images
Object.keys(reqBody.images).forEach((key, index) => {
createdExam.images[key.replace("Path", "")].Path = req.files[index].path
.split(path.sep)
.slice(-3)
.join(path.sep);
});
createdExam.patient = patient._id;
const checkDate = new Date(createdExam.examDate);
checkDate.setMonth(checkDate.getMonth() - 3);
const nearExam = await Exam.findOne({
patient: createdExam.patient,
examDate: { $gte: checkDate, $lt: createdExam.examDate },
isGradable: true,
});
if (nearExam) {
createdExam.isGradable = false;
}
exam = await createdExam.save();
patient.exams.push(createdExam);
await patient.save();
} catch (err) {
return next(errorEmitter("exams.create.fail", 500));
}
return res.status(201).json(exam.toObject({ getters: true }));
};
I am using the following package for createdAt & updatedAt records created using this plugin.
const timestamps = require("mongoose-timestamp2");
examSchema.plugin(timestamps);
This problem occurred in two or three cases only and did not happen again until then and I could not replicate the error. I do not what caused this!
What could cause this problem to occur? Is it possible that Axios sent the same request multiple times and mongoose created two documents at the same exact time, or it could be a bug in mongoose-timestamp2 plugin?
Any help would be appreciated, thank you!
I have an iteration that can take up to hours to complete.
Example:
do{
//this is an api action
let response = await fetch_some_data;
// other database action
await perform_operation();
next = response.next;
}while(next);
I am assuming that the operation doesn't times out. But I don't know it exactly.
Any kind of explanation of nodejs satisfying this condition is highly appreciated. Thanks.
Update:
The actual development code is as under:
const Shopify = require('shopify-api-node');
const shopServices = require('../../../../services/shop_services/shop');
const { create } = require('../../../../controllers/products/Products');
exports.initiate = async (redis_client) => {
redis_client.lpop(['sync'], async function (err, reply) {
if (reply === null) {
console.log("Queue Empty");
return true;
}
let data = JSON.parse(reply),
shopservices = new shopServices(data),
shop_data = await shopservices.get()
.catch(error => {
console.log(error);
});
const shopify = new Shopify({
shopName: shop_data.name,
accessToken: shop_data.access_token,
apiVersion: '2020-04',
autoLimit: false,
timeout: 60 * 1000
});
let params = { limit: 250 };
do {
try {
let response = await shopify.product.list(params);
if (await create(response, shop_data)) {
console.log(`${data.current}`);
};
data.current += data.offset;
params = response.nextPageParameters;
} catch (error) {
console.log("here");
console.log(error);
params = false;
};
} while (params);
});
}
Everything is working fine till now. I am just making sure that the execution will ever happen in node or not. This function is call by a cron every minute, and data for processing is provided by queue data.
I am trying to call one async function from inside a loop run by another async function. These functions call APIs and I am using request-promise using nodeJS.
functions.js file
const rp = require("request-promise");
// function (1)
async email_views: emailId => {
let data = {};
await rp({
url: 'myapiurl',
qs: { accessToken: 'xyz', emailID: emailId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above JSON looks like this:
...
data:{
records: [
{
...
contactID: 123456,
...
},
{
...
contactID: 456789,
...
}
]
}
...
I am running a loop to get individual record, where I am getting a contactID associated with each of them.
// function#2 (also in functions.js file)
async contact_detail: contactId => {
let data = {};
await rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above function takes one contactId as parameter and gets that contact's detail calling another API endpoint.
Both functions work fine when they are called separately. But I am trying to do it inside a loop like this:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId); // function#2
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
The problem is I am only getting the first contact back from the above code block, i.e. even I have 20 records from function#1, in the loop when I am calling contact_detail (function#2) for each contactID (cId), I get contact detail once, i.e. for the first cId only. For rest I get nothing!
What is the correct way to achieve this using nodeJs?
UPDATE:
const { App } = require("jovo-framework");
const { Alexa } = require("jovo-platform-alexa");
const { GoogleAssistant } = require("jovo-platform-googleassistant");
const { JovoDebugger } = require("jovo-plugin-debugger");
const { FileDb } = require("jovo-db-filedb");
const custom = require("./functions");
const menuop = require("./menu");
const stateus = require("./stateus");
const alexaSpeeches = require("./default_speech");
const app = new App();
app.use(new Alexa(), new GoogleAssistant(), new JovoDebugger(), new FileDb());
let sp = "";
async EmailViewsByContactIntent() {
try {
const viewEmailId =
this.$session.$data.viewEmailIdSessionKey != null
? this.$session.$data.viewEmailIdSessionKey
: this.$inputs.view_email_Id_Number.value;
let pageIndex =
this.$session.$data.viewEmailPageIndex != null
? this.$session.$data.viewEmailPageIndex
: 1;
const result = await custom.email_views_by_emailId(
viewEmailId,
pageIndex
);
const records = result.data.records;
if (records.length > 0) {
const totalRecords = result.data.paging.totalRecords;
this.$session.$data.viewEmailTotalPages = totalRecords;
sp = `i have found a total of ${totalRecords} following view records. `;
if (totalRecords > 5) {
sp += `i will tell you 5 records at a time. for next 5 records, please say, next. `;
this.$session.$data.viewEmailIdSessionKey = this.$inputs.view_email_Id_Number.value;
this.$session.$data.viewEmailPageIndex++;
}
for (let i = 0; i < records.length; i++) {
const r = records[i];
/* Here I want to pass r.contactID as contactId in the function contact_detail like this: */
const contact = await custom.contact_detail(r.contactID);
const contact_name = contact.data.firstName + " " + contact.data.lastName;
/* The above two lines of code fetch contact_name for the first r.contactID and for the rest I get an empty string only. */
const formatted_date = r.date.split(" ")[0];
sp += `contact ID ${spellOut_speech_builder(
r.contactID
)} had viewed on ${formatted_date} from IP address ${
r.ipAddress
}. name of contact is, ${contact_name}. `;
}
if (totalRecords > 5) {
sp += ` please say, next, for next 5 records. `;
}
} else {
sp = ``;
}
this.ask(sp);
} catch (e) {
this.tell(e);
}
}
I am building an alexa skill using JOVO framework and nodeJS.
UPDATE #2
As a test, I only returned the contactId which I am passing to the contact_detail function and I am getting the correct value back to the above code under my first UPDATE.
async contact_detail: contactId => {
return contactId;
}
It seems even after getting the value right, the function is somehow failing to execute. However, the same contact_detail function works perfectly OK, when I am calling it from another place. Only doesn't not work inside a loop.
What could be the reason?
I must be missing something but don't know what!
You are mixing async await and promises together which is causing you confusion. You typically would use one of the other(as async await effectivly provides syntax sugar so you can avoid dealing with the verbose promise code) in a given location.
Because you mixed the two you are in a weird area where the behavior is harder to nail down.
If you want to use async await your functions should look like
async contact_detail: contactId => {
try {
const body = await rp({
url: 'myapiurl2',
qs: { ... }
});
return JSON.parse(body);
} catch(e) {
console.log(e);
//This will return undefined in exception cases. You may want to catch at a higher level.
}
};
or with promises
async contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { ... }
})
.then( body => JSON.parse(body))
.catch( error => {
console.log(error);
//This will return undefined in exception cases. You probably dont want to catch here.
});
};
Keep in mind your current code executing the function will do each call in series. If you want to do them in parallel you will need to call the function a bit differently and use something like Promise.all to resolve the result.
Here you go:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
await Promise.all(records.map(async record => {
let cId = record.contactID;
let contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
});
console.log(names);
...
I'm posting this as an answer only because I need to show you some multi-line code as part of throubleshooting this. Not sure this solves your issue yet, but it is a problem.
Your contact_detail() function is not properly returning errors. Instead, it eats the error and resolves with an empty object. That could be what is causing your blank names. It should just return the promise directly and if you want to log the error, then it needs to rethrow. Also, there's no reason for it to be declared async or to use await. You can just return the promise directly. You can also let request-promise parts the JSON response for you too.
Also, I notice, there appears to be a syntax error in your .catch() which could also be part of the problem.
contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
json: true,
method: 'GET'
}).catch( error => {
// log error and rethrow so any error propagates
console.log(error);
throw error;
});
};
Then, you would call this like you originally were (note you still use await when calling it because it returns a promise):
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
I have an API that searches for the user-provided term, returns an array of results, then fires off async requests for each of the results and gets results for each of these second batch of requests. I'd like the API to report progress as it happens rather than just the final result. So, if I do the following request, I should get updates like so
$ curl 'http://server/?q=foobar'
searching for ${q}…
found 76… now getting images…
found 30 images… done
{
result
}
Most of relevant code is shown below. Fwiw, I am using hapijs for my application.
let imagesOfRecords = {};
const getImages = async function (q) {
console.log(`searching for ${q}…`);
const uri = `http://remoteserver/?q=${q}`;
const {res, payload} = await Wreck.get(uri);
const result = JSON.parse(payload.toString()).hits;
const numOfFoundRecords = result.total;
if (result.total) {
console.log(`found ${result.total}… now getting images…`);
const foundRecords = result.hits.map(getBuckets);
Promise.all(foundRecords).then(function() {
console.log(`found ${Object.keys(imagesOfRecords).length} images… done`);
reply(imagesOfRecords).headers = res.headers;
}).catch(error => {
console.log(error)
});
}
else {
console.log('nothing found');
reply(0).headers = res.headers;
}
};
const getBuckets = async function(record) {
const { res, payload } = await Wreck.get(record.links.self);
const bucket = JSON.parse(payload.toString()).links.bucket;
await getImageFiles(bucket, record.links.self);
};
const getImageFiles = async function(uri, record) {
const { res, payload } = await Wreck.get(uri);
const contents = JSON.parse(payload.toString()).contents;
imagesOfRecords[record] = contents.map(function(el) {
return el.links.self;
});
};
Once I can implement this, my next task would be to implement this progressive update in a web app that uses the above API.
To show result with each step of your requests for backend you can use EventEmitter, which will emit event on each progress step. You can read about events here.
Simple implementation:
const events = require('events');
const eventEmitter = new events.EventEmitter();
//your request code
Promise.all(foundRecords).then(function() {
console.log(`found ${Object.keys(imagesOfRecords).length} images… done`);
eventEmitter.emit('progress');
reply(imagesOfRecords).headers = res.headers;
})
const eventReaction = (e) => {
// do something with event, console log for example.
}
eventEmitter.on('progress', eventReaction);
More examples you can find here and here.
To show events to client you can use library socket.io. I think you can find pretty straightforward explanations how socket.io works in documentation.
If you want to send events between servers or processes and want to go little further, you can read more about 0MQ (zero mq) and it's node implementation
For clarity I have other cloud functions that all run intermittently (i.e from 'cold' in around 2-6 seconds, and all use the same boilerplate set up of importing an admin instance and exporting the function as a module)
I've seen other similar posts but this is really bugging me. I have a cloud function like so:
const admin = require('../AdminConfig');
const { reportError } = require('../ReportError');
module.exports = (event) => {
const uid = event.params.uid;
const snapshot = event.data;
if (snapshot._newData === null ) {
return null;
}
console.log('Create org begin running: ', Date.now());
const organisation = event.data.val();
const rootRef = admin.database().ref();
const ref = rootRef.child('/organisations').push();
const oid = ref.key;
const userData = {
level: 'owner',
name: organisation.name,
};
const orgShiftInfo = {
name: organisation.name,
startDay: organisation.startDay || 'Monday',
};
const updatedData = {};
updatedData[`/users/${uid}/currentOrg`] = oid;
updatedData[`/users/${uid}/organisations/${oid}`] = userData;
updatedData[`/organisations/${oid}`] = organisation;
updatedData[`/org_shift_info/${oid}`] = orgShiftInfo;
rootRef.update(updatedData, (err) => {
if (err) {
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ error: true })
.then(() => {
console.log(`error adding organisation for ${uid}: `, err);
return reportError(err, { uid });
});
}
console.log('Create org wrote succesfully: ', Date.now());
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ success: true });
});
}
I understand the 'cold start' thing but I think something is seriously wrong that it's taking 25 seconds. The logs don't return any error and are as so:
Is there some deeper way I can debug this to try and figure out why it's taking so long? It's unusable at the moment. Thanks a lot.
Solved:
Sorry,
I misunderstood the API a bit. I should have watched the promise video first!
I needed to put
return rootRef.update...
instead of
rootRef.update...