I've been trying to understand async/await and Task in NodeJS. I have implemented I need to update db when all process executed.
Below is code samples
const download = require('image-downloader');
const imageDownload = (imgurl) =>{
console.log("inside imageDownload ", new Date())
return new Promise(async(resolve) => {
let imgpath = '/mount/imageDownload/';
if (!fs.existsSync(imgpath)) {
fs.mkdirSync(imgpath, {
recursive: true
});
}
const options = {
url: imgurl,
dest: imgpath
};
console.log("before download image " + assetId + " ", new Date())
download.image(options)
.then(({ filename }) => {
resolve({"error":false, "url":filename});
})
.catch((err) => {
console.log(err);
resolve({"error":true,"message":err});
});
});
}
(async () => {
let downloadUrl1 = "";
let downloadUrl2 = "";
let downloadUrlFirst = await imageDownload("https://dummyimage.com/300.png/09f/fff");
if (typeof downloadUrlFirst.url != 'undefined' && downloadUrlFirst.url != null) {
downloadUrl1 = downloadUrlFirst.url;
}
let downloadUrlSecond = await imageDownload("https://dummyimage.com/300.png/09f/fff");
if (typeof downloadUrlSecond.url != 'undefined' && downloadUrlSecond.url != null) {
downloadUrl2 = downloadUrlSecond.url;
}
if (downloadUrl1 != '' || downloadUrl2 !='' ) {
let updateImagePayload = {}
if (portrait_mounturl != '') {
updateImagePayload.downloadUrl1 = downloadUrl1;
}
if (landscape_mounturl != '') {
updateImagePayload.downloadUrl2 = downloadUrl2;
}
updateImagePayload.modified_date_time = new Date().toISOString();
db.collection("images").findOneAndUpdate({_id:ObjectId(assetId)}, { $set: updateImagePayload }, (err, update) => {
if (err) {
resolve({"error":true,"message": err});
} else {
let resolveStatus = false;
let resolveMessage = "Successfully updated image";
resolve({"error":resolveStatus,"message":resolveMessage});
}
});
}
})();
I need to update if both has async proccess completed.
Note: Image will be optional. Means either 1 image or both
Getting High Memory and CPU utilization
Any help understanding this would be much appreciated.
I am having a extension where it can notify about new items added to RSS feed reader
All works in v2, but v3 I am unable to load jquery into service worker, since chrome doesnt allow.
As a workaround I have added as module
"background": {
"service_worker": "js/background.js","type": "module"
},
But still its a issue and says ReferenceError: $ is not defined
at Object.parseFeed
Or is there a way I can tweak my code to read xml without jquery?
import $ as module from 'js/jquery-2.1.4.min.js';
var Storage = (function() {
return {
getFeeds: function(callback, returnArray, sortArray) {
returnArray = typeof returnArray !== 'undefined' ? returnArray : true;
sortArray = typeof sortArray !== 'undefined' ? sortArray : true;
chrome.storage.sync.get(function(dataObject) {
var result = dataObject;
if (returnArray) {
var feedArray = this.parseDataObjectIntoArray(dataObject);
result = sortArray ? this.sortArrayOfObjects(feedArray, 'position') : feedArray;
} else {
delete result['RssR:Settings'];
}
callback(result)
}.bind(this));
},
getFeedByUrl: function(feedUrl, callback) {
chrome.storage.sync.get(feedUrl, function(feedData) {
callback(feedData[feedUrl]);
});
},
removeFeedByUrl: function(feedUrl) {
chrome.storage.sync.remove(feedUrl);
},
saveFeedData: function(feedData) {
var saveFeed = {};
saveFeed[feedData.url] = feedData;
this.setDataObject(saveFeed);
},
parseDataObjectIntoArray: function(object) {
var array = [];
Object.keys(object).forEach(function(objectKey) {
if (objectKey.indexOf('RssR:Settings') !== 0) {
array.push(object[objectKey]);
}
});
return array;
},
sortArrayOfObjects: function(array, sortKey) {
array.sort(function(a, b) {
if (typeof a[sortKey] === 'undefined') {
return true;
} else if (typeof b[sortKey] === 'undefined') {
return false;
}
return a[sortKey] - b[sortKey];
});
return array;
},
clearAllData: function() {
chrome.storage.sync.clear();
},
setDataObject: function(dataObject) {
chrome.storage.sync.set(dataObject);
}
}
}());
Array.prototype.max = function() {
return Math.max.apply(null, this);
};
function msToTime(ms) {
let seconds = (ms / 1000).toFixed(1);
let minutes = (ms / (1000 * 60)).toFixed(1);
let hours = (ms / (1000 * 60 * 60)).toFixed(1);
let days = (ms / (1000 * 60 * 60 * 24)).toFixed(1);
if (seconds < 60) return seconds + " Sec";
else if (minutes < 60) return minutes + " Min";
else if (hours < 24) return hours + " Hrs";
else return days + " Days"
}
var FeedService = (function() {
return {
downloadFeed: function(feed) {
if (!feed.url) {
return;
}
console.log(feed.url)
fetch(feed.url)
.then(response => response.text())
.then(xmlString => {
console.log(xmlString)
this.parseFeed(xmlString, feed);
}
)
.then(data => console.log(data))
},
parseFeed: function(rawData, existingFeedData) {
var newFeedData = {};
var $feedEntries = $(rawData).find("entry").length > 0 ? $(rawData).find("entry") : $(rawData).find('item');
var lastUpdate = Date.now()
console.log("Now = " + lastUpdate)
lastUpdate = (existingFeedData && existingFeedData.lastUpdate) || Date.now()
console.log("Last Update = " + lastUpdate)
console.log("existingFeedData = " + JSON.stringify(existingFeedData))
pubDates = [];
$feedEntries.each(function(index, elm) {
pubDates.push(new Date($(elm).find("pubDate").text()).getTime())
if (lastUpdate < new Date($(elm).find("pubDate").text()).getTime()) {
console.log($(elm).find("title").text().substring(0, 20));
chrome.notifications.create(
$(elm).find("link").text(), {
type: "basic",
iconUrl: "../icons/254.png",
title: $(elm).find("title").text(),
message: "(" + msToTime((Date.now() - new Date($(elm).find("pubDate").text()).getTime())) + ") ",
requireInteraction: true
},
function() {}
);
}
}.bind(this));
console.log("Max date = " + Math.max.apply(null, pubDates))
existingFeedData.lastUpdate = Math.max.apply(null, pubDates);
Storage.saveFeedData(existingFeedData);
},
getUrlForFeed: function($feed, feedSettings) {
if (feedSettings.linkType && $feed.find(feedSettings.linkType).length > 0) {
return $feed.find(feedSettings.linkType).text();
} else if ($feed.find('link').length == 1) {
return $feed.find('link').text();
} else {
if ($feed.find('link[rel="shorturl"]').length > 0) {
return $feed.find('link[rel="shorturl"]').attr('href');
} else if ($feed.find('link[rel="alternate"]').length > 0) {
return $feed.find('link[rel="alternate"]').attr('href');
} else if ($feed.find('link[rel="related"]').length > 0) {
return $feed.find('link[rel="related"]').attr('href');
} else {
return $feed.find('link').first();
}
}
},
saveFeed: function(existingFeedData, newFeedData) {
;
},
updateReadStatusOfNewItems: function(oldItems, newItems) {
if (typeof oldItems === 'undefined' || Object.keys(oldItems).length == 0 || Object.keys(newItems).length == 0) {
return newItems;
}
Object.keys(newItems).forEach(function(url) {
if (oldItems[url]) {
newItems[url].unread = oldItems[url].unread;
}
});
return newItems;
},
setNewBadge: function() {
chrome.browserAction.setBadgeText({
text: 'NEW'
});
}
};
}());
chrome.alarms.onAlarm.addListener(function(alarm) {
if (alarm.name == 'updateRssFeeds') {
console.log("Called!")
updateRssFeeds();
}
});
chrome.notifications.onClicked.addListener(function(notificationId) {
chrome.tabs.create({
url: notificationId
});
});
chrome.runtime.onInstalled.addListener(onInstall);
function onInstall() {
chrome.runtime.openOptionsPage();
chrome.alarms.create('updateRssFeeds', {
when: 0,
periodInMinutes: 1
});
}
function updateRssFeeds() {
Storage.getFeeds(function(feeds) {
feeds.forEach(function(feed) {
FeedService.downloadFeed(feed);
});
}, true, false)
}
if (fs.existsSync('tmp/cache.txt')) {
fs.readFile("tmp/cache.txt", function (err, data) {
if (data != "" || data != "[]") {
jdata = JSON.parse(data);
if (
jdata[jdata.length - 1].substring(4, 8) ==
new Date().getFullYear() + 543
) {
year = new Date().getFullYear() + 542;
console.log("yes this year");
}
jdata.forEach(function (value, i) {
if (
value.substring(4, 8) ==
new Date().getFullYear() + 543
) {
countloveme--;
}
});
jdata.splice(countloveme);
}
});
}
my code is running but
code is finished before fs.readFile inside ifelse have finish
i don't how to add await in fs.readFile or anyway to this code is working
As written in the comments, it would be a better choice to use fs,readFileSync
And when you use Array.forEach() you are starting a new function that is running synchronously.
I've cleared your code maybe this can help you
if (fs.existsSync('tmp/cache.txt')) {
try {
const data = fs.readFileSync("tmp/cache.txt");
if (!data || data != "" || data != "[]")
throw new Error('tmp/cache.txt file is empty');
const jdata = JSON.parse(data);
// More clear to use variables in the if elses
const arg1 = jdata[jdata.length - 1].substring(4, 8)
const arg2 = new Date().getFullYear() + 543;
if (arg1 === arg2) {
// You don't use this date anywhere?
new Date().getFullYear() + 542;
console.log("yes this year");
}
for (let dataChunk of jdata) {
if (
dataChunk.substring(4, 8) ==
new Date().getFullYear() + 543
) {
countloveme--;
}
}
jdata.splice(countloveme);
} catch (error) {
console.error(error.message);
}
}
I am using the validate-slack-request package to validate my incoming slack requests are from slack. This works fine for Slash commands and Interactive Components (buttons, etc.). However it is not working for the Events API
I noticed that the POST request body has a different format for the Events API. There's no payload. But it's not clear to me what slack is giving me to use to verify. Code is below
//This WORKS
app.post("/interactiveCommand", async (req, res) => {
const legit = validateSlackRequest(process.env.SLACK_SIGNING_SECRET, req, false);
if (!legit) {
console.log("UNAUTHORIZED ACCESS ", req.headers, req.body);
return res.status(403).send("Unauthorized");
}
await interactiveCommand(...);
return;
});
//This does NOT WORK
app.post("/slackEvents", parser, json, async (req, res) => {
const legit = validateSlackRequest(process.env.SLACK_SIGNING_SECRET, req, false);
if (!legit) {
console.log("UNAUTHORIZED ACCESS ", req.headers, req.body);
res.status(403).send("Unauthorized");
} else {
try {
switch (req.body.event.type) {
case "message":
await handleMessageEvent(...);
break;
case "app_home_opened":
res.status(200).send();
await updateUserHomePage(...);
break;
default:
res.status(200).send();
return;
}
} catch(e) {
console.log("Error with event handling! ", e);
}
}
});
const crypto = require('crypto')
const querystring = require('querystring')
// Adhering to RFC 3986
// Inspired from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
function fixedEncodeURIComponent (str) {
return str.replace(/[!'()*~]/g, function (c) {
return '%' + c.charCodeAt(0).toString(16).toUpperCase()
})
}
/**
* Validate incoming Slack request
*
* #param {string} slackAppSigningSecret - Slack application signing secret
* #param {object} httpReq - Express request object
* #param {boolean} [logging=false] - Enable logging to console
*
* #returns {boolean} Result of vlaidation
*/
function validateSlackRequest (slackAppSigningSecret, httpReq, logging) {
logging = logging || false
if (typeof logging !== 'boolean') {
throw new Error('Invalid type for logging. Provided ' + typeof logging + ', expected boolean')
}
if (!slackAppSigningSecret || typeof slackAppSigningSecret !== 'string' || slackAppSigningSecret === '') {
throw new Error('Invalid slack app signing secret')
}
const xSlackRequestTimeStamp = httpReq.get('X-Slack-Request-Timestamp')
const SlackSignature = httpReq.get('X-Slack-Signature')
const bodyPayload = fixedEncodeURIComponent(querystring.stringify(httpReq.body).replace(/%20/g, '+')) // Fix for #1
if (!(xSlackRequestTimeStamp && SlackSignature && bodyPayload)) {
if (logging) { console.log('Missing part in Slack\'s request') }
return false
}
const baseString = 'v0:' + xSlackRequestTimeStamp + ':' + bodyPayload
const hash = 'v0=' + crypto.createHmac('sha256', slackAppSigningSecret)
.update(baseString)
.digest('hex')
if (logging) {
console.log('Slack verifcation:\n Request body: ' + bodyPayload + '\n Calculated Hash: ' + hash + '\n Slack-Signature: ' + SlackSignature)
}
return (SlackSignature === hash)
}
This is how I got it to work, it was a little bit of trial and error and I make no promises. Basically, if I was validating an Event rather than a Slash Command or an Interactive Component, I would pass type="Event" to the validation function. The only change is how I construct the payload from the incoming request
export function validateSlackRequest(
slackAppSigningSecret,
httpReq,
logging,
type = ""
) {
logging = logging || false;
if (typeof logging !== "boolean") {
throw new Error(
"Invalid type for logging. Provided " +
typeof logging +
", expected boolean"
);
}
if (
!slackAppSigningSecret ||
typeof slackAppSigningSecret !== "string" ||
slackAppSigningSecret === ""
) {
throw new Error("Invalid slack app signing secret");
}
const xSlackRequestTimeStamp = httpReq.get("X-Slack-Request-Timestamp");
const SlackSignature = httpReq.get("X-Slack-Signature");
let bodyPayload;
if (type === "Event") {
bodyPayload = (httpReq as any).rawBody;
} else {
bodyPayload = fixedEncodeURIComponent(
querystring.stringify(httpReq.body).replace(/%20/g, "+")
); // Fix for #1
}
if (!(xSlackRequestTimeStamp && SlackSignature && bodyPayload)) {
if (logging) {
console.log("Missing part in Slack's request");
}
return false;
}
const baseString = "v0:" + xSlackRequestTimeStamp + ":" + bodyPayload;
const hash =
"v0=" +
crypto
.createHmac("sha256", slackAppSigningSecret)
.update(baseString)
.digest("hex");
if (logging) {
console.log(
"Slack verification:\nTimestamp: " +
xSlackRequestTimeStamp +
"\n Request body: " +
bodyPayload +
"\n Calculated Hash: " +
hash +
"\n Slack-Signature: " +
SlackSignature
);
}
return SlackSignature === hash;
}
I convert text payloads into json with this:
IFS=$'\n'
if [ "$REQUEST_METHOD" = "POST" ]; then
if [ "$CONTENT_LENGTH" -gt 0 ]; then
cat - > /tmp/file.txt
fi
fi
cat /tmp/file.txt | sed -e "s/^payload=//g" | perl -pe 's/\%(\w\w)/chr hex $1/ge' > /tmp/file.json
I am using GEOTAB apis to sync vehicle data to my DB.In my software user can register and add geotab account to sync.for sync purpose i have used node js wrapper.With 3 minutes delay this process calls regularly.
I am getting the error "Global login limit exceeded"
var syncUsers = function () {
var i;
async.forEach(geotabUsers, function (geoUser, callback) {
//get index
i = geotabUsers.indexOf(geoUser);
if (apiInstanse[geotabUsers[i].userId] === undefined)
{
apiInstanse[geotabUsers[i].userId] = new API(geotabUsers[i].apiUsername, geotabUsers[i].apiPassword, geotabUsers[i].apiDatabase, js_lang.GEOTAB_SERVER);
}
//sync status data
syncStatusData(apiInstanse[geotabUsers[i].userId], i, userInfo[geotabUsers[i].userId].statusDataFromVersion, geotabUsers[i].userId, userInfo[geotabUsers[i].userId].currentCompany, geotabUsers[i].apiUsername, geotabUsers[i].apiPassword, geotabUsers[i].apiDatabase, userInfo[geotabUsers[i].userId].currentPressureUnit, userInfo[geotabUsers[i].userId].currentDateFormat, userInfo[geotabUsers[i].userId].currentTimeFormat, userInfo[geotabUsers[i].userId].currentTemperatureUnit);
//sync fault data
syncFaultData(apiInstanse[geotabUsers[i].userId], i, userInfo[geotabUsers[i].userId].faultDataFromVersion, geotabUsers[i].userId, userInfo[geotabUsers[i].userId].currentCompany, geotabUsers[i].apiUsername, geotabUsers[i].apiPassword, geotabUsers[i].apiDatabase,function(){
callback();
});
},function(err){
if(err)
{
console.log('done errro:',err);
}
console.log('sync done');
continueSync();
});
Function to sync status data:
var syncStatusData = function (api, i, fromVersion, userId, currentCompany, apiUsername, apiPassword, apiDatabase, currentPressureUnit, currentDateFormat, currentTimeFormat, currentTemperatureUnits)
{
try {
api.call(js_lang.GEOTAB_GETFEED_METHOD, {
typeName: js_lang.GEOTAB_STATUS_DATA,
resultsLimit: js_lang.GEOTAB_API_LIMIT,
fromVersion: fromVersion
}, function (err, data) {
if (err) {
console.log('api Call Error:', userId);
console.log('apiUsername:', apiUsername);
console.log('apiPassword:', apiPassword);
console.log('apiDatabase:', apiDatabase);
console.log('Error', err);
//apiInstanse[userId] = new API(apiUsername, apiPassword, apiDatabase, js_lang.GEOTAB_SERVER);
//throw err;
}
else {
var insertStatus = [];
var sql = "INSERT INTO " + js_lang.TABLE_STATUS_DATA + " (companyId,dateTime,deviceId ,diagnosticId,value,version,uniqueId,userId,unitOfMeasure ) VALUES ?";
//iterate data
if (data.data !== undefined)
{
for (var key in data.data) {
if (diagnosticList[data.data[key].diagnostic.id] === undefined)
{
continue;
}
var normalDate = FUNCTION_CLASS.getNormalDateFromUTC(data.data[key].dateTime);
//prepare data to insert
var insertRow = [
currentCompany,
normalDate,
data.data[key].device.id,
data.data[key].diagnostic.id,
data.data[key].data,
data.data[key].version,
data.data[key].id,
userId,
diagnosticList[data.data[key].diagnostic.id].unitOfMeasure
];
insertStatus.push(insertRow);
}
}
if (insertStatus.length > 0)
{
connection.query(sql, [insertStatus], function (err) {
if (err)
{
throw err;
}
// console.log('toversion:', data.toVersion);
console.log('insert:userId:', userId);
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET statusDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
});
}
else {
console.log('update user:', userId);
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET statusDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
}
}
if ((geotabUsers.length - 1) === i)
{
console.log('loop ended');
syncStatusDone = 1;
// continueSync();
}
});
}
catch (e) {
continueSync();
}
}
Function to sync Fault data:
var syncFaultData = function (api, i, fromVersion, userId, currentCompany, apiUsername, apiPassword, apiDatabase,callback)
{
try {
api.call(js_lang.GEOTAB_GETFEED_METHOD, {
typeName: js_lang.GEOTAB_FAULT_DATA,
resultsLimit: js_lang.GEOTAB_API_LIMIT,
fromVersion: fromVersion
}, function (err, data) {
if (err) {
console.log('api faultData Call Error:', userId);
console.log('apiUsername:', apiUsername);
console.log('apiPassword:', apiPassword);
console.log('apiDatabase:', apiDatabase);
console.log('Error', err);
//apiInstanse[userId] = new API(apiUsername, apiPassword, apiDatabase, js_lang.GEOTAB_SERVER);
//throw err;
}
else {
var insertStatus = [];
var sql = "INSERT INTO " + js_lang.TABLE_FAULT_DATA + " (amberWarningLamp,controllerId,count,dateTime,deviceId ,diagnosticId,dismissDateTime,dismissUser,failureModeId,faultLampState,faultState,flashCode,uniqueId,malfunctionLamp,protectWarningLamp,redStopLamp,version,companyId,userId,deleted) VALUES ?";
//iterate data
if (data.data !== undefined)
{
for (var key in data.data) {
if (diagnosticList[data.data[key].diagnostic.id] == undefined)
{
continue;
}
var normalDate = FUNCTION_CLASS.getNormalDateFromUTC(data.data[key].dateTime);
var thisDate = moment(new Date(data.data[key].dateTime)).tz(js_lang.TIMEZONE).format(js_lang.DATETIME_FORMAT);
var thisDayDate = moment(new Date(data.data[key].dateTime)).tz(js_lang.TIMEZONE).format(js_lang.DATE_FORMAT);
var controllerId = '';
if (data.data[key].controller !== undefined && data.data[key].controller.id !== undefined)
{
controllerId = data.data[key].controller.id;
}
else if (data.data[key].controller !== undefined)
{
controllerId = data.data[key].controller;
}
var faultcount = null;
if (data.data[key].count !== undefined)
{
faultcount = parseInt(data.data[key].count);
}
var dismissDateTime = '';
if (data.data[key].dismissDateTime !== undefined)
{
dismissDateTime = moment(new Date(data.data[key].dismissDateTime)).tz(js_lang.TIMEZONE).format(js_lang.DATETIME_FORMAT);
}
var dismissUser = '';
if (data.data[key].dismissUser !== undefined)
{
dismissUser = data.data[key].dismissUser;
}
var failureModeId = '';
if (data.data[key].failureModeId !== undefined && data.data[key].failureModeId.id !== undefined)
{
failureModeId = data.data[key].failureModeId.id;
}
var faultLampState = '';
if (data.data[key].faultLampState !== undefined)
{
faultLampState = data.data[key].faultLampState;
}
var faultState = data.data[key].faultState;
var flashCode = '';
if (data.data[key].flashCode !== undefined)
{
flashCode = data.data[key].flashCode;
}
var malfunctionLamp = data.data[key].malfunctionLamp;
var protectWarningLamp = data.data[key].protectWarningLamp;
var redStopLamp = data.data[key].redStopLamp;
var version = '';
if (data.data[key].version !== undefined)
{
version = data.data[key].version;
}
//prepare data to insert
var insertRow = [
data.data[key].amberWarningLamp,
controllerId,
faultcount,
normalDate,
data.data[key].device.id,
data.data[key].diagnostic.id,
dismissDateTime,
dismissUser,
failureModeId,
faultLampState,
faultState,
flashCode,
data.data[key].id,
malfunctionLamp,
protectWarningLamp,
redStopLamp,
version,
currentCompany,
userId,
0
];
insertStatus.push(insertRow);
}
}
// console.log(insertStatus);
if (insertStatus.length > 0)
{
connection.query(sql, [insertStatus], function (err) {
if (err)
{
throw err;
}
// console.log('toversion:', data.toVersion);
console.log('insert faultData:userId:', userId);
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET faultDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
callback();
});
}
else {
console.log('update faultData user:', userId);
//test code
//sendEmail(['ankkubosstest#gmail.com','webdeveloper#gmail.com'], 'emailSubject', 'emailText', '<div>html code<div>');
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET faultDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
callback();
}
}
if ((geotabUsers.length - 1) === i)
{
console.log('loop ended');
syncFaultDone = 1;
}
});
}
catch (e) {
continueSync();
}
}