Incorrect gmail parsing with node-imap and mailparser - node.js

I'm writing a Node.js script that retrieves unread emails from my gmail inbox using node-imap, parses them with mailparser, and then does some work with the parsed emails. I'm running into a problem where the raw email being received doesn't seem to be parsed correctly by mailparser. I'm not sure if I've done something wrong in calling node-imap or mailparser, or if the email itself is in a bad format for some reason. I've included the code I'm running as well as the output produced.
var Imap = require("imap"),
MailParser = require("mailparser").MailParser,
Promise = require("bluebird"),
request = require("request-promise").defaults({jar: true}),
log = require("winston"),
_ = require("underscore"),
config = require("config").jobs;
var logConfig = _.clone(config.logConfig.email);
if (process.env.LOG_DIR) {
logConfig.filename = process.env.LOG_DIR + "/" + logConfig.filename;
}
log.add(log.transports.File, logConfig || config.logConfig);
Promise.longStackTraces();
var imap = new Imap(config.emailConfig);
Promise.promisifyAll(imap);
imap.once("ready", execute);
imap.once("error", function (err) {
log.error("Connection error: " + err.stack);
});
imap.connect();
function execute() {
imap.openBoxAsync("INBOX", false)
.then(function () {
return imap.searchAsync(["UNSEEN"]);
})
.then(function (results) {
var f = imap.fetch(results, {bodies: ["HEADER.FIELDS (FROM SUBJECT)", "TEXT"]});
f.on("message", processMessage);
f.once("error", function (err) {
return Promise.reject(err);
});
f.once("end", function () {
log.info("Done fetching all unseen messages.");
imap.end();
});
})
.catch(function (err) {
log.error("Error fetching messages: " + err.stack);
imap.end();
});
}
function processMessage(msg, seqno) {
log.info("Processing msg #" + seqno);
var parser = new MailParser();
parser.on("headers", function(headers) {
log.info("Header: " + JSON.stringify(headers));
});
parser.on("end", function(msg) {
log.info("From: " + msg.from);
log.info("Subject: " + msg.subject);
log.info("Text: " + msg.text);
log.info("Html: " + msg.html);
});
msg.on("body", function (stream) {
stream.on("data", function (chunk) {
parser.write(chunk.toString("utf8"));
});
});
msg.once("end", function () {
log.info("Finished msg #" + seqno);
parser.end();
});
}
Output:
info: Processing msg #1
info: Finished msg #1
info: Done fetching all unseen messages.
info: Header: {"--001a11339690da942a051d866a04":"","content-type":"text/plain; charset=UTF-8"}
info: From: undefined
info: Subject: undefined
info: Text: Test app mail body!
- Jared
--001a11339690da942a051d866a04
Content-Type: text/html; charset=UTF-8
<div dir="ltr"><div>Test app mail body!<br><br></div>- Jared<br></div>
--001a11339690da942a051d866a04--
Subject: Here is a test appmail email
From: Jared Wolinsky <jared.wolinsky#gmail.com>
info: Html: undefined

You're fetching the raw body of the email. mailparser is expecting the full email, headers and body both. To fetch that instead, just specify an empty string '' instead of 'TEXT'.

Related

SendGrid Multiple Emails to Multiple Recipients Node JS from Firebase Firestore

I am using SendGrid and Firebase Functions to send multiple emails to multiple recipients. The code that I am using works correctly when sending to a test list of 4 email addresses, but does not work when trying to send to 4,000 email addresses. There is also no error message from SendGrid.
This code also works and returns the list of email addresses printed in the console if the SendGrid block of code is commented out.
Do you know what could be going wrong?
Thank you
exports.adminSendGroupMessage = functions.region('europe-west2').https.onCall((data, context) => {
const emailHTMLData = emailHTMLData;
var emailDataArray = [];
//Fetch contacts list
let testContactsRef = db.collection('contacts-list');
return testContactsRef.get().then(snapshot => {
snapshot.forEach(doc => {
// console.log(doc.id, '=>', doc.data());
console.log("Fetched contact with ID: " + doc.id);
//Extract contact data
const firstName = doc.data().name || "";
const surname = doc.data().surname || "";
const emailAddress = doc.data().emailAddress;
var emailData = {
to: emailAddress,
from: 'fromEmail#email.com',
subject: messageSubject,
text: 'Email for ' + firstName,
html: emailHTMLData,
customArgs: {
ref: 'msg-ref'
},
}
//Add new email data to the array
emailDataArray.push(emailData);
});
return Promise.all(emailDataArray).then(results => {
//Send emails with all data once contact fetch complete
console.log("Success fetching contacts - send emails.");
sendGridGroupMessages(emailDataArray);
return { success : true, message: "Success sending emails" };
})
});
function sendGridGroupMessages(emailDataArray) {
console.log('Send emails to group function with data: ' + emailDataArray.length);
var i,j, splitArray,chunk = 998;
for (i=0,j=emailDataArray.length; i<j; i+=chunk) {
splitArray = emailDataArray.slice(i,i+chunk);
// do whatever
//Send emails
sgMail.send(splitArray, (error, result) => {
if (error) {
//Do something with the error
console.log("Error sending group emails: " + error);
// throw new functions.https.HttpsError('internal', 'There was an error sending the group emails - ' + error.message + ' (' + error.code + ')');
} else {
//Celebrate
console.log("Success sending group emials: " + JSON.stringify(result));
// return { success : true };
}
});
}
You must post the error log
If no errors it means they sent
but check SendGrid activities to see what happened to these emails

How to load all sql output data into html table

It is a simple email blast program which I build
I stuck in receiving data in a loop.
I want to print all the data from the result but i can able to print last data in the result.
Here is my code:
var mailer = require("nodemailer"),
config = require('./config'),
redshiftClient = require('./redshift.js'),
sql = require('./main');
var smtpTransport = mailer.createTransport({
service: "Outlook",
auth: {
user: config.user,
pass: config.passwd
}
});
console.log("Application started at : "+ new Date().getTime())
sql.log();
redshiftClient.query(getQuery())
.then(function (data) {
console.log("Query executed at " + new Date().getTime())
var element = data.rows[index];
for (let index = 0; index < data.rows.length; index++) {
var element = data.rows[index];
var mail = {
from: 'xxxxxxx#outlook.com',
to: 'xxxxx#gmail.co.in',
subject: `Daily AWS validation check for the ${new Date().getTime()}`,
html: `<h1> Hello Team, Here is the hourly data pull</h1> <br> <table><tr> <th>Hoursdata</th> <th>Book_Counts</th> <th>Search_Counts</th> <th>Hit_Counts</th> <tr><td>${element.run_hour}</td> <td>${element.book_counts}</td> <td>${element.search_counts}</td> <td>${element.hit_counts}</td></tr> </table>`
}
smtpTransport.sendMail(mail, function (error, response) {
if (error) {
console.log(error);
} else {
console.log("Email sent");
}
smtpTransport.close();
});
}
})
.catch(function (err) {
console.error(err);
});
redshiftClient.close();
My Output for the code: (It gave the last value)
Hoursdata Book_Counts Search_Counts Hit_Counts
2 2131 5645654 4355
Expected output: (I need all the value)
Hoursdata Book_Counts Search_Counts Hit_Counts
1 23425 457474 2534
2 2131 5645654 4355
What you are trying to do is, iterate over all the rows and for each row send a message, while you should be creating a mail message once with all the details in html. As suggested in my previous answer, use some npm package to create html and pass it to html while creating message and after that send email.

AWS Lambda Node.js - get output from async call (ec2.describeImages)

I am unable to get data from ec2.describeImages.
I have a function that wraps the call to ec2.describeImages:
async function GetLatestAMI (event, context, latestAMI) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var responseStatus = "FAILED";
var responseData = {};
var osBaseName = osNameToPattern[event.ResourceProperties.OSName];
console.log("OS: " + event.ResourceProperties.OSName + " -> " + osBaseName);
//var ec2 = new aws.EC2({region: event.ResourceProperties.Region});
var describeImagesParams = {
Filters: [{ Name: "name", Values: [osBaseName]}],
Owners: ["amazon"]
};
console.log( "Calling describeImages...");
// Get the available AMIs for the specified Windows version.
await ec2.describeImages(describeImagesParams, function(err, describeImagesResult) {
if (err) {
responseData = {Error: "DescribeImages call failed"};
console.log(responseData.Error + ":\n", err);
}
else {
console.log( "Got a response back from the server");
var images = describeImagesResult.Images;
console.log( "Got " + images.length + " images back" );
// Sort the images by descending creation date order so the
// most recent image is first in the array.
images.sort(function(x,y){
return x.CreationDate < y.CreationDate;
});
for (var imageIndex = 0; imageIndex < images.length; imageIndex++) {
responseStatus = "SUCCESS";
responseData["Id"] = images[imageIndex].ImageId;
latestAMI = responseData["Id"];
responseData["Name"] = images[imageIndex].Name;
console.log( "Found: " + images[imageIndex].Name + ", " + images[imageIndex].ImageId);
context.done(null, "success");
//break;
return latestAMI;
}
}
}).promise();}
I call this function on the Lambda handler:
exports.handler = async function(event, context, callback) {
console.log("Hello!")
var latestAMI = "";
var testResult = await GetLatestAMI(event, context, latestAMI);
console.log("Test result: " + latestAMI);
console.log("Test result: " + testResult);
}
I cannot get any output from GetLatestAMI(). I have tried using a return clause inside the function as you can see and even passing a variable as an argument and edit the value of that value inside the function with no luck.
Here is the output from the console:
Response:
"success"
Request ID:
"8f2dbba4-02d1-11e9-a504-83ed96cb467a"
Function Logs:
START RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a Version: $LATEST
2018-12-18T14:31:08.858Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Hello!
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a REQUEST RECEIVED:
{"ResourceProperties":{"OSName":"Windows Server 2016 (64-bit)"}}
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a OS: Windows Server 2016 (64-bit) -> Windows_Server-2016-English-Full-Base-*
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Calling describeImages...
2018-12-18T14:31:10.556Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Got a response back from the server
2018-12-18T14:31:10.573Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Got 7 images back
2018-12-18T14:31:10.593Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Found: Windows_Server-2016-English-Full-Base-2018.12.12, ami-06a27ce600d784c71
END RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a
REPORT RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a Duration: 1776.51 ms Billed Duration: 1800 ms Memory Size: 128 MB Max Memory Used: 39 MB
I'm probably misunderstanding or missing something. Hopefully it would be obvious to someone with experience on AWS Lambda and Node.js!
I have written the below lambda for you to list down the ec2 instances. And have checked in nodejs8.10. It is working as expected.
Some of the issues your code has.
a) return statement inside loop.
b) context.done(null, "success"); done at wrong place that to inside loop
c) You are not resolving your promises anywhere.
Hope the Below code will give you an idea.
const AWS = require('aws-sdk');
module.exports.api = async (event, context, callback) => {
var response = await getInstances();
return {
statusCode: '200',
body: JSON.stringify({response}),
headers: {'Content-Type': 'application/json'}
}
}
async function getInstances() {
var ec2 = new AWS.EC2();
return new Promise(function (resolve, reject) {
ec2.describeInstances(function (err, data) {
if (err) reject(err);
else resolve(data);
});
});
}

node-irc with Twitch.TV duplicate message issue

My node-irc bot is sometimes repeating what it says in the chat. Sometimes it works perfectly; other times it repeats. I've looked all through my code and I cant find any recursion going on.
Connection function (with my listeners)
//connects the bot to the channel
this.connectToChannel = function(callback) {
console.log("Connection");
botInstance.connect();
botInstance.join(currentChannel, function(){
botInstance.say(currentChannel, initalConnectionMessage);
console.log("Joined");
callback();
});
botInstance.addListener("message", function(nick, to, text, message) {
if (text == lastText){
return; // you can i've tried to avoid this issue
}
bannedWords.forEach(function(item,index){
if (text.includes(item)){
parent.sendMessage(nick + " > You're not allowed to use that type of language.");
parent.sendMessage("/timeout " + nick + " 10");
}
});
commands.forEach(function(item, index){
var splitData = text.split(' ');
if (splitData[0] == item.getExecutor()) {
//running this command
var response = item.getResponse();
response = response.replace("%user%", nick);
response = response.replace("%bot_username%", config.botName);
parent.sendMessage(response);
}
});
});
}
Bot Creation(This is being called before the connection)
this.createBot = function(channel) {
currentChannel = channel;
var irc = require('irc');
botInstance = new irc.Client(config.server, config.botName, {
channels: [currentChannel + " " + config.oAuth],
userName: config.botName,
password: config.oAuth
});
}
Bot Sending Messages
this.sendMessage = function(msg){
if(botInstance != null) {
botInstance.say(currentChannel, msg);
lastMessage = msg;
}

Mocking email function in nodejs

I've got a mailer function I've built and trying to shore up the coverage. Trying to test parts of it have proven tricky, specifically this mailer.smtpTransport.sendMail
var nodemailer = require('nodemailer')
var mailer = {}
mailer.smtpTransport = nodemailer.createTransport('SMTP', {
'service': 'Gmail',
'auth': {
'XOAuth2': {
'user': 'test#test.com',
'clientId': 'googleClientID',
'clientSecret': 'superSekrit',
'refreshToken': '1/refreshYoSelf'
}
}
})
var mailOptions = {
from: 'Some Admin <test#tester.com>',
}
mailer.verify = function(email, hash) {
var emailhtml = 'Welcome to TestCo. Click this '+hash+''
var emailtxt = 'Welcome to TestCo. This is your hash: '+hash
mailOptions.to = email
mailOptions.subject = 'Welcome to TestCo!'
mailOptions.html = emailhtml
mailOptions.text = emailtxt
mailer.smtpTransport.sendMail(mailOptions, function(error, response){
if(error) {
console.log(error)
} else {
console.log('Message sent: '+response.message)
}
})
}
I'm unsure of how to go about testing, specifically ensuring that my mailer.smtpTransport.sendMail function is passing the correct parameters without actually sending the email. I'm trying to use https://github.com/whatser/mock-nodemailer/tree/master, but I'm probably doing it wrong. Should I be mocking out the method?
var _ = require('lodash')
var should = require('should')
var nodemailer = require('nodemailer')
var mockMailer = require('./helpers/mock-nodemailer')
var transport = nodemailer.createTransport('SMTP', '')
var mailer = require('../../../server/lib/account/mailer')
describe('Mailer', function() {
describe('.verify()', function() {
it('sends a verify email with a hashto an address when invoked', function(done) {
var email ={
'to': 'dave#testco.com',
'html': 'Welcome to Testco. Click this bleh',
'text': 'Welcome to Testco. This is your hash: bleh',
'subject': 'Welcome to Testco!'
}
mockMailer.expectEmail(function(sentEmail) {
return _.isEqual(email, sentEmail)
}, done)
mailer.verify('dave#testco.com','bleh')
transport.sendMail(email, function() {})
})
})
You can use a 'Stub' transport layer on your test instead of SMTP.
var stubMailer = require("nodemailer").createTransport("Stub"),
options = {
from: "from#email.com",
to: "to#email.com",
text: "My Message!"
};
stubMailer.sendMail(options, function(err, response){
var message = response.message;
})
So, in that case, 'message' will be the email in text format. Something like this:
MIME-Version: 1.0
X-Mailer: Nodemailer (0.3.43; +http://www.nodemailer.com/)
Date: Fri, 25 Feb 2014 11:11:48 GMT
Message-Id: <123412341234.e23232#Nodemailer>
From: from#email.com
To: to#email.com
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: quoted-printable
My Message!
For more examples, take a look at nodemailer test suite:
https://github.com/andris9/Nodemailer/blob/master/test/nodemailer-test.js
You can directly mock the sendMail function but it's not obvious how to access it from the tests. A Mailer instance is returned when you create a transport so you need to directly import that class in to your test.
const Mailer = require('nodemailer/lib/mailer')
Then you can mock or stub the sendMail method on the prototype in the usual way. Using Jasmine, you can do it like this:
beforeEach(function () {
spyOn(Mailer.prototype, 'sendMail').and.callFake(function (mailOptions, cb) {
cb(null, true)
})
})
The callFake ensures that the sendMail's callback is still executed encase you need to test what happens next. You can easily simulate an error by passing a first argument to cb: cb(new Error('Email failed'))
Now that the mock is set up, you can check everything is working as intended:
expect(Mailer.prototype.sendMail).toHaveBeenCalled()
expectEmail simply hooks into the transport layer, and expects you to identify the email ( return true if this is the email you are expecting ) by looking at the sentEmail contents.
In this case, return sentEmail.to === 'dave#testco.com' should suffice.
Keep in mind however, this module was designed in an environment where tests are ran in a random order and concurrently. You should propably randomize your data heavily to prevent collisions and false positives. BTW we use something like: var to = Date.now().toString(36) + Faker.Internet.email();
This example works fine for me
======== myfile.js ========
// SOME CODE HERE
transporter.sendMail(mailOptions, (err, info) => {
// PROCESS RESULT HERE
});
======== myfile.spec.js (unit test file) ========
const sinon = require('sinon');
const nodemailer = require('nodemailer');
const sandbox = sinon.sandbox.create();
describe('XXX', () => {
afterEach(function() {
sandbox.restore();
});
it('XXX', done => {
const transport = {
sendMail: (data, callback) => {
const err = new Error('some error');
callback(err, null);
}
};
sandbox.stub(nodemailer, 'createTransport').returns(transport);
// CALL FUNCTION TO TEST
// EXPECT RESULT
});
});

Resources