What is wrong with this javascript regex string.replace? - node.js

Question

UPDATE: I've discovered this problem is because of the "stringified" output which contains no spaces or newlines and is different than the original JSON.
Stringified output:
{"eventType":"Delivery","mail":{"timestamp":"2021-08-09T20:41:51.515Z","source":"removed#email.com","sourceArn":"arn:aws:ses:us-east-1:1234567890:identity/removed.com","sendingAccountId":"1234567890","messageId":"1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b","destination":["removedl#email.com"],"headersTruncated":false,"headers":[{"name":"From","value":"removed#email.com"},{"name":"To","value":"removed#email.com"},{"name":"Subject","value":"Test"},{"name":"MIME-Version","value":"1.0"},{"name":"Content-Type","value":"text/plain; charset=UTF-8"},{"name":"Content-Transfer-Encoding","value":"7bit"}],"commonHeaders":{"from":["removed#email.com"],"to":["removed#email.com"],"messageId":"1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b","subject":"Test"},"tags":{"ses:operation":["SendEmail"],"ses:configuration-set":["GenericLog"],"ses:source-ip":["123.123.123.123"],"ses:from-domain":["email.com"],"ses:caller-identity":["user1"],"ses:outgoing-ip":["234.234.234.234"]}},"delivery":{"timestamp":"2021-08-09T20:41:52.880Z","processingTimeMillis":1365,"recipients":["removed#email.com"],"smtpResponse":"250 2.6.0 <1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b#email.amazonses.com> [InternalId=11231247472, Hostname=CO6PR12MB5396.namprd12.prod.outlook.com] 12090 bytes in 0.214, 55.135 KB/sec Queued mail for delivery","reportingMTA":"a48-108.smtp-out.amazonses.com"}}
ORIGINAL:
I want to remove the : from the key names and replace it with a _ in the following JSON. (In the tags section)
{
"eventType": "Delivery",
"mail": {
"timestamp": "2021-08-09T20:41:51.515Z",
"source": "removed#email.com",
"sourceArn": "arn:aws:ses:us-east-1:1234567890:identity/removed.com",
"sendingAccountId": "1234567890",
"messageId": "1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b",
"destination": [
"removedl#email.com"
],
"headersTruncated": false,
"headers": [
{
"name": "From",
"value": "removed#email.com"
},
{
"name": "To",
"value": "removed#email.com"
},
{
"name": "Subject",
"value": "Test"
},
{
"name": "MIME-Version",
"value": "1.0"
},
{
"name": "Content-Type",
"value": "text/plain; charset=UTF-8"
},
{
"name": "Content-Transfer-Encoding",
"value": "7bit"
}
],
"commonHeaders": {
"from": [
"removed#email.com"
],
"to": [
"removed#email.com"
],
"messageId": "1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b",
"subject": "Test"
},
"tags": {
"ses:operation": [
"SendEmail"
],
"ses:configuration-set": [
"GenericLog"
],
"ses:source-ip": [
"123.123.123.123"
],
"ses:from-domain": [
"email.com"
],
"ses:caller-identity": [
"user1"
],
"ses:outgoing-ip": [
"234.234.234.234"
]
}
},
"delivery": {
"timestamp": "2021-08-09T20:41:52.880Z",
"processingTimeMillis": 1365,
"recipients": [
"removed#email.com"
],
"smtpResponse": "250 2.6.0 <1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b#email.amazonses.com> [InternalId=11231247472, Hostname=CO6PR12MB5396.namprd12.prod.outlook.com] 12090 bytes in 0.214, 55.135 KB/sec Queued mail for delivery",
"reportingMTA": "a48-108.smtp-out.amazonses.com"
}
}
I've created a node.js Lambda function that is being called by AWS Kinesis. It processes the above JSON and the following code is being used. I am debugging, so I am simply trying to search and replace the string for testing:
console.log('Loading function');
exports.handler = async (payload, context) => {
payload = JSON.stringify(payload);
payload = payload.replace(/".*:.*":/g, 'replaced');
console.log('Replaced payload:', payload);
};
I've run the regex through multiple code testing sites and it correctly identifies all 6 occurrences of the key names with a : in them. I've even used this tester, and it correctly replaces all 6 strings as expected. But, when it runs on Lambda I get the following output:
Replaced payload: {replaced"a48-108.smtp-out.amazonses.com"}}
Am I missing something obvious or is this some weird thing happening in Lambda?
Note, if I simply output the unmodified payload variable it matches the input JSON, so that is not the issue.
I've got additional code which should complete this task successfully once I solve this mystery. But, if you could provide the code to replace : with _ in the key names, that would be great as well.

Answer 1

Expanding on your update comment:
UPDATE: I've discovered this problem is because of the "stringified" output which contains no spaces or newlines and is different than the original JSON.
You can add the following options to JSON.stringify to properly format the stringified output.
JSON.stringify(payload, null, 4);
Demo:
let payload = {"eventType":"Delivery","mail":{"timestamp":"2021-08-09T20:41:51.515Z","source":"removed#email.com","sourceArn":"arn:aws:ses:us-east-1:1234567890:identity/removed.com","sendingAccountId":"1234567890","messageId":"1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b","destination":["removedl#email.com"],"headersTruncated":false,"headers":[{"name":"From","value":"removed#email.com"},{"name":"To","value":"removed#email.com"},{"name":"Subject","value":"Test"},{"name":"MIME-Version","value":"1.0"},{"name":"Content-Type","value":"text/plain; charset=UTF-8"},{"name":"Content-Transfer-Encoding","value":"7bit"}],"commonHeaders":{"from":["removed#email.com"],"to":["removed#email.com"],"messageId":"1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b","subject":"Test"},"tags":{"ses:operation":["SendEmail"],"ses:configuration-set":["GenericLog"],"ses:source-ip":["123.123.123.123"],"ses:from-domain":["email.com"],"ses:caller-identity":["user1"],"ses:outgoing-ip":["234.234.234.234"]}},"delivery":{"timestamp":"2021-08-09T20:41:52.880Z","processingTimeMillis":1365,"recipients":["removed#email.com"],"smtpResponse":"250 2.6.0 <1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b#email.amazonses.com> [InternalId=11231247472, Hostname=CO6PR12MB5396.namprd12.prod.outlook.com] 12090 bytes in 0.214, 55.135 KB/sec Queued mail for delivery","reportingMTA":"a48-108.smtp-out.amazonses.com"}};
payload = JSON.stringify(payload, null, 4)
payload = payload.replace(/".*:.*":/g, 'replaced');
console.log('Replaced payload:', payload);

Answer 2

Mind that simply using .* means "as many characters as you can match", i.e. it's eager. You can add a ? for lazy matching, as in "match as few as possible".
I suggest using /".*?:.*?":/g since this basically means "get all characters to the closest : and then the characters to the closest ":".
Using a recursive function to map the object instead of using a RegExp on the JSON version:
const data = { "eventType": "Delivery", "mail": { "timestamp": "2021-08-09T20:41:51.515Z", "source": "removed#email.com", "sourceArn": "arn:aws:ses:us-east-1:1234567890:identity/removed.com", "sendingAccountId": "1234567890", "messageId": "1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b", "destination": ["removedl#email.com"], "headersTruncated": false, "headers": [{ "name": "From", "value": "removed#email.com" }, { "name": "To", "value": "removed#email.com" }, { "name": "Subject", "value": "Test" }, { "name": "MIME-Version", "value": "1.0" }, { "name": "Content-Type", "value": "text/plain; charset=UTF-8" }, { "name": "Content-Transfer-Encoding", "value": "7bit" }], "commonHeaders": { "from": ["removed#email.com"], "to": ["removed#email.com"], "messageId": "1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b", "subject": "Test" }, "tags": { "ses:operation": ["SendEmail"], "ses:configuration-set": ["GenericLog"], "ses:source-ip": ["123.123.123.123"], "ses:from-domain": ["email.com"], "ses:caller-identity": ["user1"], "ses:outgoing-ip": ["234.234.234.234"] } }, "delivery": { "timestamp": "2021-08-09T20:41:52.880Z", "processingTimeMillis": 1365, "recipients": ["removed#email.com"], "smtpResponse": "250 2.6.0 <1123bfe-123123b12e12d2c212a-123b12c2e1123-213f312d32c123b#email.amazonses.com> [InternalId=11231247472, Hostname=CO6PR12MB5396.namprd12.prod.outlook.com] 12090 bytes in 0.214, 55.135 KB/sec Queued mail for delivery", "reportingMTA": "a48-108.smtp-out.amazonses.com" } };
function convert(obj) {
if (typeof obj !== 'object') return obj;
if (Array.isArray(obj)) return obj.map(convert);
const result = {};
for(const key in obj) {
result[key.replace(/:/g, '_')] = convert(obj[key]);
}
return result;
}
const converted = convert(data);
console.log(converted);
console.log(JSON.stringify(converted, null, 4));