Curl and Postman giving different response? - node.js

I am writing API in node.js.
url http://localhost:8000/api/request?connId=19&timeout=8000
when I call above API from Postman then it takes 8 seconds to give the response. same as expected.
but when call this from curl it gives output instantly.
curl http://localhost:8000/api/request?connId=19&timeout=8000
Code is:
app.get('/api/request', ( req, res ) => {
let timeoutID = null; // return by setTimeout
let starttime = null; // time when request is begin
const time = req.query.timeout; // time send by client
const connId = req.query.connId; // id send by client
// checking client data is valid or not
if ( typeof(connId) !== undefined && typeof(time) !== undefined ) {
starttime = new Date(); // get current time
starttime = starttime.setMilliseconds(starttime.getMilliseconds()); // convert into millisecon
// initiate setTimeout
timeoutID = setTimeout(() => {
// remove that element from global array which reqest has been complted
removeByAttr(timeout, 'connId', connId);
// res.send({ status: "ok"}); // send response
}, time);
// initiate setInterval
const setInv = setInterval(() => {
// check timeout [] conatin the current request
const arrLength = timeout.length && timeout.filter(({ connId }) => req.query.connId === connId).length;
if ( arrLength === 0 ) {
res.send({ status: "ok"}); // send response
clearInterval(setInv); // clear/destroy current Interval
}
}, 80);
// insert the current request into global [] timeout
timeout.push({
connId,
time,
timeoutID,
starttime,
'endtime': +starttime + +time
});
}
});

As I found from your code and curl tutorial that When you GET a endpoint with curl your command should be like this:
curl "http://localhost:8000/api/request?connId=19&timeout=8000"
Without double inverted comma in url, console.log(req.query) will result in { connId: '19' } which is causing the problem.

Related

send multiple data from backend to front end

I am using the fetch api to get data from backend. The data I am getting is dynamic and more and more data keeps producing. What I want is to send the data to the front end when I get the data in the backend. How can I achieve this? I have coded a sample example of my senario below. Thanks in advance
fetch('/api/blah', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
request: `{"requestType": "numbers"}`
})
})
.then((res) => res.json())
.then(data => {
if (data.status == 'success') {
const numbers = data.numbers
console.log(numbers)
}
});
const distribution = async(req, res) => {
const request = JSON.parse(req.body.request)
if (request.requestType == 'numbers') {
var ceiling = 100;
var floor = 1;
var x = 1;
var step = 1;
setInterval(function() {
res.send({
status: 'success',
numbers: x
})
x += step;
if (x === ceiling || x === floor) {
step = -step;
}
}, 500);
}
}
You can use sockets to get your desired output. You can follow this link for more info Send message to specific client with socket.io and node.js
const
{Server} = require("socket.io"),
server = new Server(8000);
var ceiling = 100;
var floor = 1;
var x = 1;
var step = 1;
let
sequenceNumberByClient = new Map();
// event fired every time a new client connects:
server.on("connection", (socket) => {
console.info(`Client connected [id=${socket.id}]`);
// initialize this client's sequence number
sequenceNumberByClient.set(socket, 1);
// when socket disconnects, remove it from the list:
socket.on("disconnect", () => {
sequenceNumberByClient.delete(socket);
console.info(`Client gone [id=${socket.id}]`);
});
});
//emit your data to specific channel
setInterval(function() {
for (const [client, sequenceNumber] of sequenceNumberByClient.entries()) {
client.emit("numbers", x);
x += step;
if (x === ceiling || x === floor) {
step = -step;
}
}
}, 500);
For passing data from the back-end to the front-end dynamically, you can use jQuery AJAX
https://www.w3schools.com/js/js_ajax_intro.asp
In your case, you can call the AJAX endpoint once every few minutes or so to get the new data.
In REST API structure you can send a response to an API only once. You cannot use setInterval to generate multiple response.
instead you should add all possible data in array and put that array in your response once.
const distribution = async (req, res) => {
const request = JSON.parse(req.body.request);
if (request.requestType == "numbers") {
var ceiling = 100;
var step = 1;
var return_arr = [];
for (let i = 1; i <= ceiling; i += step) {
return_arr.push(i);
}
res.send({
status: "success",
numbers: return_arr,
});
}
};
If I understand correctly you want to send server events from your server to your frontend as soon as data changes on the server. In order to achieve this you will need websocket connections from frontend to server. websockets are a bidirectional connection between your server and client. while the clients are connected to the server via sockets, the server can send events to the frontend without receiving a request first. I suggest using socket.io for this.
You can find more info about socket.io here:
https://socket.io/docs/v4/
There are more alternatives like the websocketAPI. more info about websocketAPI you can find in mozilla webdocs here:
https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API
Also I found this article on how to implement websockets without socket.io as socket.io is more like a framework on top of normal websockets:
https://www.piesocket.com/blog/nodejs-websocket

Add delay response in nodejs

I am a beginner at nodejs. My problem is that I have 16 clients sending requests to the server. The server must aggregate 16 requests and send them to the client. (each client will receive 16 requests, of which 15 are from other clients).
How can I add delay to server (nodejs) while waiting for request aggregation then send it to each client.
Can you help me.
You have to ensure that all the clients will send a parameter called batchId. You can simply,
Keep a cache
Put the request body & response in there with a batchId
On every client request, push the body & response using batchId.
On every client request, check if the batch size is 16
If 16, aggregate and response all the requests at once.
Delete the local cache of the batch.
const requestsCache = {};
function aggregateAndRespond(batchId) {
const arrivedRequests = requestsCache[batchId];
// process "body" property in the "arrivedRequests" elements
const result = processRequests(arrivedRequests);
for(let arrivedRequest of arrivedRequests) {
arrivedRequest.response.json(result);
}
delete requestsCache[batchId];
}
app.post('/client-requests', function(request, response) {
const body = request.body;
const batchId = body.batchId;
const arrivedRequests = requestsCache[batchId] || [];
arrivedRequests.push({ response, body });
if(arrivedRequests.length === 16) {
aggregateAndRespond(batchId);
}
});
Update:
const requestsCache = {};
function aggregateAndRespond(batchId) {
const hours = 2;
const batchDetails = requestsCache[batchId];
const arrivedRequests = batchDetails.requests;
// process "body" property in the "arrivedRequests" elements
const result = processRequests(arrivedRequests);
for(let arrivedRequest of arrivedRequests) {
arrivedRequest.response.json(result);
}
batchDetails.processed = true;
// batch details cannot be deleted immediately because,
// if a same batch request comes after the time out,
// it will end up creating a new batch details
// if that is desired, replace the next timer lines with "delete requestsCache[batchId];"
// otherwise, as the batch details is marked as "processed", remaining requests of same batch
// will simply be dropped.
// The batch details will also be dropped after 2 hours.
// If you think shorter duration would suffice, update accordingly.
setTimeout(function someMoreTimeLater() {
delete requestsCache[batchId];
}, hours * 60 * 60 * 1000);
}
function tryAggregateAndRespond(batchId, timeoutInMinutes, requestData) {
const defaultBatchDetails = {
processed: false,
requests: [],
aggregateTimerReference: null
};
const batchDetails = requestsCache[batchId] || defaultBatchDetails;
if(batchDetails.processed === true) {
return true;
}
batchDetails.requests.push(requestData);
// timer is reset every time the request is arrived.
// if no request is arrived for configured time after the last request,
// aggregation will kick in
// if you want the timer to be set only at the first request,
// delete the next line and uncomment the "if" block after that.
clearTimeout(batchDetails.aggregateTimerReference);
//if(batchDetails.aggregateTimerReference !== null) {
// return false;
//}
batchDetails.aggregateTimerReference = setTimeout(function someTimeLater() {
aggregateAndRespond(batchId);
}, timeoutInMinutes * 60 * 1000);
}
app.post('/client-requests', function(request, response) {
const timeoutInMinutes = 2;
const body = request.body;
const batchId = body.batchId;
const endRequest = tryAggregateAndRespond(batchId, timeoutInMinutes, { response, body });
if(endRequest === true) {
response.json({ message: 'Batch already processed', batchId });
}
});

Asynchronous Issue with createCustomerProfile in Sample Node.JS code

Our goal is to integrate Authorize.NET into our application using the Node SDK sample code.
Node SDK: https://github.com/AuthorizeNet/sdk-node
Recommended Sample Code: https://github.com/AuthorizeNet/sample-code-node/tree/ef9e5c2d9e0379b5f47a0ebcb6847e711fe196ef
I am trying to create a customer payment profile and while I am able to create a customer profile successfully and receive a successful response from the API call 
CustomerProfilesModule.createCustomerProfile, the remainder of my auth.controller.js runs before I get the API result. All of the create-customer-profile.js runs up until the ctrl.execute() runs, then the console.log("xxx") in auth.controller.js runs before grabbing the API result. 
I understand this is a synchronous issue with my code, but I don't know how to solve this. I am using the sample code authorize.NET provided, however the code is using the real data from my app rather than the sample data. I am more than happy to provide further information upon request and really appreciate any help! 
// AUTH.CONTROLLER.JS
const httpStatus = require("http-status");
const ApiContracts = require("authorizenet").APIContracts;
const ApiControllers = require("authorizenet").APIControllers;
const SDKConstants = require("authorizenet").Constants;
const User = require("../models/user.model");
const RefreshToken = require("../models/refreshToken.model");
const moment = require("moment-timezone");
const { jwtExpirationInterval } = require("../../config/vars");
const sgMail = require("#sendgrid/mail");
const bcrypt = require("bcryptjs");
const CustomerProfilesModule = require("../utils/authorizeNet/CustomerProfiles");
sgMail.setApiKey(process.env.SENDGRID_API_KEY.replace(/\r?\n|\r/g, ""));
exports.register = async (req, res, next) => {
try {
const userData = req.body;
let customerProfileResult =
await CustomerProfilesModule.createCustomerProfile(userData);
console.log(
"❌ ❌ ❌ ❌ ❌ customerProfile Result ",
customerProfileResult
);
if (!userData || userData.error) {
return next(error);
} else {
const { isTrial } = userData;
const user = await new User(userData).save();
const token = generateTokenResponse(user, user.token());
res.status(httpStatus.CREATED);
return res.json({ token, user });
}
} catch (error) {
console.log(error.message);
return next(User.checkDuplicateEmail(error));
}
};
//////////
// CREATE-CUSTOMER-PROFILE.JS
var ApiContracts = require("authorizenet").APIContracts;
var ApiControllers = require("authorizenet").APIControllers;
var utils = require("../utils.js");
async function createCustomerProfile(user) {
console.log(" user parameter", user);
var merchantAuthenticationType =
new ApiContracts.MerchantAuthenticationType();
merchantAuthenticationType.setName(process.env.AUTHORIZE_NET_API_LOGIN_KEY);
merchantAuthenticationType.setTransactionKey(
process.env.AUTHORIZE_NET_TRANSACTION_KEY
);
var creditCard = new ApiContracts.CreditCardType();
creditCard.setCardNumber(user.cardNumber);
if (user.cardExpiry.length > 4) {
creditCard.setExpirationDate(
`${user.cardExpiry.slice(0, 1)}${user.cardExpiry.slice(3, 4)}`
);
} else {
creditCard.setExpirationDate(user.cardExpiry);
}
console.log("creditCard", creditCard);
var paymentType = new ApiContracts.PaymentType();
paymentType.setCreditCard(creditCard);
var customerAddress = new ApiContracts.CustomerAddressType();
customerAddress.setFirstName(user.firstName);
customerAddress.setLastName(user.lastName);
customerAddress.setAddress(user.mailingAddress);
customerAddress.setCity(user.mailingCity);
customerAddress.setState(user.mailingState);
customerAddress.setZip(user.mailingZip);
customerAddress.setCountry("USA");
customerAddress.setPhoneNumber(user.userPhone);
var customerPaymentProfileType =
new ApiContracts.CustomerPaymentProfileType();
customerPaymentProfileType.setCustomerType(
ApiContracts.CustomerTypeEnum.INDIVIDUAL
);
customerPaymentProfileType.setPayment(paymentType);
customerPaymentProfileType.setBillTo(customerAddress);
var paymentProfilesList = [];
paymentProfilesList.push(customerPaymentProfileType);
console.log(
"paymentProfilesList",
paymentProfilesList
);
var customerProfileType = new ApiContracts.CustomerProfileType();
customerProfileType.setMerchantCustomerId(
"M_" + utils.getRandomString("cust")
);
customerProfileType.setDescription(
`${user.firstName} ${user.lastName}'s Account'`
);
customerProfileType.setEmail(user.userEmail);
customerProfileType.setPaymentProfiles(paymentProfilesList);
var createRequest = new ApiContracts.CreateCustomerProfileRequest();
createRequest.setProfile(customerProfileType);
createRequest.setValidationMode(ApiContracts.ValidationModeEnum.TESTMODE);
createRequest.setMerchantAuthentication(merchantAuthenticationType);
var ctrl = new ApiControllers.CreateCustomerProfileController(
createRequest.getJSON()
);
// All above code is ran when CustomerProfilesModule.createCustomerProfile(userData) is executed in auth.controller.js
// However the following line (line 130 in auth.controller.js) is ran before the below ctrl.execute() code is completed
//
// console.log("❌ ❌ ❌ ❌ ❌ customerProfile Result ", customerProfileResult);
//
// All the above code is executed before that console.log("❌ ❌ ❌") statement above, however the below code doesn't run before that console.log
// I'd like the below code to execute before the remaining register route is finished, but just don't know what is going on!
ctrl.execute(async function () {
var apiResponse = await ctrl.getResponse();
console.log("apiResponse", apiResponse);
var response = new ApiContracts.CreateCustomerProfileResponse(apiResponse);
console.log("response", response);
//pretty print response
//console.log(JSON.stringify(response, null, 2));
if (response != null) {
if (
response.getMessages().getResultCode() ==
ApiContracts.MessageTypeEnum.OK
) {
console.log(
"Successfully created a customer profile with id: " +
response.getCustomerProfileId()
);
} else {
console.log("Result Code: " + response.getMessages().getResultCode());
console.log(
"Error Code: " + response.getMessages().getMessage()[0].getCode()
);
console.log(
"Error message: " + response.getMessages().getMessage()[0].getText()
);
return {
error:
"Error message: " +
response.getMessages().getMessage()[0].getText(),
};
}
} else {
console.log("Null response received");
return { error: "Null response received" };
}
});
}
module.exports.createCustomerProfile = createCustomerProfile;
ctrl.execute is a method handling an IIFE function which is not called.
IIFE functions are invoked as soon as is defined.
You will not be able to run IIFE before declaration.
Possible solution:
Try to create a callback route inside register route and then exclude the IIFE to get the response from the callback inside register before actual route finished.
I don't fully understand your code enough to know how you want it to work. But from my understanding the console.log is running between when you get a call from the api and after you get the api, putting it in an awkward phase.
How asynchronous code works is that JavaScript will let the asynchronous function run, leave, do something else in the meantime, and get back to it when it is done.
The issue I see with your code is that createCustomerProfile doesn't return anything when it's done. You have a function that returns a promise of void. First off, that's a problem because you're using the return value of the function in console.log().
I highly recommend to promisify so that it properly resolves or has an error, which when you're working with API's you're likely to encounter potential errors in which you want to handle that.
You say the console.log() is being called before ctrl.execute() but I don't even see where it is being executed at all because I don't see it in the createCustomerProfile function.

Do node js worker never times out?

I have an iteration that can take up to hours to complete.
Example:
do{
//this is an api action
let response = await fetch_some_data;
// other database action
await perform_operation();
next = response.next;
}while(next);
I am assuming that the operation doesn't times out. But I don't know it exactly.
Any kind of explanation of nodejs satisfying this condition is highly appreciated. Thanks.
Update:
The actual development code is as under:
const Shopify = require('shopify-api-node');
const shopServices = require('../../../../services/shop_services/shop');
const { create } = require('../../../../controllers/products/Products');
exports.initiate = async (redis_client) => {
redis_client.lpop(['sync'], async function (err, reply) {
if (reply === null) {
console.log("Queue Empty");
return true;
}
let data = JSON.parse(reply),
shopservices = new shopServices(data),
shop_data = await shopservices.get()
.catch(error => {
console.log(error);
});
const shopify = new Shopify({
shopName: shop_data.name,
accessToken: shop_data.access_token,
apiVersion: '2020-04',
autoLimit: false,
timeout: 60 * 1000
});
let params = { limit: 250 };
do {
try {
let response = await shopify.product.list(params);
if (await create(response, shop_data)) {
console.log(`${data.current}`);
};
data.current += data.offset;
params = response.nextPageParameters;
} catch (error) {
console.log("here");
console.log(error);
params = false;
};
} while (params);
});
}
Everything is working fine till now. I am just making sure that the execution will ever happen in node or not. This function is call by a cron every minute, and data for processing is provided by queue data.

Use headless chrome to intercept image request data

I have a use case that needs to use Headless Chrome Network (https://chromedevtools.github.io/devtools-protocol/tot/Network/) to intercept all images requests and find out the image size before saving it (basically discard small images such as icons).
However, I am unable to figure out a way to load the image data in memory before saving it. I need to load it in Img object to get width and height. The Network.getResponseBody is taking requestId which I don't have access in Network.requestIntercepted. Also Network.loadingFinished always gives me "0" in encodedDataLength variable. I have no idea why. So my questions are:
How to intercept all responses from jpg/png request and get the image data? Without saving the file via URL string to the disk and load back.
BEST: how to get image dimension from header response? Then I don't have to read the data into memory at all.
My code is below:
const chromeLauncher = require('chrome-launcher');
const CDP = require('chrome-remote-interface');
const file = require('fs');
(async function() {
async function launchChrome() {
return await chromeLauncher.launch({
chromeFlags: [
'--disable-gpu',
'--headless'
]
});
}
const chrome = await launchChrome();
const protocol = await CDP({
port: chrome.port
});
const {
DOM,
Network,
Page,
Emulation,
Runtime
} = protocol;
await Promise.all([Network.enable(), Page.enable(), Runtime.enable(), DOM.enable()]);
await Network.setRequestInterceptionEnabled({enabled: true});
Network.requestIntercepted(({interceptionId, request, resourceType}) => {
if ((request.url.indexOf('.jpg') >= 0) || (request.url.indexOf('.png') >= 0)) {
console.log(JSON.stringify(request));
console.log(resourceType);
if (request.url.indexOf("/unspecified.jpg") >= 0) {
console.log("FOUND unspecified.jpg");
console.log(JSON.stringify(interceptionId));
// console.log(JSON.stringify(Network.getResponseBody(interceptionId)));
}
}
Network.continueInterceptedRequest({interceptionId});
});
Network.loadingFinished(({requestId, timestamp, encodedDataLength}) => {
console.log(requestId);
console.log(timestamp);
console.log(encodedDataLength);
});
Page.navigate({
url: 'https://www.yahoo.com/'
});
Page.loadEventFired(async() => {
protocol.close();
chrome.kill();
});
})();
This should get you 90% of the way there. It gets the body of each image request. You'd still need to base64decode, check size and save etc...
const CDP = require('chrome-remote-interface');
const sizeThreshold = 1024;
async function run() {
try {
var client = await CDP();
const { Network, Page } = client;
// enable events
await Promise.all([Network.enable(), Page.enable()]);
// commands
const _url = "https://google.co.za";
let _pics = [];
Network.responseReceived(async ({requestId, response}) => {
let url = response ? response.url : null;
if ((url.indexOf('.jpg') >= 0) || (url.indexOf('.png') >= 0)) {
const {body, base64Encoded} = await Network.getResponseBody({ requestId }); // throws promise error returning null/undefined so can't destructure. Must be different in inspect shell to app?
_pics.push({ url, body, base64Encoded });
console.log(url, body, base64Encoded);
}
});
await Page.navigate({ url: _url });
await sleep(5000);
// TODO: process _pics - base64Encoded, check body.length > sizeThreshold, save etc...
} catch (err) {
if (err.message && err.message === "No inspectable targets") {
console.error("Either chrome isn't running or you already have another app connected to chrome - e.g. `chrome-remote-interface inspect`")
} else {
console.error(err);
}
} finally {
if (client) {
await client.close();
}
}
}
function sleep(miliseconds = 1000) {
if (miliseconds == 0)
return Promise.resolve();
return new Promise(resolve => setTimeout(() => resolve(), miliseconds))
}
run();

Resources