node-fetch timeout issue/ express timeout/ lambda timeout or sth else? - node.js

below code works fine when running locally, but "getAccessToken ()" does not work as expected when running inside an aws lambda function, when doing "POST" to /webhook endpoint. I am using node 8.10, "aws-serverless-express": "^3.3.6", "express": "^4.16.4" and "node-fetch": "^2.5.0", basically it print the correct jwt token but node-fetch does not return anything, sample log.
START RequestId: c8efba59-1869-4eaa-b9d8-aa15a7507d52 Version: $LATEST
2019-05-27T19:55:32.328Z c8efba59-1869-4eaa-b9d8-aa15a7507d52 start getExecution
2019-05-27T19:55:32.328Z c8efba59-1869-4eaa-b9d8-aa15a7507d52 exectution_url:
2019-05-27T19:55:32.328Z c8efba59-1869-4eaa-b9d8-aa15a7507d52 https://cloudmanager.adobe.io/somevalidurl
2019-05-27T19:55:32.328Z c8efba59-1869-4eaa-b9d8-aa15a7507d52 start getAccessToken
END RequestId: c8efba59-1869-4eaa-b9d8-aa15a7507d52
REPORT RequestId: c8efba59-1869-4eaa-b9d8-aa15a7507d52 Duration: 6657.00 ms Billed Duration: 6700 ms Memory Size: 128 MB Max Memory Used: 37 MB
I made sure lambda timeout is 30 seconds, tried to disable "node-fetch" timeout by setting it to 0 and , used a middleware for all the routes "app.use(timeout("30000"))" also for that specific webhook request timeout. (I receive the 200 pong response immediately but getexectuion async function does not work properly)
const express = require('express')
const bodyParser = require('body-parser')
const crypto = require('crypto')
const jsrsasign = require('jsrsasign')
const fetch = require('node-fetch')
const timeout = require('connect-timeout')
const URL = require('url').URL
const URLSearchParams = require('url').URLSearchParams
//require('dotenv').config()
const app = express()
async function getAccessToken () {
console.log("start getAccessToken")
const EXPIRATION = 60 * 60 // 1 hour
const header = {
'alg': 'RS256',
'typ': 'JWT'
}
const payload = {
'exp': Math.round(new Date().getTime() / 1000) + EXPIRATION,
'iss': process.env.ORGANIZATION_ID,
'sub': process.env.TECHNICAL_ACCOUNT_ID,
'aud': `https://ims-na1.adobelogin.com/c/${process.env.API_KEY}`,
'https://ims-na1.adobelogin.com/s/ent_cloudmgr_sdk': true
}
const jwtToken = jsrsasign.jws.JWS.sign('RS256', JSON.stringify(header), JSON.stringify(payload), process.env.PRIVATE_KEY)
//console.log("jwt token:")
//console.log(jwtToken)
const body = new URLSearchParams({
client_id: process.env.API_KEY,
client_secret: process.env.CLIENT_SECRET,
jwt_token: jwtToken
})
const response = await fetch('https://ims-na1.adobelogin.com/ims/exchange/jwt', {
method: 'POST',
options: { timeout: 0},
timeout: 0,
size: 0,
body: body
})//.catch(error => {
// console.log("an error happend in fetchg")
// console.log(error)
//})
const json = await response.json()
if ((response.status !== 200) && (response.status !== 201)) {
console.error(`Invalid response status ${ response.status }.`);
throw json;
}
console.log("access_token:")
console.log(json['access_token'])
return json['access_token']
}
async function makeApiCall (accessToken, url, method) {
console.log("start make api call")
const response = await fetch(url, {
'method': method,
'headers': {
'x-gw-ims-org-id': process.env.ORGANIZATION_ID,
'x-api-key': process.env.API_KEY,
'Authorization': `Bearer ${accessToken}`
}
})
console.log("finish make api call")
const json = await response.json()
return json
}
function getLink (obj, linkType) {
return obj['_links'][linkType].href
}
async function getExecution (executionUrl) {
console.log("start getExecution")
console.log("exectution_url:")
console.log(executionUrl)
const accessToken = await getAccessToken()
console.log("access-token:")
console.log(accessToken)
const execution = await makeApiCall(accessToken, executionUrl, 'GET')
console.log(execution)
console.log("aaaa")
const program = await makeApiCall(accessToken, new URL(getLink(execution, 'http://ns.adobe.com/adobecloud/rel/program'), executionUrl))
console.log(execution)
console.log("here")
execution.program = program
return execution
}
//app.use(bodyParser.json())
app.use(bodyParser.json({
verify: (req, res, buf, encoding) => {
const signature = req.header('x-adobe-signature')
if (signature) {
const hmac = crypto.createHmac('sha256', process.env.CLIENT_SECRET)
hmac.update(buf)
const digest = hmac.digest('base64')
if (signature !== digest) {
throw new Error('x-adobe-signature HMAC check failed')
}
} else if (!process.env.DEBUG && req.method === 'POST') {
throw new Error('x-adobe-signature required')
}
}
}))
app.use(timeout("30000"))
app.post('/webhook', (req, res) => {
req.setTimeout(120000, function(){
console.log('Request has timed out.');
res.send(408);
});
res.writeHead(200, { 'Content-Type': 'application/text' })
res.end('pong')
getExecution("https://cloudmanager.adobe.io/<somevalidurl>").then(execution => {
console.log(`Execution for ${execution.program.name} started`)
})
})
module.exports = app;
//const port = process.env.PORT || 3000
//app.listen(port, () =>
// console.log(`App is listening on port ${port}.`)
//)

Related

Expressjs API variables undefined

i'm trying to create my first API using nodejs / express js for backend and React for frontend. I'm able to call the API via postman but not via localhost on frontend as the path returns "undefined" and my "mediaType" and "id" are undefined.
The frontend should call the API
BACKEND
const express = require("express");
const app = express();
const PORT = 8080;
const axios = require("axios");
app.use(express.json());
app.get("/test", (req, res) => {
// const { mediaType, id } = req.params;
const { mediaType, id } = req.body;
const options = {
methond: "GET",
url: `https://api.themoviedb.org/3/${mediaType}/${id}?api_key=${APIKEY}&language=en-US`,
// headers: {
// }
};
axios.request(options).then((response) => {
res.send(response.data);
}).catch((error) => {
console.log(error)
})
});
FRONTEND
const fetchData = async () => {
const { data } = await axios.get(
`http://localhost:8080/test`, { params: { mediaType: mediaType, id: id } }
);
setContent(data);
};
PART OF HEADER
_header: 'GET /3/undefined/undefined?api_key=XXX&language=en-US HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Host: api.themoviedb.org\r\n' +
'Connection: close\r\n' +
'\r\n',
Alright, for anyone interested...
i updated the FE req
const fetchData = async () => {
const { data } = await axios.get(
`http://localhost:8080/test2?mediaType=${mediaType}&id=${id}`
);
setContent(data);
};
And changed the BE req also, to use query parameters
app.get("/test2", (req, res) => {
// const { mediaType, id } = req.params;
mediaType = req.query.mediaType;
id = req.query.id;
const options = {
methond: "GET",
url: `https://api.themoviedb.org/3/${mediaType}/${id}?api_key=${apiKey}&language=en-US`,
// headers: {
//
}
};
axios
.request(options)
.then((response) => {
res.send(response.data);
})
.catch((error) => {
console.log(error);
});
});
Also needed to update CORS (no idea how it works????) as it blocked my localhost testing
const cors = require("cors");
app.use(
cors({
origin: "http://localhost:3000",
credentials: true, //access-control-allow-credentials:true
optionSuccessStatus: 200,
})
);
Now it seems that everything works!
You accept body parameters in your API endpoint
so with axios you should use data parameter with your body data included and send it to your backend:
const fetchData = async () => {
const { data } = await axios.get(
`http://localhost:8080/test`, { data: { mediaType: mediaType, id: id } } <--- check data key here
);
setContent(data);
};
by the way, it's not a good idea to send body HTTP request with GET http verb

Nodejs - Axios - Timeouts lead to memory leak

I'm running an app which basically:
Receives a request through Express.js
Send multiples requests to various endpoints
Once those endpoints responses, we compute a response and send it to the client.
I'm using Axios Instances per endpoints to send the requests.
const axios = require('axios');
const Agent = require('agentkeepalive');
const { HttpsAgent } = Agent;
const httpKeepAliveAgent = new Agent({
maxSockets: 100,
maxFreeSockets: 10,
timeout: 60000, // active socket keepalive for 60 seconds
freeSocketTimeout: 30000, // free socket keepalive for 30 seconds
});
const httpsKeepAliveAgent = new HttpsAgent({
maxSockets: 100,
maxFreeSockets: 10,
timeout: 60000, // active socket keepalive for 60 seconds
freeSocketTimeout: 30000, // free socket keepalive for 30 seconds
});
const createAxiosInstance = () => axios.create({
httpAgent: httpKeepAliveAgent,
httpsAgent: httpsKeepAliveAgent,
maxRedirects: 10,
});
I send requests to third party endpoint using the following
const fakeServer = require('../test/fake-server');
const logger = require('../utils/logger');
const { LOG_VERBOSE } = process.env;
// let promiseCount = 0;
module.exports = async (axiosInstance, ssp, payload, endpoint, method, timeout, headers) => {
const cmd = process.env.NODE_ENV === 'test' ? fakeServer : axiosInstance;
const start = Date.now();
const config = {
ssp,
url: endpoint,
method,
timeout,
headers: {
'Content-Type': 'application/json; charset=utf-8;',
Accept: 'application/json',
},
data: payload,
};
if (headers !== undefined && typeof headers === 'object') {
// eslint-disable-next-line no-return-assign
Object.keys(headers).forEach((key) => config.headers[key] = headers[key]);
}
try {
const response = await cmd(config);
return {
ssp,
uri: config.url,
requestbody: payload,
requestheaders: config.headers,
responsebody: response.data,
status: response.status,
responsetimemillis: Date.now() - start,
};
} catch (error) {
if (LOG_VERBOSE === 'true') logger.error(`Error on ${ssp} call: ${error.message}`);
let responsebody;
let status;
if (error.response === undefined) {
responsebody = error.code;
status = error.code;
} else {
responsebody = error.response.data ? error.response.data : error.message;
status = error.response.status;
}
return {
ssp,
uri: config.url,
requestbody: payload,
requestheaders: config.header,
responsebody,
status,
responsetimemillis: Date.now() - start,
};
}
};
The issues is that when I get timeouts or 400 error, there might happen some memory leaks, I can't get a successful linked connection, and so at the end the app crashes...
Any ideas?
https://github.com/axios/axios/issues/2783
maxRedirects may be causing this issue, try setting lower redirects or 0, as the resolved issue suggests.

SyntaxError: Unexpected end of JSON input at JSON.parse (<anonymous>) at Server.requestListene ()

Have problem with Node Js backend, if i use test client (not React) it works, if place it to React app I have subj error.
Server is running on http://localhost:8000
undefined:1
SyntaxError: Unexpected end of JSON input
at JSON.parse ()
at Server.requestListener (/home/boris/bookni_ru_srv/backend.js:27:38)
at processTicksAndRejections (node:internal/process/task_queues:96:5)
Here is server part:
let { Logincheck } = require('./ConnectDB');
const http = require("http");
var fs = require('fs');
const host = 'localhost';
const port = 8000;
const rbkp_srv ='192.168.56.101';
const test = fs.readFileSync('test.json', 'utf-8');
const requestListener = async function (req, res) {
const buffers = [];
for await (const chunk of req) {
buffers.push(chunk);
}
const data = Buffer.concat(buffers);
res.setHeader("Content-Type", "application/json");
switch (req.url) {
case "/login":
res.writeHead(200);
const rbkp_passwd = JSON.parse(data).passwd;
const rbkp_user = JSON.parse(data).user;
let login = Logincheck(rbkp_srv, rbkp_user, rbkp_passwd);
login.then(function(result){if (result===true) {console.log('Login sucessed'); res.end('Login');} else { console.log(result); res.end(result.toString())}
});
break
}
};
const server = http.createServer(requestListener);
server.listen(port, host, () => {
console.log(`Server is running on http://${host}:${port}`);
});
Here is FrontEnd part:
export default function Auth(){
const requestOptions = {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({"user" : "boris",
"passwd" : "test"})
};
fetch('http://localhost:8000/login', requestOptions)
.then(response => response.json())
.then(data => this.setState({ postId: data.id }));
}`
Have no clu hoe to fix it yet need help.

Extracting values from API data in node.js

Thank you for your time.
I'm trying to use OAuth2 in discord, but I'm having a hard time figuring out how to retrieve the username.
Can someone please tell me how to do it?
code
const fetch = require('node-fetch');
const express = require('express');
const app = express();
app.get('/', async ({ query }, response) => {
const { code } = query;
if (code) {
try {
const oauthResult = await fetch('https://discord.com/api/oauth2/token', {
method: 'POST',
body: new URLSearchParams({
client_id: process.env['ci'],
client_secret: process.env['cs'],
code,
grant_type: 'authorization_code',
redirect_uri: `https://oauth.aiueominato1111.repl.co`,
scope: 'identify',
}),
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
});
const oauthData = await oauthResult.json();
const userResult = await fetch('https://discord.com/api/users/#me', {
headers: {
authorization: `${oauthData.token_type} ${oauthData.access_token}`,
},
});
console.log( await userResult.json());
} catch (error) {
// NOTE: An unauthorized token will not throw an error;
// it will return a 401 Unauthorized response in the try block above
console.error(error);
}
}
return response.sendFile('index.html', { root: '.' });
});
app.listen(port, () => console.log(`App listening at http://localhost:${port}`));
thank you
You already have the JSON data (from await userResult.json()) with it you can either get the property from the object or destructure it.
Getting property
const user = await userResult.json();
const username = user.username
Destructuring
const { username, id } = await userResult.json()
You can find out more about what properties you can extract from the Discord documentation

Server-Sent Events (SSE) problem with SSL/HTTPS

Hello I am developing a web application in React that receives SSE data from an Express server with Nginx.
SERVER.JS
const express = require('express');
const bodyParser = require('body-parser');
const cors = require('cors');
const crypto = require('crypto');
const app = express();
var lastClientRes = null;
function eventsHandler(req, res, next) {
const headers = {
'Content-Type': 'text/event-stream',
'Connection': 'keep-alive',
'Cache-Control': 'no-cache'
};
res.writeHead(200, headers);
const clientId = Date.now();
const newClient = {
id: clientId,
nonce: null,
cart: null,
res
};
requests.push(newClient);
const data = `data: ${JSON.stringify({client: clientId})}\n\n`;
res.write(data);
req.on('close', () => {
console.log(`${clientId} Connection closed`);
clients = clients.filter(c => c.id !== clientId);
});
}
function sendEventsToAll(newNest) {
clients.forEach(c => c.res.write(`data: ${JSON.stringify(newNest)}\n\n`))
}
async function addCart(req, res) {
const newCart = req.body;
requests.forEach(r => {
if(newCart.client == r.id){
var nonce = crypto.randomBytes(16).toString('base64');
r.nonce = nonce;
r.cart = newCart.cart;
r.res.write(`data: ${JSON.stringify({nonce: nonce})}\n\n`);
}
})
}
async function addCart(req, res) {
const newCart = req.body;
requests.forEach(r => {
if(newCart.client == r.id){
var nonce = crypto.randomBytes(16).toString('base64');
r.nonce = nonce;
r.cart = newCart.cart;
r.res.write(`data: ${JSON.stringify({nonce: nonce})}\n\n`);
}
})
}
async function confirmCart(req, res){
var nonce = req.body.nonce;
var found = -1;
requests.forEach((item, i) => {
if(item.nonce == nonce){
found = i;
return;
}
});
if(found)
{
console.log("OK");
requests[found].res.write(`data: ${JSON.stringify({confirm: true})}\n\n`);
}
}
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: false}));
app.post('/addCart', addCart);
app.post('/confirmCart', confirmCart);
app.get('/events', eventsHandler);
app.get('/status', (req, res) => res.json({clients: clients.length}));
const PORT = 3001;
let requests= [];
let clients = [];
let nests = [];
app.listen(PORT, () => console.log(`SSE service listening on port ${PORT}`));
INDEX:JS
import React from 'react';
import ReactDOM from 'react-dom';
import axios from 'axios';
class App extends React.Component {
constructor(props) {
super(props);
this.state = {
jsonCart: "",
cartNonce: "",
clientId: "",
cartConfirmed: false,
cart: Array(),
timerId: null,
listening: false,
cartConfermato: ""
};
}
buy(){
if (!this.state.listening) {
const events = new EventSource('https://api.myDomain.com/events', );
events.onmessage = (event) => {
const parsedData = JSON.parse(event.data);
console.log(event.data);
if(parsedData.client != null)
{
this.setState({
clientId: parsedData.client,
});
this.sendCart();
}
if(parsedData.nonce != null)
this.setState({
cartNonce: parsedData.nonce,
});
if(parsedData.confirm == true)
this.setState({
cartNonce: "",
cartConfermato: "Il carrello รจ stato confermato!"
});
};
this.setState({
listening: true
});
}
}
sendCart(){
var cart = JSON.stringify(this.state.cart.slice());
this.setState({
jsonCart: cart
});
axios.post(`https://api.myDomain.com/addCart`, {client: this.state.clientId, cart: cart});
}
*** ... ***
const events = new EventSource('https://api.myDomain.com/events', );
axios.post(https://api.myDomain.com/addCart, {client:
this.state.clientId, cart: cart});
In http everything works perfectly but if I set https generating the certificates with certbot I no longer receive "events" from the express server.
The only errors that appears in the chrome console is this
I replaced sub.domain with my domain
These errors appear a few minutes after the first request
GET https://sub.domain.com/events net::ERR_INCOMPLETE_CHUNKED_ENCODING 200 (OK)
2sub.domain.com/addCart:1 POST https://sub.domain.com/addCart 504 (Gateway Time-out)
(index):1 Access to XMLHttpRequest at 'https://sub.domain.com/addCart' from origin 'https://example.com' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
createError.js:16 Uncaught (in promise) Error: Network Error
at e.exports (createError.js:16)
at XMLHttpRequest.p.onerror (xhr.js:83)
e.exports # createError.js:16
p.onerror # xhr.js:83
error (async)
(anonymous) # xhr.js:80
e.exports # xhr.js:12
e.exports # dispatchRequest.js:50
Promise.then (async)
u.request # Axios.js:61
r.forEach.u.<computed> # Axios.js:86
(anonymous) # bind.js:9
value # index.js:156
state.listening.EventSource.onmessage # index.js:121
index.js:114 {"client":1579885346578}
index.js:150 send
sub.domain.com/events:1 GET https://sub.domain.com/events net::ERR_INCOMPLETE_CHUNKED_ENCODING 200 (OK)
2sub.domain.com/addCart:1 POST https://sub.domain.com/addCart net::ERR_ABORTED 504 (Gateway Time-out)
As Darren Cook described here
You need to disable buffering on your server-side.
Adding the "X-Accel-Buffering" parameter and set it to "no" in the response header fixed the issue for me.
const headers = {
'Content-Type': 'text/event-stream',
'Connection': 'keep-alive',
'Cache-Control': 'no-cache',
'X-Accel-Buffering': 'no'
};

Resources