How to intercept multiple http requests (API calls) with jest & puppeteer environment?
I have the following code but I get error Request already handled when I make more than one API call.
Code:
export const apiPostMethodResponseStatusCheckUtil = async (
apiObject: APIObject) => {
let completeAPIURL = URLs.apiURL + apiObject.endpoint;
describe(apiObject.testDescribe, () => {
let expectedResponse = 200;
it(apiObject.testCaseMsg, async () => {
console.log("API Request: " + completeAPIURL);
console.log("API Payload: " + JSON.stringify(apiObject.apiPayload));
await page.setRequestInterception(true);
page.on("request", (interceptedRequest) => {
var data = {
method: "POST",
postData: JSON.stringify(apiObject.apiPayload),
};
interceptedRequest.continue(data);
});
await page.goto(completeAPIURL, {
waitUntil: "load",
timeout: apiObject.responseTimeout,
});
page.on("response", async (response) => {
if (response.url().includes(apiObject.includesInURL)) {
console.log("API response status...." + response.status());
expect(response.status()).toBe(expectedResponse);
}
});
});
});
};
How to disable request interception after each API call?
let apiBody1=new APIObject(page, "{entity}/getFirstSet", "getFirstSet", {"_from":0,"_size":200,"_sort":{"id":"asc"}}, 0, `set 1`, `Making API call for set 1 `);
// call util method
apiPostMethodResponseStatusCheckUtil(apiBody1)
let apiBody2=new APIObject(page, "{entity}/getSecondSet", "getSecondSet", {"_from":0,"_size":200,"_sort":{"name":"asc"}}, 0, `set 2`, `Making API call for set 2`);
// call util method
apiPostMethodResponseStatusCheckUtil(apiBody2)
Related
I have a kibana instance behind a nodejs 16.x aws lambda reverse proxy. It works fine for almost everything, except for the "Discover" section when I add 2 or more filters for 2+ months timerange, it gives back bad gateway error.
Detail of the error:
Wrapper#https://mydomain/_dashboards/909221/bundles/core/core.entry.js:6:4249
_createSuperInternal#https://mydomain/_dashboards/909221/bundles/core/core.entry.js:6:3388
...
...
Implementation:
function proxy(event, context, lambdaCallback) {
delete(event.headers["accept-encoding"])
var path = event.path;
if (event.multiValueQueryStringParameters) {
path += '?' + deserializeQueryString(event.multiValueQueryStringParameters)
}
// Calculate the options for the HTTPS request
var opts = {
host: my_es_endpoint,
path: path,
method: event.httpMethod,
service: 'es',
region: my_region,
headers: event.headers
}
if ((event.httpMethod == 'POST') || (event.httpMethod == 'PUT')) {
if (event.body) {
var buff = new Buffer(event.body, "base64")
var payload = buff.toString('utf-8')
opts.body = payload
}
}
// Use aws4 to sign the request so we can talk with ElasticSearch directly
aws4.sign(opts);
const req = https.request(opts, function (res) {
var bodyParts = [];
// We need to read all the incoming data
res.on('data', (chunk) => {
bodyParts.push(chunk)
});
res.on("end", () => {
// We re-create the read content
var body = Buffer.concat(bodyParts).toString()
// We send back uncompressed data
delete(res.headers['content-encoding'])
res.headers['X-Frame-Options'] = 'DENY'
// res.headers['content-security-policy'] = "default-src 'self'; frame-ancestors 'none'"
res.headers['X-XSS-Protection'] = '1; mode=block'
res.headers['X-Content-Type-Options'] = 'nosniff'
var response = {
statusCode: res.statusCode,
body: body,
headers: res.headers
}
lambdaCallback(null, response);
})
});
req.on('error', (e) => {
console.log(`Error caught when calling ElasticSearch: ${e}`)
})
// For POST/PUT request, we send the content of the paylod as well
if ((event.httpMethod == 'POST') || (event.httpMethod == 'PUT')) {
if (event.body) {
var buff = new Buffer(event.body, "base64")
var payload = buff.toString('utf-8')
req.write(payload)
}
}
req.end();
return req;
}
I tried:
Incraesing timeout on request
Increase lambda memory from 128 to 512
Increase lambda timeout
I think you can try is to increase the memory allocation for the Lambda function. it might resolve it.
Or for better and just remove all your headaches. You should use an Amazon API Gateway instead of a reverse proxy in a Lambda function to proxy requests to Kibana. You can also increase the timeout for Kibana requests. this will be more efficient and scalable to your project and less issues.
I try to optimized your code can you check this if it will improve or correct your issue?
const https = require('https');
const aws4 = require('aws4');
function deserializeQueryString(query) {
const params = new URLSearchParams();
for (const key of Object.keys(query)) {
for (const value of query[key]) {
params.append(key, value);
}
}
return params.toString();
}
exports.handler = async (event, context, lambdaCallback) => {
const { headers, path, httpMethod, multiValueQueryStringParameters, body } = event;
// Remove accept-encoding header
delete headers['accept-encoding'];
// Append query string parameters to the path
if (multiValueQueryStringParameters) {
path += '?' + deserializeQueryString(multiValueQueryStringParameters);
}
const opts = {
host: my_es_endpoint,
path,
method: httpMethod,
service: 'es',
region: my_region,
headers
};
// Add request body for POST and PUT methods
if (['POST', 'PUT'].includes(httpMethod) && body) {
opts.body = Buffer.from(body, 'base64').toString('utf-8');
}
// Sign the request using aws4
aws4.sign(opts);
return new Promise((resolve, reject) => {
const req = https.request(opts, res => {
let body = '';
res.on('data', chunk => {
body += chunk;
});
res.on('end', () => {
// Remove content-encoding header
delete res.headers['content-encoding'];
// Set security headers
res.headers['X-Frame-Options'] = 'DENY';
res.headers['X-XSS-Protection'] = '1; mode=block';
res.headers['X-Content-Type-Options'] = 'nosniff';
const response = {
statusCode: res.statusCode,
body,
headers: res.headers
};
resolve(response);
});
});
req.on('error', error => {
console.error(`Error caught when calling ElasticSearch: ${error}`);
reject(error);
});
// Write request body for POST and PUT methods
if (['POST', 'PUT'].includes(httpMethod) && body) {
req.write(opts.body);
}
req.end();
}).then(lambdaCallback)
.catch(error => {
lambdaCallback(error);
});
};
I am not sure if the error is related to the proxy implementation, however I found something that helped and now I am not experiencing the problem anymore.
I reduced the value of discover:sampleSize in Kibana Advanced Settings as suggested here
I am new to using Jest for unit tests. How can I mock this simple http request method "getData"? Here is the class:
const got = require("got")
class Checker {
constructor() {
this.url
this.logData = this.logData.bind(this);
this.getData = this.getData.bind(this);
}
async getData(url) {
const response = await got(url);
const data = await response.body;
return data;
}
async logData(first, second, threshold) {
let data = await this.getData(this.url)
console.log("received " + data.body);
}
}
I am trying to mock "getData" so I can write a unit test for "logData". Do I need to mock out the entire "got" module? Thanks.
If you change invoking got to got.get you should be able to have a working test like so:
const got = require('got');
const Checker = require('../index.js');
describe("some test", () => {
beforeEach(() => {
jest.spyOn(got, 'get').mockResolvedValue({ response: { body: { somekey: "somevalue" } } } );
});
it("works", async () => {
new Checker().getData();
expect(got.get).toBeCalledTimes(1);
})
})
One approach is to use dependency injection. Instead of calling 'got' directly, you can 'ask for it' in the class constructor and assign it to a private variable. Then, in the unit test, pass a mock version instead which will return what you want it to.
const got = require("got");
class Checker {
constructor(gotService) {
this.got = gotService;
this.logData = this.logData.bind(this);
this.getData = this.getData.bind(this);
}
async getData(url) {
const response = await this.got(url);
const data = await response.body;
return data;
}
async logData(first, second, threshold) {
let data = await this.getData(this.url)
console.log("received " + data.body);
}
}
//real code
const real = new Checker(got);
//unit testable code
const fakeGot = () => Promise.resolve(mockedData);
const fake = new Checker(fakeGot);
Here is what we are doing:
'Inject' got into the class.
In the class, call our injected version instead of directly calling the original version.
When it's time to unit test, pass a fake version which does what you want it to.
You can include this directly inside your test files. Then trigger the test that makes the Http request and this will be provided as the payload.
global.fetch = jest.fn(() =>
Promise.resolve({
json: () => Promise.resolve({ data: { eth: 0.6, btc: 0.02, ada: 1 } }),
})
);
it('should return correct mock token values', async () => {
const addresses = ["mockA", "mockB", "mockC"];
const res = await getTokenData(addresses);
expect(res.data).toEqual({ eth: 0.6, btc: 0.02, ada: 1 });
});
I am trying to make a bot with Eris and I need it to be able to call people in direct messaging. This is what I came up with, but clearly, I'm missing something as it does not work.
if (contacts[name].dm) {
const channel = await bot.getDMChannel(contacts[name].id)
await channel.ring([contacts[name].id])
}else{
bot.joinVoiceChannel(contacts[name].id)
}
Might not be the most effective or clean way to do it, but I managed.
function discordAPI(authToken,apiEndpoint,JSONparams,type="GET") {
var xhr=new XMLHttpRequest()
console.log("Connecting to https://discord.com/api/v9"+apiEndpoint)
xhr.open(type,"https://discord.com/api/v9"+apiEndpoint,true)
xhr.setRequestHeader('Content-Type','application/json')
xhr.setRequestHeader('Authorization',authToken)
console.log(authToken)
xhr.onload = function () {
console.log('DONE: ', xhr.status);
console.log('REPONSE: ', xhr.reponseText);
// console.log('=======================');
// console.log('REPONSE_HEADERS:\n', xhr.getAllResponseHeaders());
// console.log('=======================')
};
xhr.send(JSON.stringify(JSONparams))
}
//...
const channel = await bot.getDMChannel(id)
await sleep(time*1000)
discordAPI(bot.token, ('/channels/' + channel.id + '/call/ring'), '', 'POST')
bot.joinVoiceChannel(channel.id).then((vconnect) => { //THIS IS NECESSARY TO JOIN THE CALL!!
console.log(vconnect)
if(sf!='null'){vconnect.play(__dirname + "/../sfx/" + sf + ".mp3")}
vconnect.once("userDisconnect", () => { if(userID == id){bot.switchChannel(null)} });
vconnect.once("end", () => { bot.switchChannel(null) });
vconnect.once("error", () => { console.log("----\n"+Error.toString()+"----\n") });
})
await sleep(timespent*60*1000)
bot.disconnect()
console.log('Called.')
bot.connect()
How do I get the data from a https request outside of its scope?
Update
I've seen Where is body in a nodejs http.get response?, but it doesn't answer this question. In fact, that question isn't answered accurately, either. In the accepted answer (posted by the asker), a third party library is used. Since the library returns an object different from that returned by http.get() it doesn't answer the question.
I tried to set a variable to the return value of http.get() using await, but that returns a http.clientRequest and doesn't give me access to the response data that I need.
I'm using Node v8.9.4 with Express and the https module to request data from Google's Custom Search.
I have two routes. One for a GET request and one for a POST request used when submitting a form on the front page. They both basically serve the same purpose... request the data from CSE and present the data as a simple JSON string. Rather than repeat myself, I want to put my code for the CSE request into a function and just call the function within the callback for either route.
I thought about returning all the way up from the innermost callback, but that won't work because it wouldn't get to the request's error event handler or the necessary .end() call.
Here's a subset of the actual code:
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
res.status(200)
res.set('Content-Type', 'application/json')
// This doesn't work
let searchResults = JSON.stringify(cseSearch(req))
res.end(searchResults)
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams)
}
let cseRequest = https.request(options, cseResponse => {
let jsonString = ''
let searchResults = []
cseResponse.on('data', data => {
jsonString += data
})
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
// This doesn't work... wrong scope, two callbacks deep
return searchResults
})
})
cseRequest.on('error', e => {
console.log(e)
})
cseRequest.end()
}
If you're curious, it's for a freeCodeCamp project: Image Search Abstraction Layer
using promise method solve this issue.
cseSearch(req).then(searchResults=>{
res.end(searchResults)
}).catch(err=>{
res.status(500).end(searchResults)
})
function cseSearch (request) {
return new Promise((resolve, reject)=>{
...your http request code
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
resolve(searchResults);
})
})
}
Based on what I explained in the comments, to give you an idea how compact your code could be using the request-promise library, here's what you could use:
const rp = require('request-promise-native');
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
cseSearch(req).then(results => {
res.json(results);
}).catch(err => {
console.log("newQueryError ", err);
res.sendStatus(500);
});
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams),
json: true
};
return rp(options).then(data => {
return data.items.map(item => {
return {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
};
});
});
I have a use case that needs to use Headless Chrome Network (https://chromedevtools.github.io/devtools-protocol/tot/Network/) to intercept all images requests and find out the image size before saving it (basically discard small images such as icons).
However, I am unable to figure out a way to load the image data in memory before saving it. I need to load it in Img object to get width and height. The Network.getResponseBody is taking requestId which I don't have access in Network.requestIntercepted. Also Network.loadingFinished always gives me "0" in encodedDataLength variable. I have no idea why. So my questions are:
How to intercept all responses from jpg/png request and get the image data? Without saving the file via URL string to the disk and load back.
BEST: how to get image dimension from header response? Then I don't have to read the data into memory at all.
My code is below:
const chromeLauncher = require('chrome-launcher');
const CDP = require('chrome-remote-interface');
const file = require('fs');
(async function() {
async function launchChrome() {
return await chromeLauncher.launch({
chromeFlags: [
'--disable-gpu',
'--headless'
]
});
}
const chrome = await launchChrome();
const protocol = await CDP({
port: chrome.port
});
const {
DOM,
Network,
Page,
Emulation,
Runtime
} = protocol;
await Promise.all([Network.enable(), Page.enable(), Runtime.enable(), DOM.enable()]);
await Network.setRequestInterceptionEnabled({enabled: true});
Network.requestIntercepted(({interceptionId, request, resourceType}) => {
if ((request.url.indexOf('.jpg') >= 0) || (request.url.indexOf('.png') >= 0)) {
console.log(JSON.stringify(request));
console.log(resourceType);
if (request.url.indexOf("/unspecified.jpg") >= 0) {
console.log("FOUND unspecified.jpg");
console.log(JSON.stringify(interceptionId));
// console.log(JSON.stringify(Network.getResponseBody(interceptionId)));
}
}
Network.continueInterceptedRequest({interceptionId});
});
Network.loadingFinished(({requestId, timestamp, encodedDataLength}) => {
console.log(requestId);
console.log(timestamp);
console.log(encodedDataLength);
});
Page.navigate({
url: 'https://www.yahoo.com/'
});
Page.loadEventFired(async() => {
protocol.close();
chrome.kill();
});
})();
This should get you 90% of the way there. It gets the body of each image request. You'd still need to base64decode, check size and save etc...
const CDP = require('chrome-remote-interface');
const sizeThreshold = 1024;
async function run() {
try {
var client = await CDP();
const { Network, Page } = client;
// enable events
await Promise.all([Network.enable(), Page.enable()]);
// commands
const _url = "https://google.co.za";
let _pics = [];
Network.responseReceived(async ({requestId, response}) => {
let url = response ? response.url : null;
if ((url.indexOf('.jpg') >= 0) || (url.indexOf('.png') >= 0)) {
const {body, base64Encoded} = await Network.getResponseBody({ requestId }); // throws promise error returning null/undefined so can't destructure. Must be different in inspect shell to app?
_pics.push({ url, body, base64Encoded });
console.log(url, body, base64Encoded);
}
});
await Page.navigate({ url: _url });
await sleep(5000);
// TODO: process _pics - base64Encoded, check body.length > sizeThreshold, save etc...
} catch (err) {
if (err.message && err.message === "No inspectable targets") {
console.error("Either chrome isn't running or you already have another app connected to chrome - e.g. `chrome-remote-interface inspect`")
} else {
console.error(err);
}
} finally {
if (client) {
await client.close();
}
}
}
function sleep(miliseconds = 1000) {
if (miliseconds == 0)
return Promise.resolve();
return new Promise(resolve => setTimeout(() => resolve(), miliseconds))
}
run();