How do I get data out of a Node http(s) request? - node.js

Question

How do I get the data from a https request outside of its scope?
Update
I've seen Where is body in a nodejs http.get response?, but it doesn't answer this question. In fact, that question isn't answered accurately, either. In the accepted answer (posted by the asker), a third party library is used. Since the library returns an object different from that returned by http.get() it doesn't answer the question.
I tried to set a variable to the return value of http.get() using await, but that returns a http.clientRequest and doesn't give me access to the response data that I need.
I'm using Node v8.9.4 with Express and the https module to request data from Google's Custom Search.
I have two routes. One for a GET request and one for a POST request used when submitting a form on the front page. They both basically serve the same purpose... request the data from CSE and present the data as a simple JSON string. Rather than repeat myself, I want to put my code for the CSE request into a function and just call the function within the callback for either route.
I thought about returning all the way up from the innermost callback, but that won't work because it wouldn't get to the request's error event handler or the necessary .end() call.
Here's a subset of the actual code:
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
res.status(200)
res.set('Content-Type', 'application/json')
// This doesn't work
let searchResults = JSON.stringify(cseSearch(req))
res.end(searchResults)
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams)
}
let cseRequest = https.request(options, cseResponse => {
let jsonString = ''
let searchResults = []
cseResponse.on('data', data => {
jsonString += data
})
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
// This doesn't work... wrong scope, two callbacks deep
return searchResults
})
})
cseRequest.on('error', e => {
console.log(e)
})
cseRequest.end()
}
If you're curious, it's for a freeCodeCamp project: Image Search Abstraction Layer

Answer 1

using promise method solve this issue.
cseSearch(req).then(searchResults=>{
res.end(searchResults)
}).catch(err=>{
res.status(500).end(searchResults)
})
function cseSearch (request) {
return new Promise((resolve, reject)=>{
...your http request code
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
resolve(searchResults);
})
})
}

Answer 2

Based on what I explained in the comments, to give you an idea how compact your code could be using the request-promise library, here's what you could use:
const rp = require('request-promise-native');
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
cseSearch(req).then(results => {
res.json(results);
}).catch(err => {
console.log("newQueryError ", err);
res.sendStatus(500);
});
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams),
json: true
};
return rp(options).then(data => {
return data.items.map(item => {
return {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
};
});
});