Synchronous http request in node js? - node.js

I'm looking for a simple way to perform synchronous http-requests in node.js, but it's still getting async responses ...
I've realised that node.js is recommended to async jobs, but in my case,
I need the synchronous response to call other functions that use this data, if it's null/undefined, I can't proceed with the flow...
What's the better way to do that?
Here's my code:
function callCellId(data) {
console.log("Data: " + data);
var towers = [],
rscp = [];
var request = require('sync-request');
for (var x = 0; x < data.length; x++) {
console.log("Request data: \n");
rscp[x] = data[x].rscp;
var res = request('POST', 'http://opencellid.org/cell/get?key=xxxxx&mcc=' + data[x].mcc + '&mnc=' + data[x].mnc + '&lac=' + data[x].LAC + '&cellid=' + data[x].cellID + '&format=json');
console.log("loop " + x);
data = res.getBody().toString();
console.log("rsp: " + data);
towers[x] = {
'latitude': data.lat,
'longitude': data.lon,
'rscp': rscp[x],
'signal': data.averageSignalStrength
};
}
console.log("Content for triangulation" + JSON.stringify(towers));
return towers;
}

Using async in a loop cloud be tricky.
I solved this without external libraries using generators:
LoopOperation: function() {
//define generator with the main loop
var loopIterator = function*() {
for (var index = 0; index < 10; index++) {
var result = yield asyncOperation( (res) => loopIterator.next(res)); //do something asyc and execute () => loopIterator.next() when done as callback
console.log(result);
}
}();
loopIterator.next(); //start loop
}

Since the nodejs nature is async, every time we need some sync call (like this nested request stack), we're able to use promises
"A Promise is an object representing the eventual completion or failure of an asynchronous operation"
reference
I.E:
const request = require('request-promise');
function callCellId(data) {
let towers = [];
let options = {
url: 'http://opencellid.org/cell/get',
method: 'POST',
json: true
};
data.forEach(location => {
options.body = {
key: 'YOUR_PRIVATE_KEY',
mcc: location.mcc,
mnc: location.mnc,
lac: location.lac,
cellId: location.cellID
}
request(options).then(cellInfo => {
towers.push({
latitude: cellInfo.lat,
longitude: cellInfo.lon,
rscp: location.rscp,
signal: cellInfo.averageSignalStrength
});
}).catch(err => {
console.log('Could not get cellId Info for',location);
console.log(err);
});
});
return towers;
}

Related

Simple fetch from API in Nodejs but UnhandledPromiseRejectionWarning: nestedFunction is not a Function

I'm a non-professional using nodejs server (backend) and javascript/html (frontend) to fetch data from two API's: one API gives a response and I use an ID from the first API to fetch data from the other API. The API returns XML so I use XML2Json and JSON.parse to get the Javascript Object.
everything works perfect until I get to the "return nestedFunction(new_details")-function in the second API-call
so this is where the results are sent back to the client
I do it for the first API and it works fine (backend + frontend)
I tried Async/await but the problem isn't solved
I get the error: "UnhandledPromiseRejectionWarning: TypeError: nestedFunction is not a function"
What could be the problem?
app.get('/AATGetTermMatch', function(req,res) {
let termMatch = req.query.term;
let termLogop = req.query.logop;
let termNotes = req.query.notes;
AATGetTermMatch(termMatch, termLogop, termNotes, function (conceptResults) {
res.send(conceptResults);
});
});
function AATGetTermMatch(termMatch, termLogop, termNotes, nestedFunction) {
let URL = baseURL + "AATGetTermMatch?term=" + termMatch + "&logop=" + termLogop + "&notes=" + termNotes;
fetch(URL)
.then(function (response){
return response.text();
}).then(function (response) {
APICallResults = response;
parseJson();
let objectAPI = JSON.parse(APICallResults);
let full_Concepts = objectAPI.Vocabulary.Subject;
let i;
for (i = 0; i < full_Concepts.length; i++) {
global.ID = full_Concepts[i].Subject_ID;
searchTermDetails(global.ID);
} return nestedFunction(full_Concepts);
});
}
app.get('/subjectID', function(req, res) {
let selectedID = req.query.subjectID;
searchTermDetails(selectedID, function (termDetails) {
res.json(termDetails);
});
});
2nd API : http://vocabsservices.getty.edu/AATService.asmx/AATGetSubject?subjectID=300004838
function searchTermDetails(selectedID, nestedFunction) {
selectedID = global.ID;
let URL_Details = baseURL + "AATGetSubject?" + "subjectID=" + selectedID;
fetch(URL_Details)
.then(function (response) {
return response.text();
}).then(function (response) {
APICallResults_new = response;
parseJsonAgain();
let detailAPI = JSON.parse(APICallResults_new);
let new_details = detailAPI.Vocabulary.Subject;
let Subject_ID = new_details[0].Subject_ID;
let descriptive_Notes_English = new_details[0].Descriptive_Notes[0].Descriptive_Note[0].Note_Text;
} **return nestedFunction(new_details);**
}).catch(function (error) {
console.log("error");
});
}
function parseJson() {
xml2js.parseString(APICallResults, {object: true, trim:true, sanitize: true, arrayNotation: true, mergeAttrs: true}, (err, result) => {
if (err) {
throw err;
}
const resultJson = JSON.stringify(result, null, 4);
//JSON.parse(resultJson);
APICallResults = resultJson;
});
}
function parseJsonAgain() {
xml2js.parseString(APICallResults_new, {object: true, trim:true, sanitize: true, arrayNotation: true, mergeAttrs: true}, (err, result) => {
if (err) {
throw err;
}
const resultJsonAgain = JSON.stringify(result, null, 4);
APICallResults_new = resultJsonAgain;
//console.log(APICallResults_new);
});
}
I've read many threads about this error but the proposed solutions don't seem to work.
In here:
for (i = 0; i < full_Concepts.length; i++) {
global.ID = full_Concepts[i].Subject_ID;
searchTermDetails(global.ID);
}
where you call searchTermDetails(), you are not passing the nestedFunction second argument. Thus, when searchTermDetails() tries to use that argument, it causes the error you get.
You can either add the callback to this call or, if the callback could be optional, then you can modify searchTermDetails to check to see if the second argument is a function and, if not, then don't try to call it.

Resolving a recursive socket.on("data") call

I have a node socket server that needs to receive data more than once during a single socket.on('data') call. It's sorta like socket.on('data') is running recursively. I think I've just configured this really really wrong, but I don't know enough NodeJS to figure it out.
Here's an example of the issue:
const net = require("net");
const socket = new net.Socket();
const port = process.argv[2];
socket.connect(port, "127.0.0.1", () => {
// We communicate with another server, always with "type" -> "result" messages.
socket.write(JSON.stringify({ type: "connect", result: "JavaScript" }));
});
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
// This is the critical line that I don't know how to resolve.
// Right at this point I need wait for another socket.on('data') call
return socket.giveMeThatQuery()
}
socket.on("data", (data) => {
let jsonData = JSON.parse(data);
if (jsonData["type"] == "call_func") {
let funcToCall = func_reg[jsonData["result"]];
// This will need to call get_query 0-n times, dynamically
result = funcToCall();
let finishMessage = JSON.stringify({
type: "finish_func",
result,
});
socket.write(finishMessage);
} else if (jsonData["type"] == "query_response") {
// This has to connect back to get_query somehow?
// I really have no idea how to approach this.
// async functions? globals?
giveDataToQuery()
}
});
function func1() {
return "bird"; // some funcs are simple
}
function func2() {
// some are more complicated
let data = get_query("Need " + get_query("some data"));
return "cat" + data;
}
function func3() {
let data = get_query("Need a value");
let extra = '';
if (data == "not good") {
extra = get_query("Need more data");
}
return data + "dog" + extra;
}
var func_reg = { func1, func2, func3};
This server is just an example, I can edit or provide more context if it's needed. I suppose if you're looking at this question right after it's posted I can explain more on the live stream.
Edit: adding an example of using globals and while true (this is how I did it in Python). However, in Python, I also put the entire funcToCall execution on another thread, which it doesn't seem like is possible in Node. I'm beginning to think this is actually just impossible, and I need to refactor the entire project design.
let CALLBACK = null;
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
while (true) {
if (CALLBACK) {
let temp = CALLBACK;
CALLBACK = null;
return temp;
}
}
...
} else if (jsonData["type"] == "query_response") {
CALLBACK = jsonData["result"];
}
I was correct, what I wanted to do was impossible. Node (generally) only has one thread, ever. You can only be waiting in one place at a time. Async threads or callbacks doesn't solve this, since the callback I would need is another socket.on, which makes no sense.
The solution is yield. This allows me to pause the execution of a function and resume at a later time:
function* func2() {
let data = (yield "Need " + (yield "some data"));
yield "cat" + data;
yield 0;
}
function* func3() {
let data = (yield "Need a value");
let extra = '';
if (data == "not good") {
extra = (yield "Need more data");
}
yield data + "dog" + extra;
yield 0;
}
I updated the server.on to handle these generator function callbacks and respond differently to the yield 0;, which indicates the function has no queries. Here's the final server.on.
server.on("data", (data) => {
try {
let jsonData = JSON.parse(data);
let queryResult= null;
let queryCode = null;
if (jsonData["type"] == "call_func") {
var script = require(jsonData["func_path"]);
FUNC_ITER = script.reserved_name();
} else if (jsonData["type"] == "queryResult") {
queryResult = jsonData["result"];
}
queryCode = FUNC_ITER.next(queryResult);
if (queryCode.value == 0) {
let finishMessage = JSON.stringify({
type: "finish_func",
result: null,
});
server.write(finishMessage);
} else {
let exeMessage = JSON.stringify({
type: "queryCode",
result: queryCode.value,
});
server.write(exeMessage);
}
} catch (err) {
crashMessage = JSON.stringify({
type: "crash",
result: err.stack,
});
server.write(crashMessage);
}
});
And an example of the node files I generated containing the custom functions:
function* reserved_name() {
(yield "return " + String("\"JavaScript concat: " + (yield "value;") + "\"") + ";")
yield 0;
}
module.exports = { reserved_name };
If you're wondering, this is for a language called dit, which can execute code in other languages. There's a Python server that talks to these other languages. Here's an example I finished last week which uses Python, Node, and Lua as the guest languages:

Using node.js for-loop index in a coinbase-api callback function

I am new to node.js and i am trying to make a simple script that will connect to the coinbase-api and get the current price of whatever markets are defined in the MARKET array.
The problem i am having is that the for-loop that iterates through the array is asynchronous and the callback function is not getting the correct index value for the array.
The two main solutions i have found are to use promises or force the loop to wait. I think i need to be using promises rather than forcing the for loop to wait but honestly i have failed to implement a solution either way. I have found may example of promises but i just cant seem to figure out how to implement them into my script. I would appreciate any help.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
let tickerCount = null;
const getCallback = (error, response, data) =>
{
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
}
}
setInterval(() =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
publicClient.getProductTicker(MARKET[tickerCount], getCallback);
}
}, 10000);
I was able to figure out how to use promises with trial and error from the helpful examples on the Mozilla Developer Network. I am sure i am making some mistakes but at least it is working now. Another little bonus is that i was able to remove a global.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
function getProductTicker(tickerCount) {
return new Promise(resolve => {
publicClient.getProductTicker(MARKET[tickerCount],function callback(error, response, data){
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
resolve();
}
});
});
}
setInterval( async () =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (var tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
await getProductTicker(tickerCount);
}
}, 10000);

Get express/node to loop through request sent to NOAA API

So I am making a kind of API middleware for my company that will grab information from the NOAA API and then store in in my database. It does more then but that a separate part. I have set it up so that it works it will get the information and store it in my sql database perfectly The issue is the information I get is based off of zipcode. One request is the information for one zipcode. I need to be able to 'loop" through a list of zipcode one at a time and store the information in the database. I am not sure how to properly get it to work. I have tested a couple of ways but have not been able to get it to work so if someone can get me pointed in the right direction it would be appreciated.
Sorry in advance my code is not cleaned up.
Everything below apiRequest.end() has little function for the question. I keep it for context.
let mysql = require('mysql');
let config = require('./config.js');
var https = require("https");
var express = require("express");
var app = express();
const port = 3000;
var fs= require('fs');
var csv = require('fast-csv');
//last test
//array will replace this zip variable
let zip = '90012';
api(zip);
function api(zips){
//All of the parts for building the get requests url
app.get("/", function(req, response) {
var apiKey = "gPaEVizejLlbRVbXexyWtXYkfkWkoBhd";
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid="datasetid=GHCND";
let datatypeid="&datatypeid=TMAX";
let location="&locationid=ZIP:";
const zipcode = zips;
let startdate="&startdate=2019-01-01";
let enddate="&enddate=2020-01-01";
let units = "&units=standard";
let limit="&limit=1000";
let url = webapi + datasetid + datatypeid + location + zipcode + startdate + enddate + units + limit;
var options = {
port: 443,
method: "GET",
headers: {
"token": apiKey
}
};
let data = "";
//request to grab from NOAA api
let apiRequest = https.request(url, options, function(res) {
console.log("Connected");
//grabing all data
res.on("data", chunk => {
data += chunk;
});
res.on("end", () => {
console.log("data collected");
//Format JSON data
response.send(JSON.parse(data));
var getData = JSON.parse(data);
if(isEmpty(getData)){
emptyCorrect();
}
dataFormat(getData);
});
});
apiRequest.end();
});
//fix format for date Can add more formating if needed here
function dataFormat(formData){
for(x in formData.results){
let date = formData.results[x].date;
formData.results[x].date = date.slice(0,10);
}
jsonToSQL(formData.results);
}
//test function is going to be used for inserting the zip
function test(){
var content = "";
console.log("your test worked see ***************");
return "92507";
}
//function to add grabed JSON data into the SQL database
function jsonToSQL(datafin){
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let values = [];
for(let x in datafin){
values.push([zipcode,datafin[x].date,datafin[x].value]);
}
// execute the insert statment
connection.query(stmt, [values], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function emptyCorrect(){
console.log("Eror correction");
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let valueE = [];
valueE.push([zipcode,"0","No Data"]);
// execute the insert statment
connection.query(stmt, [valueE], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function isEmpty(obj) {
for(var key in obj) {
if(obj.hasOwnProperty(key))
return false;
}
return true;
}
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
}
As I understand your problem can roughly be summarized as "How to loop through asynchronous evaluations in Nodejs".
There are some options for you. I would recommend wrapping call to the NOAA API with a promise and then chain those promises. This can be done as follows:
app.get('/', async function(req, response) {
var apiKey = 'some value';
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid = 'datasetid=GHCND';
let datatypeid = '&datatypeid=TMAX';
let location = '&locationid=ZIP:';
let startdate = '&startdate=2019-01-01';
let enddate = '&enddate=2020-01-01';
let units = '&units=standard';
let limit = '&limit=1000';
var options = {
port: 443,
method: 'GET',
headers: {
token: apiKey
}
};
const zipCodes = ['90012', '90013']; // Place a call to your function for fetching zip codes here
let datas = [];
prom = Promise.resolve();
zipCodes.forEach(zipcode => {
prom = prom.then(() =>
new Promise((resolve, reject) => {
let url =
webapi +
datasetid +
datatypeid +
location +
zipcode +
startdate +
enddate +
units +
limit;
let apiRequest = https.request(url, options, function(res) {
console.log('Connected');
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
console.log('data collected for zip ' + zipcode);
datas.push(data);
resolve();
});
});
apiRequest.end();
})
);
});
prom.then(() => {
// All requests have now been handled sequentially
response.send(/* You'll need to figure out what to do here */);
});
});
An alternative is to use something like the async library for dealing with sequentially calling callbacks. The async library (https://github.com/caolan/async) describes itself as:
Async is a utility module which provides straight-forward, powerful functions for working with asynchronous JavaScript.
See e.g. Node.js: How do you handle callbacks in a loop? for a similar problem (not with regards to callign an API, but dealing with asynchronous function in a loop).

Node.js how to iterate with async callback?

What I am doing is using fs to stitch 5 pieces of html-page parts(html, head, chead, topaside, main, footer) together. The file name is htmlpage.js, so you can just run node htmlpage.js file1 file2 file3 ... in command line tool, and it will stitch those html-page parts together, then spit out file1.html, file2.html, file3.html .... I don't like to use template engine/library/framework or whatever, especially when I am learning.
Here is the sorce code:
'use strict';
const fs = require('fs'),
head = fs.createReadStream('./html-parts/head.html', 'utf8'),
topaside = fs.createReadStream('./html-parts/topaside.html', 'utf8'),
footer = fs.createReadStream('./html-parts/footer.html', 'utf8');
let name = process.argv.slice(2),
htmlray = [],
ni = 0,
nl = name.length;
for (ni; ni < nl; ni ++) {
let cheadP = './html-parts/' + name[ni] + '-head.html',
mainP = './html-parts/' + name[ni] + '-main.html',
htmlP = name[ni] + '.html',
chead = fs.createReadStream(cheadP, 'utf8'),
main = fs.createReadStream(mainP, 'utf8'),
html = fs.createWriteStream(htmlP, 'utf8');
//let those parts form an array
htmlray = [html, head, chead, topaside, main, footer];
openendPipe(htmlray[1], htmlray[0]);
htmlray[1].on('end', () => {
openendPipe(htmlray[2], htmlray[0]);
htmlray[2].on('end', () => {
openendPipe(htmlray[3], htmlray[0]);
htmlray[3].on('end', () => {
openendPipe(htmlray[4], htmlray[0]);
htmlray[4].on('end', () => {
htmlray[5].pipe(htmlray[0]);
htmlray[5].on('end', () => {
console.log(name + '.html' + ' created');
});
});
});
});
});
}
function openendPipe(src, dst) {
return src.pipe(dst, {end: false});
}
But what if the htmlray has 100 parts, I want to be able to do an iteration to replace these code, let's call it pipeblock:
openendPipe(htmlray[1], htmlray[0]);
htmlray[1].on('end', () => {
openendPipe(htmlray[2], htmlray[0]);
htmlray[2].on('end', () => {
openendPipe(htmlray[3], htmlray[0]);
htmlray[3].on('end', () => {
openendPipe(htmlray[4], htmlray[0]);
htmlray[4].on('end', () => {
htmlray[5].pipe(htmlray[0]);
htmlray[5].on('end', () => {
console.log(name + '.html' + ' created');
});
});
});
});
});
I tried these solutions, they didn't work:
Solution 1:
(function () {
let i = 0, count = 1;
function nextpipe() {
let arr = arguments[0];
i ++;
if (count > 5) return;
openendPipe(arr[i], arr[0]);
count ++;
arr[i].on('end', nextpipe);
}
return nextpipe;
})();
//then replace 'pipeblock' with 'nextpipe(htmlray)';
//console.log: nextpipe is undefined.
Solution 2:
//replace 'pipeblock' with these code
let pi = 1,
pl = htmlray.length - 1;
htmlray[pi].pipe(htmlray[0], {end: false});
htmlray[pi].on('end', nextpipe);
function nextpipe() {
if (pi > pl) return console.log(name + '.html' + ' created');;
pi ++;
htmlray[pi].pipe(htmlray[0], {end: false});
htmlray[pi].on('end', nextpipe);
}
//cosole.log:
//htmlray[pi].pipe(htmlray[0], {end: false});
//TypeError: Cannot read property 'pipe' of undefined
This thing calls "callback hell" and you should use either some library to handle async calls like async.js or (better) use promises.
Just simply promisify fs.readFile like this (or write your own promise for fs.createReadStream it you want to use it)
const readFile = Promise.promisify(require('fs').readFile);
and then combine your request promises using Promise.all()
Here are examples http://bluebirdjs.com/docs/api/promise.promisify.html, http://bluebirdjs.com/docs/api/promise.all.html for fs for Bluebird promise library.
While I am reading through async and promise documentation, I get to the part about parallel, series and waterfall. My problem is about creating an html page, so it can't be done in parallel. But the documentation starts and talks a lot about parallel, and they just add confusion to my head. To grasp all of them is probably going to take a long time. Anyway, while I am experimenting, I come up with an easy solution of my own.
In my problem, the repetitive part is about check if previous html-part has done writing by using readingStream.on('end', <do next writing>);, so I make it into a function:
function ifend(src, src2, dst) {
return src.on('end', () => {return openendPipe(src2, dst);});
}
Then I can turn pipeblock into:
openendPipe(htmlray[0], html);
ifend(htmlray[0], htmlray[1], html);
ifend(htmlray[1], htmlray[2], html);
ifend(htmlray[2], htmlray[3], html);
ifend(htmlray[3], htmlray[4], html);
Then I can do an iteration in a function:
function createHtml(src, dst) {
let l = src.length - 1,
i = 0;
openendPipe(src[0], dst);
for (i; i < l; i ++) {
//iterate ifend function.
ifend(src[i], src[i + 1], dst);
}
return dst;
}
Now I can replace pipeblock with this:
createHtml(htmlray, html);

Resources