Request JSON within a loop - node.js

Question

I'm having trouble to surf correctly between objects that are been returned by an API request.
Basically, I have an Array (tickerArr with 25 elements that I get from another request) which I use for a forEach loop for another request.
Here's what the code looks like:
var request = require('request');
var async = require('async');
var fs = require('fs')
var getTickerList = require('./getTickerList').getTickerList;
var unixTimeStamp = new Date().toISOString();
async function getTickerInfo() {
var tickerArr = await getTickerList;
var arrLength = tickerArr.length;
var init = 0;
console.log(`${arrLength} to process to the batchfile...`);
var getTickerInfo = new Promise((resolve, reject) => {
async.forEach(tickerArr, (item, callback) => {
request({
url:`https://api.kraken.com/0/public/Ticker?pair=${item}`,
json: true
}, (error, response, body) => {
var tickerInfo = JSON.stringify(body.result)
fs.appendFile(`./modules/batches/${unixTimeStamp}batch.json`, tickerInfo, (err) => {
if (err) throw err;
init ++;
var progress = Math.round(init / arrLength * 100);
console.log(`[${progress}%] - ${item} successfully added!`);
});
resolve();
})
});
});
}
getTickerInfo()
Unfortunately, and even if the request works correctly, the objects being returned have a specific path:
Error: []
Result:
Pair(x):
a: [1, 2, 3]
b: [1, 2, 3] ect...
You can find an exact example of the information being returned from the request here:
{
"error": [],
"result": {
"XXBTZEUR": {
"a": ["2212.31000", "3", "3.000"],
"b": ["2212.21100", "1", "1.000"],
"c": ["2212.31000", "0.15800000"],
"v": ["4999.06419498", "9993.55448658"],
"p": ["2206.04624", "2181.36028"],
"t": [15065, 29524],
"l": ["2167.00000", "2122.92000"],
"h": ["2239.00000", "2239.00000"],
"o": "2184.99000"
}
}
}
The only problem is: I would like to 'transform' the object I get from the request, to another object (just arranging stuff around, most of all: put the 'Pair(x)' attribute as a value for a name key) but since I don't know the pair in advance (the values in my tickerArray are enough to make the request but do not apparently correspond to reference the object), I can't access the information contained within result to manipulate it.
Anybody have any idea?
Thanks in advance!

Answer 1

Your question has multiple levels and maybe it's best to break up the task into its constituent parts first.
Calling the Kraken REST API and unwrapping the response
Creating small, composable functions
Transforming an object into a new layout
Writing the results to a file
0. Preface
Since you want to work with await, the first thing to know is that this is syntactical sugar for "wait for promise resolution". async is the keyword that enables await semantics inside a function.
So given this promise-returning function:
function foo() {
return Promise.resolve('some value');
}
this:
async function worker() {
var fooResult = await foo();
console.log(fooResult);
}
worker();
and this:
foo().then(function worker(fooResult) {
console.log(fooResult);
});
are effectively the same thing. (As always, it's a little more complicated than that, but that's the gist of it.)
Since we are working with promises here, the most sensible thing to do is to use libraries that also use promises. request has the request-promise counter-part, so let's use that.
var request = require('request-promise');
1. Calling the Kraken REST API and unwrapping the response
Making an HTTP request and breaking up the response (err and result parts) works the same for all API endpoints, so we can write a function that handles this task.
function getFromApi(endpoint, params) {
return request({
url: "https://api.kraken.com/0/public/" + endpoint,
qs: params,
json: true
}).then(function (data) {
if (data.error && data.error.length) {
throw new Error(`API error xyz for ${endpoint} with params ...`);
}
return data.result;
});
}
Users of this function will not have to deal with err at all and have access to result directly.
2. Creating small, composable functions
Our getFromApi() function returns a promise, therefore we can re-use it in wrappers for various API endpoints:
function getTickerList() {
return getFromApi("TickerList"); // ...or something like that
}
function getSingleTicker(pair) {
return getFromApi("Ticker", {pair: pair}).then(transformTicker);
}
function getTickerInfo(tickerList) {
return Promise.all(tickerList.map(getSingleTicker));
}
Note that, like in getFromApi(), we can use Promise#then to modify the overall output of an operation.
getTickerInfo() accepts an array of ticker names. It uses Array#map to run all API requests in parallel, and Promise#all to allow awaiting the overall result. The fairly complex operation of fetching and transforming multiple results in parallel composes into a pretty straight-forward one-liner.
3. Transforming an object into a new layout
transformTicker() is meant to accept an object in form {"XXBTZEUR": {"a": [...], ... }} and return a transformed variant of it.
function transformTicker(ticker) {
var result = {};
// ...massage ticker into desired form
Object.keys(ticker).forEach( pair => {
Object.keys(ticker[pair]).forEach( k => {
result[k] = { /* ... */ };
});
});
return result;
}
4. Writing the results to a file
Appending to a JSON file does not work. JSON can only be read and written as a whole. Let's fetch a list of ticker names, the associated tickers and write the results to a file.
async function worker() {
var tickerList = await getTickerList();
var tickerInfo = await getTickerInfo(tickerList);
var filename = `./modules/batches/${new Date().toISOString()}batch.json`;
fs.writeFile(filename, JSON.stringify(tickerInfo), (err) => {
if (err) throw err;
console.log("Done");
});
}
worker();
You can switch use a promisified version (see Bluebird#promisifyAll) of the fs module, since the plain version breaks the nice async/await semantics again by requiring a continuation callback.
With a promisified fs module, the worker could look like this:
async function worker() {
var tickerList = await getTickerList();
var tickerInfo = await getTickerInfo(tickerList);
var filename = `./modules/batches/${new Date().toISOString()}batch.json`;
await fs.writeFileAsync(filename, JSON.stringify(tickerInfo));
console.log("Done");
}