Variable inside readline.createInterface({}).on({}) is not updating - node.js

Question

Trying to construct a request body with values from a .csv file, but my points array is not getting updated. I've done console.log() for the points array inside and at the end of the on() function, and the values are there as expected. But I console.log outside of the on() and it says the array is empty.
I know this is a scope issue, but not familiar with Node.js so much, so can't figure it out. Thanks.
// Detect anomalies in your spreadsheet
var points = []
var dict = {}
// Read the .csv file, convert date to Date object and number to float.
readline.createInterface({
input: fs.createReadStream(CSV_FILE),
terminal: false
}).on('line', function(line) {
var row = line.split(",")
dict[new Date(row[0])] = parseFloat(row[1])
points.push(dict)
});
// Create request body for API call
let body = { series: points, granularity: 'hourly' }
// Make the call
anomalyDetectorClient.entireDetect(body)
.then((response) => {
for (item in response.isAnomaly) {
if (item) {
console.log("An anomaly was detected from the series.")
console.log("Value: " + response.expectedValues[response.indexOf(item)])
}
}
}).catch((error) => {
console.log(error)
})
UPDATE:
I'm trying to move the async function into the on(), so the points[] has values in it...but there is a bad body in the request, my points array is taking an accumulative dictionary instead of a new dictionary each time it pushes. So that needs to be fixed first, before I can look again at the un-updated points[] outside of the last on().
Current code, I moved the API call into the last on() function (previously it was outside of it, with an await before it):
async function main() {
let CSV_FILE = './request-data.csv'
// Detect anomalies in your spreadsheet
var points = []
// Read the .csv file
await readline.createInterface({
input: fs.createReadStream(CSV_FILE),
terminal: false
}).on('line', function(line) {
var row = line.split(",")
// Convert date to Date object and number to float.
var date = new Date(row[0])
var dict = {}
dict[date] = parseFloat(row[1])
points.push(dict)
console.log(points)
}).on('close', function() {
// Create request body for API call
let body = { series: points, granularity: 'hourly' }
console.log("Body series: " + body.series.timestamp + "Body granularity: " + body.granularity)
// Make the call
anomalyDetectorClient.entireDetect(body)
.then((response) => {
for (item in response.isAnomaly) {
if (item) {
console.log("An anomaly was detected from the series.")
console.log("Value: " + response.expectedValues[response.indexOf(item)])
}
}
}).catch((error) => {
console.log(error)
})
})
UPDATE 2:
Using new code in the answer, there is a new error:
Body series: undefinedBody granularity: hourly
{ Error: Time points should be uniformly spaced in time in hourly granularity with 1 gran as interval, ratio of missing points should be less than 10%, between 2018-03-01 00:00:00 and 2018-04-16 00:00:00 there should be at least 994 points, but got 47 points. What does this means?
Confused because the body object looks good, it was printed with the error:
{ Error: Time points should be uniformly spaced in time in hourly granularity with 1 gran as interval, ratio of missing points should be less than 10%, between 2018-03-01 00:00:00 and 2018-04-16 00:00:00 there should be at least 994 points, but got 47 points.
at new RestError (C:\Users\v-wiazur\Desktop\Anomaly Detector\node_modules\#azure\ms-rest-js\dist\msRest.node.js:1399:28)
at C:\Users\v-wiazur\Desktop\Anomaly Detector\node_modules\#azure\ms-rest-js\dist\msRest.node.js:2494:37
at process._tickCallback (internal/process/next_tick.js:68:7)
code: 'InvalidSeries',
statusCode: 400,
request:
WebResource {
streamResponseBody: false,
url:
'https://winonaanomalydetector.cognitiveservices.azure.com/anomalydetector/v1.0/timeseries/entire/detect',
method: 'POST',
headers: HttpHeaders { _headersMap: [Object] },
body:
'{"series":[{"timestamp":"2018-03-01T00:00:00.000Z","value":32858923},{"timestamp":"2018-03-02T00:00:00.000Z","value":29615278},{"timestamp":"2018-03-03T00:00:00.000Z","value":22839355},{"timestamp":"2018-03-04T00:00:00.000Z","value":25948736},{"timestamp":"2018-03-05T00:00:00.000Z","value":34139159},{"timestamp":"2018-03-06T00:00:00.000Z","value":33843985},{"timestamp":"2018-03-07T00:00:00.000Z","value":33637661},{"timestamp":"2018-03-08T00:00:00.000Z","value":32627350},{"timestamp":"2018-03-09T00:00:00.000Z","value":29881076},{"timestamp":"2018-03-10T00:00:00.000Z","value":22681575},{"timestamp":"2018-03-11T00:00:00.000Z","value":24629393},{"timestamp":"2018-03-12T00:00:00.000Z","value":34010679},{"timestamp":"2018-03-13T00:00:00.000Z","value":33893888},{"timestamp":"2018-03-14T00:00:00.000Z","value":33760076},{"timestamp":"2018-03-15T00:00:00.000Z","value":33093515},{"timestamp":"2018-03-16T00:00:00.000Z","value":29945555},{"timestamp":"2018-03-17T00:00:00.000Z","value":22676212},{"timestamp":"2018-03-18T00:00:00.000Z","value":25262514},{"timestamp":"2018-03-19T00:00:00.000Z","value":33631649},{"timestamp":"2018-03-20T00:00:00.000Z","value":34468310},{"timestamp":"2018-03-21T00:00:00.000Z","value":34212281},{"timestamp":"2018-03-22T00:00:00.000Z","value":38144434},{"timestamp":"2018-03-23T00:00:00.000Z","value":34662949},{"timestamp":"2018-03-24T00:00:00.000Z","value":24623684},{"timestamp":"2018-03-25T00:00:00.000Z","value":26530491},{"timestamp":"2018-03-26T00:00:00.000Z","value":35445003},{"timestamp":"2018-03-27T00:00:00.000Z","value":34250789},{"timestamp":"2018-03-28T00:00:00.000Z","value":33423012},{"timestamp":"2018-03-29T00:00:00.000Z","value":30744783},{"timestamp":"2018-03-30T00:00:00.000Z","value":25825128},{"timestamp":"2018-03-31T00:00:00.000Z","value":21244209},{"timestamp":"2018-04-01T00:00:00.000Z","value":22576956},{"timestamp":"2018-04-02T00:00:00.000Z","value":31957221},{"timestamp":"2018-04-03T00:00:00.000Z","value":33841228},{"timestamp":"2018-04-04T00:00:00.000Z","value":33554483},{"timestamp":"2018-04-05T00:00:00.000Z","value":32383350},{"timestamp":"2018-04-06T00:00:00.000Z","value":29494850},{"timestamp":"2018-04-07T00:00:00.000Z","value":22815534},{"timestamp":"2018-04-08T00:00:00.000Z","value":25557267},{"timestamp":"2018-04-09T00:00:00.000Z","value":34858252},{"timestamp":"2018-04-10T00:00:00.000Z","value":34750597},{"timestamp":"2018-04-11T00:00:00.000Z","value":34717956},{"timestamp":"2018-04-12T00:00:00.000Z","value":34132534},{"timestamp":"2018-04-13T00:00:00.000Z","value":30762236},{"timestamp":"2018-04-14T00:00:00.000Z","value":22504059},{"timestamp":"2018-04-15T00:00:00.000Z","value":26149060},{"timestamp":"2018-04-16T00:00:00.000Z","value":35250105}],"granularity":"hourly"}',
query: undefined,
formData: undefined,
withCredentials: false,
abortSignal: undefined,
timeout: 0,
onUploadProgress: undefined,
onDownloadProgress: undefined,
proxySettings: undefined,
operationSpec:
{ httpMethod: 'POST',
path: 'timeseries/entire/detect',
urlParameters: [Array],
requestBody: [Object],
responses: [Object],
serializer: [Serializer] } },
response:
{ body:
'{"code":"InvalidSeries","message":"Time points should be uniformly spaced in time in hourly granularity with 1 gran as interval, ratio of missing points should be less than 10%, between 2018-03-01 00:00:00 and 2018-04-16 00:00:00 there should be at least 994 points, but got 47 points."}\n',
headers: HttpHeaders { _headersMap: [Object] },
status: 400 },
body:
{ code: 'InvalidSeries',
message:
'Time points should be uniformly spaced in time in hourly granularity with 1 gran as interval, ratio of missing points should be less than 10%, between 2018-03-01 00:00:00 and 2018-04-16 00:00:00 there should be at least 994 points, but got 47 points.' } }

Answer 1

Your array is getting updated. If you do console.log(points) inside your line event handler, you will see it accumulate date each time.
The issue is that you are looking at the array in the wrong place, before it has been populated. The readline interface is asynchronous. So, you call readline.createInterface() and set your event handler for the line event and then your code just keeps on running. You are immediately calling anomalyDetectorClient.entireDetect() before the readline operation has done it's job.
The solution is to wait until the readline interface is done reading the whole file and until your points array is fully populated before you try to use the points array.
Here's one way to do that, where you process the points array in the close event handler for the readline interface:
// Detect anomalies in your spreadsheet
var points = [];
var dict = {};
// Read the .csv file, convert date to Date object and number to float.
readline.createInterface({
input: fs.createReadStream(CSV_FILE),
terminal: false
}).on('line', function(line) {
var row = line.split(",");
let point = {
timestamp: new Date(row[0]),
value: parseFloat(row[1])
};
dict[point.date] = point.value;
points.push(point);
console.log(points);
}).on('close', function() {
// now we're done reading the file
console.log(points); // all the points are here
// now process the points
// Create request body for API call
let body = { series: points, granularity: 'hourly' };
// Make the call
anomalyDetectorClient.entireDetect(body).then((response) => {
for (let item in response.isAnomaly) {
if (item) {
console.log("An anomaly was detected from the series.")
console.log("Value: " + response.expectedValues[response.indexOf(item)])
}
}
}).catch((error) => {
console.log(error)
});
}).on('error', function(err) {
// handle errors here
console.log(err);
});
FYI, are you really sure you should be using item in response.isAnomaly? in is NOT for arrays, it's for properties of objects. It sorta, kinda works for arrays, but it can also not work lots of times. for/of is built specifically for arrays as in for (let item of response.isAnomaly) {...}. Also, for an array, in gets you the index of the value, not the array value. of gets you the array value which is usually what you want.