I am trying to write a nodejs program. I have a file data.json , it contains json objects. For every object i have to add another key as review ,value as a text file data. here , I wrote code for reading data from json file, in that file , for every object I inserted key and value pairs. and stored in a array named 'matter', In below code, I used callback to post data to calling function. but callback is executing before 'for loop' in Fetchdata function. How to call callback after for loop.
var fs = require('fs');
var jf = require('jsonfile')
var file = '../data/data.json'
function readfile(str,callback) {
fs.readFile(str, function (err, data) {
callback && callback(data.toString());
});
}
function Fetchdata(callback) {
var matter = [];
jf.readFile(file, function (err, jsonData) {
var j=0;
for (var i = 0; i < jsonData.length; ++i) {
obj = jsonData[i];
var purchase_url = obj["purchase_url"];
if (purchase_url.indexOf("flipkart") > -1) {
var ss = purchase_url.split("pid=");
if (ss[1]) {
var s2 = ss[1].split('&');
readfile(s2[0],function(some){
"use strict";
obj["review"]= some;
matter.push(obj);
})
}
}
}
callback && callback(matter);
});
}
Fetchdata(function (some) {
console.log(some[0]);
});
You can use sync version of readFile
function readfile(filename,callback) {
callback(fs.readFileSync(filename).toString())
}
The sync version of readFile, will not continue to the next line. It will return the content of the file.
You can also, do it in one line fs.readFileSync(str,'utf-8') instead of using toString.
More info
readFileSync API
Synchronous vs Asynchronous code with Node.js
Related
I tried to make the function async but when I print the attacks it prints out {} without anything in it but when I print the values right after adding them in attacks I can print them why is it like that? how can I use the value?
var fs = require('fs');
var http = require('http');
var attacks = {};
var phase_name;
var directory = 'cti-master\\enterprise-attack\\attack-pattern\\';
// getting all files names.
async function getData(directory){
fs.readdir(directory, (err, files) => {
if(err) { return;}
var fileNum = 0;
// opening all the files and sorting the data in them.
while (fileNum < files.length - 1)
{
fs.readFile(directory + files[fileNum], 'utf8', (err, data) =>
{
// parsing the data from json.
var fileData = JSON.parse(data);
// sometimes there is no phase name.
if(fileData['objects'][0]['kill_chain_phases'] == undefined){phase_name = undefined;}
else{phase_name = fileData['objects'][0]['kill_chain_phases'][0]['phase_name'];}
// sorting data by name to make it easier later.
attacks[fileData['objects'][0]['name']] = {
id: fileData['objects'][0]['id'],
type: fileData['objects'][0]['type'],
description: fileData['objects'][0]['description'],
x_mitre_platforms: fileData['objects'][0]['x_mitre_platforms'],
x_mitre_detection: fileData['objects'][0]['x_mitre_detection'],
phase_name: phase_name};
});
fileNum += 1;
};
});
var keys = Object.keys(attacks);
console.log(attacks);
}
getData(directory);
The reason for the empty log here because the node does not wait to finish while loop Hence you are getting empty log. Basically, you can improve this code by using the async-await method.
But if you are stick with this code, I am just suggesting this logic.
Just bring your log inside an if condition block. which have condition "print only if expected file count reached"
for example.
if(fileNum === files.length) {
var keys = Object.keys(attacks);
console.log(attacks);
}
So now log print only when this condition is satisfied which means after completion of while loop
Little info, i have an arp.js file which takes a subnet address "192.168.2" and gets all strings returned from arp -a and stores in an array.
I can't figure out why my arpList function is returning an undefined value in my index.js file.
All the console.logs are returning the correct values in the arp.js page when called from the index.js, but the ipObj is coming up undefined. Even the console.log before i return of ipObj works.
Any help would be greatly appreciated.
var { spawn } = require('child_process');
const arpLs = spawn('arp', ['-a']);
var bufferData;
module.exports = {
arpList: function (subnet) {
arpLs.stdout.on('data', data => {
bufferData += data
})
arpLs.stderr.on('data', data => {
console.log('error: ' + data);
});
arpLs.on('exit', function (code) {
if (code != 0) {
console.log("Error exiting"); //if error occurs
}
console.log("exit start 1"); // checking internal processes at stages
var dataArray = bufferData.split(' ');
var ipArray = [];
for (i = 0; i < dataArray.length; i++) {
if (dataArray[i].includes(subnet)) {
ipArray.push(dataArray[i]);
console.log("loop working");
}
}
var ipObj = { "lanIps": ipArray };
console.log("Object is there: "+ipObj)
return ipObj; // this obj should be returned to the index.js call using
})
},
sayMyName: function () {
return "Hello";
}
}
//arpList(ipSubnet);
//INDEX.js
//the index page looks like this
//var arp = require('./arp.js);
//var ipSubnet = "192.168.2";
//var lanIps = arp.arpList(ipSubnet);
//console.log(lanIps);
I ended up adding a callback function to arpList - function (subnet, callback)
Then instead of returning the value pass it into the callback
Then on the index.js side instead of
var lanIps = arp.arpList(value)
i used
arp.arpList(value, function(res){lanIps = res}
return ipObj; // this obj should be returned to the index.js call using
It won't be returned. The reference say nothing about return value. Node-style callbacks rarely work like that because they are potentially asynchronous and returned value cannot be taken into account.
This a special case of this well-known problem. The process is asynchronous and is finished after arp.arpList(ipSubnet) call, there's nothing to assign to lanIps. This is a use case for promises. There are already third-party promisified counterparts like child-process-promise.
The problem can be also solved by moving to synchronous API. child_process functions have synchronous counterparts, including spawnSync.
I have this below code, but as soon as widgetsAddCall is added into array, it gets executed, and promise.each is of no use.
Function widgetsAddCall is making async request to API server and returns bluebird promisified request. I want that API call is made one after another, so that each call sends chunk of data one after another.
var chunkCnt = Math.ceil(widgetsIds.length/4000);
var responseT = Array();
var Promise = require('bluebird');
for(var cntTemp =0 ; cntTemp<chunkCnt;cntTemp++){
var tempWidgs = widgetsIds.slice(cntTemp,cntTemp+4000);
var query = {
id: tempWidgs.join(',')
};
responseT.push(widgetsAddCall(tempWidgs,query,campRemoteId,campaign));
}
return Promise.each(responseT,function(responses) {
// Use the responses here
return getWidgets(campRemoteId,campaign).then((ids) => {
var toRemove = [];
for(var id of ids){
if(widgetsIds.indexOf(id)===-1){
toRemove.push(id);
}
}
if(toRemove.length) {
return removeWidgets(campaign, campRemoteId, toRemove);
}
});
})
I want that API call is made one after another, so that each call
sends chunk of data one after another.
The simplest way I can think of to serialize all your calls is to prebuild your chunks into an array and then use Bluebird's Promise.mapSeries() to serially iterate through the array:
var Promise = require('bluebird');
// pre-build chunks into an array
var chunks = [];
var chunkCnt = Math.ceil(widgetsIds.length / 4000);
for (var chunkIndex = 0; chunkIndex < chunkCnt; chunkIndex++) {
chunks.push(widgetsIds.slice(cntIndex,cntIndex + 4000));
}
// now serially iterate the array
Promise.mapSeries(chunks, function(item) {
return widgetsAddCall(item, {id: item.join(',')}, campRemoteId, campaign);
}).then(function(results) {
// now process the results of widgetsAddCall() here
});
FYI, your original use of Promise.each() was not making any sense to me because you were iterating an array of promises, but then not using any info from the iteration. There seemed to be no point to the iteration. Plus, you weren't properly serializing your calls to widgetsAddCall() anyway since you were launching all those calls in parallel.
Given Node.js boasts of asynchronous event driven model,
I was expecting, I should be able to write any Nodejs function,
e.g as simple as going through a loop, e.g IamLooper() below,
which might or might not involve file I/O and then pass that looping function to a mother nodeJs function e.g Invoke(),to which I also pass another call back functiont e.g happyend() below.
My expectation was after IamLooper is finished ,happyend () will be invoked by the NodeJs supplied function .
e.g :
==>
gdata =[];
function IamLooper() {
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++)
{
pi[ii] = 13* ii;;
gdata.push(ii);
}
console.log("looper done -tell the callback") ;
}
function happyend() { console.log("looper says done");}
I want to invoke IamLooper() and supply the happyend at time of invocation.
i.e. I am looking for a ready made node function e.g Invoke, which can be called like this:
Invoke(IamLooper(), happyend());
if(gdata.length > 0) {console.log("looping has started");}
In essence Invoke should do the same for any two functions I supply to it so that we have just a working template of a callback execution strategy.
Also the Invoke being executed async, my program progresses beyond Invoke before it finishes.
Is my expectation is misguided ? Can any one give me some guidance here.
If you are looking for a preexisting way of easily doing callbacks in node, you should use event emitters (https://nodejs.org/api/events.html):
var EventEmitter = require('events').EventEmitter;
var eventExample = new EventEmitter;
//You can create event listeners:
eventExample.on('anEvent', function(someData){
//Do something with someData
});
//To trigger an event listener you must emit:
eventExample.emit('anEvent', someData);
With your code, it'd look something like this:
var EventEmitter = require('events').EventEmitter;
var looper = new EventEmitter;
looper.on('invoke', function(data){
var callFunction = data.callFunction;
var finishFunction = data.finishFunction;
var callParameters = data.callParameters;
var finishParameters = data.finishParameters;
if(callParameters == null){
callFunction({callbackPara: finishParameters, callbackFunction: finishFunction});
}
else{
callFunction(callParameters, {callbackParameters: finishParameters, callbackFunction: finishFunction});
}
});
looper.on('finish', function(data){
var finishFunction = data.callbackFunction;
var parameters = data.callbackParameters;
if(parameters == null){
finishFunction();
}
else{
finishFunction(parameters);
}
});
gdata =[];
function IamLooper(g, callback){
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++){
pi[ii] = 13* ii;;
g.push(ii);
}
looper.emit('finish', callback);
}
function happyend() { console.log("looper says done");}
And then call it like:
looper.emit('invoke', {callFunction: IamLooper, finishFunction: happyend, callParameters: gdata, finishParameters: null});
You can also always do normal callbacks:
gdata =[];
function IamLooper(g, callback){
var pi = Array;
for (var ii = 0 ; ii <4 ; ii ++){
pi[ii] = 13* ii;;
g.push(ii);
}
callback();
}
IamLooper(gdata, function(){ console.log("looper says done");}
I was trying to learn the streaming in Nodejs by writing a small script. But after executing this one the last stream is not pushing all the data.
var stream = require('stream');
var fs = require('fs');
var util = require('util');
function Newliner () {
stream.Transform.call(this);
}
util.inherits(Newliner, stream.Transform);
Newliner.prototype._transform = function(chunk, enc, done) {
var split = 0;
for( var i =0; i <chunk.length; i++){
if(chunk[i] == 10) {
this.push(chunk.slice(split,i));
split = i+1;
}
}
}
function Greper(options) {
stream.Transform.call(this);
this.regex = new RegExp(options);
}
util.inherits(Greper, stream.Transform);
Greper.prototype._transform = function(chunk, enc, done) {
this.push(chunk); //Even this is not working.
/*
var a = chunk.toString();
if(this.regex.test(a)){
this.push(chunk);
}
*/
}
var n = new Newliner();
var g = new Greper("line");
var f = fs.createReadStream('a.txt');
f.pipe(n).pipe(g).pipe(process.stdout);
Input file a.txt is,
This is line one.
Another line.
Third line.
While executing only one line is displayed. What is the reason for this?
$ node test.js
This is line one.
Note: When i am piping the File read stream directely to the 'g' it works correctly.
You need to call the callback of the _transform() function when you are done processing the chunk. From the documentation:
callback (Function) Call this function (optionally with an error argument) when you are done processing the supplied chunk.
No more data will be pushed into the stream until the callback is called. If you don't call it, then the chunk will not be considered as processed… which is why your program stops after processing only one line.
Just add:
done();
at the end of the Newliner.prototype._transform and Greper.prototype._transform functions.