Background
I have taken some sample data from a large XML response I will be working with in my app. I decided to save the xml to a response.xml file in my app so I can parse the data without making a bunch of unnecessary request to the api.
I am going to be using xml2js to convert the xml response to a JS Object.
Problem
I have been able to open the file and console.log() it.
I have been able to run xml2js by passing a small xml string to it which I also console.log() it.
But I have only been able to console.log() the file after xml2js creates the object. No matter what I try I continue to get null when using return or passing trying to pass the data outside of the initial creation of the object.
Example
This prints xml tree to the console,
var fs = require('fs');
var openFile = fs.readFile('./response.xml', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
console.log(data);
});
function requestCreditReport() {
return openFile;
}
requestCreditReport();
This prints a small xml string to console statically added in with
xml2js,
var parseString = require('xml2js').parseString;
var xml = "<root>Hello xml2js!</root>";
parseString(xml, function (err, result) {
console.dir(result);
});
Question
How do I use the object once it is created outside of the method below. Initially when it is created I can output it to the console but cannot seem to access it outside of that console.log(). in the example below I am trying return result. This leaves the value null when I try to pass the object returned by the function to a variable like this,
var response = requestReport();
Recent try,
var fs = require('fs');
var parseString = require('xml2js').parseString;
function requestReport() {
fs.readFile('./response.xml', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
return parseString(data, function (err, result) {
return result;
});
});
}
var response = requestReport();
console.log(response);
This returns null. But if I console.log(result) instead or using return then trying outside of the function it returns this,
{ RESPONSE_DATA:
{ '$': { MYAPI: '0.0.0' },
DETAILS: [ [Object] ],
DETAILS_ACCOUNT: [ [Object] ],
RESPONSE: [ [Object] ] } }
requestReport is async. It doesn't return anything so response is undefined.
You have to use a callback.
var fs = require('fs');
var parseString = require('xml2js').parseString;
function requestReport(callback) {
fs.readFile('./response.xml', 'utf8', function(err, data) {
if (err) return callback(err);
parseString(data, callback);
});
}
requestReport(function(err, result) {
if (err) return console.error(err);
console.log(result);
});
Related
I am a newbie in Node JS and trying to get my head around functional programming.
I have the following code in a file called findinfo.js and I'm trying to pass the result to the main server.js:
const fs = require('fs');
const values = ["yes", "no", "?", "unknown", "partial"];
var cInfo = [];
function getFile (cb) {
fs.readFile('./scripts/blahblah.json', 'utf-8', function (err, jfile) {
if (err) {
throw new Error (err);
}
console.log("Function is executing...")
JSON.parse(jfile);
console.log('Parsing file done');
cb(jfile);
});
}
Then I'm trying to call this function from server.js,
var findinfo = require('./findinfo');
console.log(getFile());
which as expected crashes the program.
So what changes should I make in order to make it work?
You need to export getFile so it can be imported using require.
const fs = require('fs');
const values = ["yes", "no", "?", "unknown", "partial"];
var cInfo = [];
function getFile (cb) {
fs.readFile('./scripts/blahblah.json', 'utf-8', function (err, jfile) {
if (err) {
// throw new Error (err); // don't throw inside async callback
return cb(err);
}
console.log("Function is executing...")
JSON.parse(jfile);
console.log('Parsing file done');
cb(null, jfile);
});
}
module.exports = getFile;
server.js
var getFile = require('./findinfo');
getFile(function(err, file) {
console.log(err, file);
});
since getFile is an asynchronous function, you have to wait until it finishes, when cb is called, to console.log the result.
And using throw in a asynchronous callback is not recommended, since it will crash the server, it's recommended to pass the error to the callback.
You should take a look at this question, so you learn more about how to handle asynchronous code.
How do I return the response from an asynchronous call?
I updated the function to create the CSV file but now I'm getting an error:
In upload function
internal/streams/legacy.js:57
throw er; // Unhandled stream error in pipe.
^
Error: ENOENT: no such file or directory, open 'C:\Users\shiv\WebstormProjects\slackAPIProject\billingData\CSV\1548963844106output.csv'
var csvFilePath = '';
var JSONFilePath = '';
function sendBillingData(){
var message = '';
axios.get(url, {
params: {
token: myToken
}
}).then(function (response) {
message = response.data;
fields = billingDataFields;
// saveFiles(message, fields, 'billingData/');
saveFilesNew(message, fields, 'billingData/');
var file = fs.createReadStream(__dirname + '/' + csvFilePath); // <--make sure this path is correct
console.log(__dirname + '/' + csvFilePath);
uploadFile(file);
})
.catch(function (error) {
console.log(error);
});
}
The saveFilesNew function is:
function saveFilesNew(message, options, folder){
try {
const passedData = message;
var relevantData='';
if (folder == 'accessLogs/'){
const loginsJSON = message.logins;
relevantData = loginsJSON;
console.log(loginsJSON);
}
if(folder == 'billingData/'){
relevantData = passedData.members;
const profile = passedData.members[0].profile;
}
//Save JSON to the output folder
var date = Date.now();
var directoryPath = folder + 'JSON/' + date + "output";
JSONFilePath = directoryPath + '.json';
fs.writeFileSync(JSONFilePath, JSON.stringify(message, null, 4), function(err) {
if (err) {
console.log(err);
}
});
//parse JSON onto the CSV
const json2csvParser = new Json2csvParser({ fields });
const csv = json2csvParser.parse(relevantData);
// console.log(csv);
//function to process the CSV onto the file
var directoryPath = folder + 'CSV/' + date + "output";
csvFilePath = directoryPath + '.csv';
let data = [];
let columns = {
real_name: 'real_name',
display_name: 'display_name',
email: 'email',
account_type: 'account_type'
};
var id = passedData.members[0].real_name;
console.log(id);
console.log("messageLength is" +Object.keys(message.members).length);
for (var i = 0; i < Object.keys(message.members).length; i++) {
console.log("value of i is" + i);
var display_name = passedData.members[i].profile.display_name;
var real_name = passedData.members[i].profile.real_name_normalized;
var email = passedData.members[i].profile.email;
var account_type = 'undefined';
console.log("name: " + real_name);
if(passedData.members[i].is_owner){
account_type = 'Org Owner';
}
else if(passedData.members[i].is_admin){
account_type = 'Org Admin';
}
else if(passedData.members[i].is_bot){
account_type = 'Bot'
}
else account_type = 'User';
data.push([real_name, display_name, email, account_type]);
}
console.log(data);
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err;
fs.writeFileSync(csvFilePath, output, function(err) {
console.log(output);
if (err) {
console.log(err);
}
console.log('my.csv saved.');
});
});
} catch (err) {
console.error(err);
}
}
The upload file function is:
function uploadFile(file){
console.log("In upload function");
const form = new FormData();
form.append('token', botToken);
form.append('channels', 'testing');
form.append('file', file);
axios.post('https://slack.com/api/files.upload', form, {
headers: form.getHeaders()
}).then(function (response) {
var serverMessage = response.data;
console.log(serverMessage);
});
}
So I think the error is getting caused because node is trying to upload the file before its being created. I feel like this has something to do with the asynchronous nature of Node.js but I fail to comprehend how to rectify the code. Please let me know how to correct this and mention any improvements to the code structure/design too.
Thanks!
You don't wait for the callback provided to stringify to be executed, and it's where you create the file. (Assuming this stringify function really does acccept a callback.)
Using callbacks (you can make this cleaner with promises and these neat async/await controls, but let's just stick to callbacks here), it should be more like:
function sendBillingData() {
...
// this callback we'll use to know when the file writing is done, and to get the file path
saveFilesNew(message, fields, 'billingData/', function(err, csvFilePathArgument) {
// this we will execute when saveFilesNew calls it, not when saveFilesNew returns, see below
uploadFile(fs.createReadStream(__dirname + '/' + csvFilePathArgument))
});
}
// let's name this callback... "callback".
function saveFilesNew(message, options, folder, callback) {
...
var csvFilePath = ...; // local variable only instead of your global
...
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err; // or return callbcack(err);
fs.writeFile(csvFilePath , output, function(err) { // NOT writeFileSync, or no callback needed
console.log(output);
if (err) {
console.log(err);
// callback(err); may be a useful approach for error-handling at a higher level
}
console.log('my.csv saved.'); // yes, NOW the CSV is saved, not before this executes! Hence:
callback(null, csvFilePath); // no error, clean process, pass the file path
});
});
console.log("This line is executed before stringify's callback is called!");
return; // implicitly, yes, yet still synchronous and that's why your version crashes
}
Using callbacks that are called only when the expected events happen (a file is done writing, a buffer/string is done transforming...) allows JS to keep executing code in the meantime. And it does keep executing code, so when you need data from an async code, you need to tell JS you need it done before executing your piece.
Also, since you can pass data when calling back (it's just a function), here I could avoid relying on a global csvFilePath. Using higher level variables makes things monolithic, like you could not transfer saveFilesNew to a dedicated file where you keep your toolkit of file-related functions.
Finally, if your global process is like:
function aDayAtTheOffice() {
sendBillingData();
getCoffee();
}
then you don't need to wait for the billing data to be processed before starting making coffee. However, if your boss told you that you could NOT get a coffee until the billing data was settled, then your process would look like:
function aDayAtTheOffice() {
sendBillingData(function (err) {
// if (err) let's do nothing here: you wanted a coffee anyway, right?
getCoffee();
});
}
(Note that callbacks having potential error as first arg and data as second arg is a convention, nothing mandatory.)
IMHO you should read about scope (the argument callback could be accessed at a time where the call to saveFilesNew was already done and forgotten!), and about the asynchronous nature of No... JavaScript. ;) (Sorry, probably not the best links but they contain the meaningful keywords, and then Google is your buddy, your friend, your Big Brother.)
Below is the code I tried:
fetchUserDetails: function () {
var url = 'http://ldniguiapp02.eur.ad.tullib.com/matchbox-forwarddeal/services/RefDataWebServices?wsdl';
var args = {'args0':
{
'mnemonic':'ttan',
'postingId':'75655',
'customerId':'180816',
'organisation':{
'customerId':'180816',
'firmName':'POLITICAL.GROUP'
},
'userType':'TRADER'
}};
var defered = q.defer();
soap.createClient(url, CreateClient);
function CreateClient(err, client) {
client.getUserDetails(args, function (err, result) {
if (err) {
defered.reject(err);
}else{
defered.resolve(result);
}
console.log(result);
});
}
return defered.promise;
}
and the equivalent SOAP request from SOAP UI looks something like this:
How can I format args json so as to get expected result?
You should transform your json in xml
Here you can do this online (to check/verify/test) http://convertjson.com/json-to-xml.htm
And off course there is an npm module that does just this:
https://www.npmjs.com/package/jsontoxml
...
var jsonxml = require('jsontoxml');
var xmlArgs = jsonxml(args);
...
client.getUserDetails(xmlArgs, function (err, result) {
...
})
...
I hope this helps
I am using express to create a webservice that will read string data from a stream, and respond to the HTTP POST request with that value. Here is the code for the S3Store.js file that defines the readFileFromS3(.) function:
S3Store.js
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = '';
readStream.on('data', function(data) {
//data = Buffer.concat([allData, data]);
data = allData + data;
console.log("data: " + data);
});
readStream.on('error', function(err) {
callback(err, null);
});
Now, if I call this method from a terminal like this:
s3Instance.readFileFromS3('123.json', function(err, data) {
console.log(data);
});
I see the appropriate string for data logged to the console. However, when I call the same method from inside one of the routes in express for HTTP POST requests, the service responds with a value of data set to empty string. Code for the POST request:
router.post('/resolve', function(req, res) {
var commandJson = req.body;
var appId = commandJson['appId'];
var command = commandJson['text'];
if (appId == undefined || command == undefined) {
res.status(400).send("Malformed Request: appId: " + appId + ", command: " + command);
};
s3Store.readFileFromS3('123.json', function(err, data) {
res.send(data);
});
});
Why does it return an empty string when calling the readFileFromS3(.) from the HTTP POST method and not when I ran the same method directly from the node console?
You're logging the data but you're not passing anything to the completion callback (see below for some more explanation):
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = [];
// Keep collecting data.
readStream.on('data', function(data) {
allData.push(data);
});
// Done reading, concatenate and pass to completion callback.
readStream.on('end', function() {
callback(null, Buffer.concat(allData));
});
// Handle any stream errors.
readStream.on('error', function(err) {
callback(err, null);
});
};
I took the liberty to rewrite the data collection to use Buffer's instead of strings, but this obviously isn't a requirement.
The callback argument is a completion function, meant to be called when either reading the S3 stream is done, or when it has thrown an error. The error handling was already in place, but not the part where you would call back when all the data from the stream was read, which is why I added the end handler.
At that point, the readStream is exhausted (everything from it has been read into allData), and you call the completion callback when the collected data as second argument.
The common idiom throughout Node is that completion callbacks take (at least) two arguments: the first is either an error, or null when there aren't errors, and the second is the data you want to pass back to the caller (in your case, the anonymous function in your route handler that calls res.send()).
I have a problem in getting a .json file in express and displaying in a view. Kindly share your examples.
var fs = require("fs"),
json;
function readJsonFileSync(filepath, encoding){
if (typeof (encoding) == 'undefined'){
encoding = 'utf8';
}
var file = fs.readFileSync(filepath, encoding);
return JSON.parse(file);
}
function getConfig(file){
var filepath = __dirname + '/' + file;
return readJsonFileSync(filepath);
}
//assume that config.json is in application root
json = getConfig('config.json');
Do something like this in your controller.
To get the json file's content :
ES5
var foo = require('./path/to/your/file.json');
ES6
import foo from './path/to/your/file.json';
To send the json to your view:
function getJson(req, res, next){
res.send(foo);
}
This should send the json content to your view via a request.
NOTE
According to BTMPL
While this will work, do take note that require calls are cached and will return the same object on each subsequent call. Any change you make to the .json file when the server is running will not be reflected in subsequent responses from the server.
This one worked for me. Using fs module:
var fs = require('fs');
function readJSONFile(filename, callback) {
fs.readFile(filename, function (err, data) {
if(err) {
callback(err);
return;
}
try {
callback(null, JSON.parse(data));
} catch(exception) {
callback(exception);
}
});
}
Usage:
readJSONFile('../../data.json', function (err, json) {
if(err) { throw err; }
console.log(json);
});
Source