How to convert NodeJS form-data object into JSON string - node.js

I am using form-data package in my NodeJS application to send formdata. I am using Axios interceptor for logging the requests in a file. In axiosIns.config.data, I need the JSON string corresponding to the formdata set but currently it's FormData Object.
This library provides a toString method but on using that I have found that it returns a static string [object FormData] instead of the stringified input. I have opened an issue regarding that but seems unattended.
I have created a repl for regenerating that.
Is there any way so that I can convert my formdata object into a readable, loggable, preferably JSO string?

I solved It
const FormData = require("form-data");
var data = new FormData();
data.append("modid", "IM");
data.append("token", "provider\nagg");
data.append("cat_type", "3");
var boundary = data.getBoundary();
var data_row = data.getBuffer().toString();
console.log(rawFormDataToJSON(data_row,boundary));
function rawFormDataToJSON(raw_data,boundary){
var spl = data_row.split(boundary);
var data_out = [];
spl.forEach(element => {
let obj = {};
let ll = element.split("\n");
if(ll[1]){
let key = ll[1].split("=")[1].replace('"',"").replace('"\r',"");
let val = "";
if(ll.length > 3){
for (let i = 3; i < ll.length; i++) {
val += ll[i]+"\n";
}
}
obj[key] = val.replace("--","").replace("\r\n\n","");
data_out.push(obj);
}
});
return data_out;
}
Expected Output
[ { modid: 'IM' }, { token: 'provider\nagg' }, { cat_type: '3' } ]

Edit: I got reply on the github issue mentioned and as per that "this package doesn't intend to implement the toString() in a way to return stringified data. If I want spec-compliant FormData, I'll need to install the other packages mentioned. So it's not an issue but an intended unimplemented feature."
formdata-node
formdata-polyfill
undici
I tried below code, seems fine but not recommended as it's based on text splitting and filtering, if text format changes, it might create issue. This is the sandbox for the same.
const FormData = require("form-data");
var data = new FormData();
data.append("modid", "IM");
data.append("token", "provider");
data.append("cat_type", "3");
const objectifyFormdata = (data) => {
return data
.getBuffer()
.toString()
.split(data.getBoundary())
.filter((e) => e.includes("form-data"))
.map((e) =>
e
.replace(/[\-]+$/g, "")
.replace(/^[\-]+/g, "")
.match(/\; name\=\"([^\"]+)\"(.*)/s)
.filter((v, i) => i == 1 || i == 2)
.map((e) => e.trim())
)
.reduce((acc, cur) => {
acc[cur[0]] = cur[1];
return acc;
}, {});
};
console.log(objectifyFormdata(data));
// { modid: 'IM', token: 'provider', cat_type: '3' }

Related

Async communicating with serialport gives mixed data

I'm trying to communicating with an arduino that has sensors.
So i have an object called motherboard who has sensors and each sensor has metric which may have threshold and/or polling which has a methode called getValue that sends data to the arduino and returns data with a promise. The problem is that if i async the sensors to get their values all sensors get the same value.
I don't know why this is happening. I only have programmed with javascript for 1 year and with angular for 5 months. I checked the post async/await using serialport in node.js but i checked my code and i did the something that was suggested in the post.
The communicate method is inside a service.
Can anyone help?
tl;dr :
send data to arduino get data back in a promise.
Metric A and B get the same promise.
The component polling also gets the promise of threshold. (metric has threshold and polling)
Me
communication.service.ts
communicate(cmd: string, serialPort: SerialPort, expectedResponse: string, notExpectedResponse){
const parser = serialPort.pipe(new Delimiter({delimiter: '\n'}));
return new Promise((resolve, reject) => {
serialPort.write(cmd, () => {
console.log('message written');
parser.on('data', data => {
const dataString = data.toString();
if (dataString != null && dataString.includes(expectedResponse)) {
let responseRemoved = dataString.replace(expectedResponse + ' ', '');
resolve(responseRemoved);
} else {
let response;
if (dataString != null && dataString.includes(notExpectedResponse)) {
response = dataString.replace(notExpectedResponse + ' ', '');
}
reject(response);
}
});
setTimeout(() => {
reject('');
}, this.timeOutTime);
});
});
}
threshold.component.ts
private getValuesThreshold(): void{
console.log(this.metricId);
this.motherboardInUse.getValues(this.metricId, GlobalVariableCMD.GET_THRESHOLD_VALUES,
GlobalVariableResponse.GET_THRESHOLD_VALUES, GlobalVariableResponse.GET_THRESHOLD_VALUES).then(data => {
let dataString = data.toString();
if(dataString){
console.log(dataString);
let responseSplit = dataString.toString().split(' ');
let minimumValue = parseInt(responseSplit[1]);
let maximumValue = parseInt(responseSplit[2]);
minimumValue < this.floor ? this.minimumThreshold = this.floor : this.minimumThreshold = minimumValue;
maximumValue > this.ceil ? this.maximumThreshold = this.ceil : this.maximumThreshold = 90;
this.enabled = responseSplit[0].includes('1');
console.log(this.minimumThreshold);
console.log(this.maximumThreshold);
console.log(this.enabled);
}
}).catch(err => {
let errString = err.toString();
if(errString){
console.log(errString);
}
});
}
motherboard.component.ts
getValuesThreshold(metricId: string, ATcmd: string, expectedResponse: string, notExpectedResponse: string) {
let command = this.communicateBuilder.BuildCommandGetMetricValue(ATcmd, this.usedSensorId, metricId);
console.log('motherboard get values' + command);
let responseOk = this.commandBuilderService.respondsSuccess(expectedResponse);
let responseNotOk = this.commandBuilderService.respondsFail(notExpectedResponse);
return this.communicateService.communicate(command, this.motherboard.serialPort, responseOk, responseNotOk);
}
Maybe you can try this
async getValuesThreshold(metricId: string, ATcmd: string, expectedResponse: string, notExpectedResponse: string) {
let command = this.communicateBuilder.BuildCommandGetMetricValue(ATcmd, this.usedSensorId, metricId);
console.log('motherboard get values' + command);
let responseOk = this.commandBuilderService.respondsSuccess(expectedResponse);
let responseNotOk = this.commandBuilderService.respondsFail(notExpectedResponse);
return await this.communicateService.communicate(command, this.motherboard.serialPort, responseOk, responseNotOk);
}
The problem was sending async data with serialport.
Serialport node had no way of knowing what response was linked to what data that was send.
So it acted on the first response it got and returned that.
Only way to solve this was to ask them sync.

Calling a method within a dynamically named object

I am trying to invoke a method of a dynamically named object.
I have a few objects each containing a method named 'getWeight'. I need to invoke this method of a subset of these objects. However, the subset of objects depends on user inputted info, which is why I'm attempting to dynamically construct the object name and invoke it's 'getWeight' method within a loop.
My code is below:
// Importing objects containing 'getWeight' method
const Apple = require('../classses/Apple');
const Pear = require('../classes/Pear);
const Carrot = require('../classes/Carrot);
const Potato = require('../classes/Potato);
const promises = {};
const user = 'userX'; // This is the inputted info, could also equal 'userY'.
const searchableFoods = {
userX: ['Apple', 'Pear'],
userY: ['Carrot', 'Potato']
};
for (i = 0; i < searchableFoods[user].length; ++i) {
promises[searchableFoods[user][i]] = new Promise(function(resolve, reject) {
// Below line should behave like: Apple.getWeight(arg1, function....
searchableFoods[user][i].getWeight(arg1, function(response) {
resolve(response);
})
})
}
Unfortunately, I get this error:
[searchableFoods[user][i].getweight] is not a function
I've tried a number of variations but can't make it work. Is it possible to do this?
Require all of those into a single object rather than many standalone variables, and then you can use simple property lookup to get to the appropriate requireed value. Also, forEach will likely result in more readable code, and you can pass the function name alone (resolve) rather than defining an extra anonymous function for the getWeight callback:
const foods = {
Apple: require('../classses/Apple'),
Pear: require('../classes/Pear),
Carrot: require('../classes/Carrot),
Potato: require('../classes/Potato)
};
// ...
searchableFoods[user].forEach((foodName) => {
promises[foodName] = new Promise((resolve) => {
foods[foodName].getWeight(arg1, resolve);
});
});
Can you define array not using strings? Something like that?
const searchableFoods = {
userX: [Apple, Pear],
userY: [Carrot, Potato]
};
You're trying to access getWeight as a property of string 'Apple' and not of the actual object Apple that you are importing.
Change the subset array to something like this
const searchableFoods = {
userX: [Apple, Pear],
userY: [Carrot, Potato]
};
Which makes the final code to be
const Apple = require('../classses/Apple');
const Pear = require('../classes/Pear);
const Carrot = require('../classes/Carrot);
const Potato = require('../classes/Potato);
const promises = {};
const user = 'userX' // This is the inputted info, could also equal 'userY'.
const searchableFoods = {
userX: [Apple, Pear],
userY: [Carrot, Potato]
};
for (i = 0; i < searchableFoods[user].length; ++i) {
promises[searchableFoods[user][i]] = new Promise(function(resolve, reject) {
// Below line should behave like: Apple.getWeight(arg1, function....
searchableFoods[user][i].getWeight(arg1, function(response) {
resolve(response);
})
})
}

Using knex SELECT query results for another SELECT query

I am trying to run a PostgreSQL query with Knex and then use the results to run another query.
exports.buildBuoyFeaturesJSON = function (conditionA, conditionB) {
var query = null;
var selectedFields = knex.select
(
knex.raw('t_record.id AS id'),
...
knex.raw('t_record.latitude AS latitude'),
knex.raw('t_record.longitude AS longitude')
)
.from('t_record')
.then(function (response) {
var geometry_array = [];
var rows = response.rows;
var keys = [];
for (var key = 0; key <= rows.length - 1; key++) {
var geometry =
{
"id" : rows[key].id,
"type" : "Feature",
"geometry" : rows[key].geometry,
"properties" : {
...
"sensors" : []
}
};
keys.push(rows[key].id);
geometry_array.push(geometry);
}
getMeasurementsAndSensors(keys, geometry_array);
});
};
The latter function uses some of the results from the previous function. Due to asynchronous nature of Knex, I need to call the second function from inside the first function's .then() statement:
function getMeasurementsAndSensors (keys, geometry_array) {
var query = knex
.select
(
't_record_id',
'i_sensor_id',
'description',
'i_measurement_id',
't_sensor_name',
't_measurement_name',
'value',
'units'
)
.from('i_record')
...
.whereRaw('i_record.t_record_id IN (' + keys + ')')
.orderByRaw('t_record_id, i_sensor_id ASC')
.then(function (response) {
var rows = response.rows;
var t_record_id = 0;
var i_sensor_id = 0;
var record_counter = -1;
var sensor_counter = -1;
for (var records = 0; records <= rows.length -1; records++) {
if (t_record_id !== rows[records].t_record_id) {
t_record_id = rows[records].t_record_id;
record_counter++;
sensor_counter = -1;
}
if (i_sensor_id !== rows[records].i_sensor_id) {
i_sensor_id = rows[records].i_sensor_id;
geometry_array[record_counter].properties.sensors[++sensor_counter] =
{
'i_sensor_id' : rows[records].i_sensor_id,
't_sensor_name' : rows[records].t_sensor_name,
'description' : rows[records].description,
'measurements' : []
};
}
geometry_array[record_counter].properties.sensors[sensor_counter].measurements.push
({
'i_measurement_id': rows[records].i_measurement_id,
'measurement_name': rows[records].t_measurement_name,
'value': rows[records].value,
'units': rows[records].units
});
}
//wrapping features with metadata.
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
JSONToFile(feature_collection, 'buoy_features');
});
}
Currently I save end result to a JSON file because I couldn't get Promises to work. JSON is later used to power a small OpenLayers application, hence the JSON-ification after getting results.
I am quite sure that getting the data from a database, saving it to file, then accessing it from another process and using it for OpenLayers is a very redundant way to do it, but so far, it is the only one that works.
I know there are a lot of ways to make these functions work better, but I am new to promises and don't know how to work with them outside of most basic functions. Any suggestions how to make this code better are welcome.
All you appear to be missing is a bunch of returns.
Here are skeletonized versions of the two functions, including the necessary returns :
exports.buildBuoyFeaturesJSON = function(conditionA, conditionB) {
return knex.select (...)
^^^^^^
.from(...)
.then(function(response) {
// synchronous stuff
// synchronous stuff
return getMeasurementsAndSensors(keys, geometry_array);
^^^^^^
}).then(function(geometry_array) {
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
return feature_collection;
^^^^^^
});
};
function getMeasurementsAndSensors(keys, geometry_array) {
return knex.select(...)
^^^^^^
.from(...)
...
.whereRaw(...)
.orderByRaw(...)
.then(function(response) {
// heaps of synchronous stuff
// heaps of synchronous stuff
// heaps of synchronous stuff
return geometry_array;
^^^^^^^^^^^^^^^^^^^^^
});
}
I moved the feature_collection collection part into buildBuoyFeaturesJSON() on the basis that it seems to sit there more logically. If not, then it would be very simple to move it back into getMeasurementsAndSensors().
I have not tried to fix the additional issue highlighted by #vitaly-t.

Why is my for-loop only counting to one object in the JSON-response?

I'm trying to make a map with map-annotations which are being generated from a REST-JSON response. I've succeeded with making one, the JSON response contains two objects. Why are only one printed out?
I'm using RestSharp and Xamarin.iOS.
Here's a Gist-clone of the original respones
The function that grabs the data to later-on make map-annotations on our map:
Action getAllMarkers = () => {
var client = new RestClient("http://www.example.com/");
var request = new RestRequest(String.Format("api/?function=searchByName&key=&name=Sundsvall"));
client.ExecuteAsync (request, response => {
JsonValue data = JsonValue.Parse(response.Content);
for (var i = 0; i < data.Count; i++){
Double lat = data["result"][i]["lat"];
Double lng = data["result"][i]["lng"];
String name = data["result"][i]["title"];
String adress = data["result"][i]["adress"];
var store = new BasicMapAnnotation (new CLLocationCoordinate2D(lat, lng), name, adress);
Console.WriteLine(response.Content);
InvokeOnMainThread ( () => {
// manipulate UI controls
map.AddAnnotation(store);
});
}
});
};
getAllMarkers();
data.Count is 1, because there is one top level "result" node in your json. Use data["result"].Count instead.
your result is main array in which rest of the data is so use:data["result"].Count instead of data.Count

Memory consideration using buffer and buffer.toString() in node.js streams

I am performing readstream.pipe(transformstream).pipe(writeStream). The application is read xml from a file system using readstream --> do some manipulation based on different tags using transform stream --> write it to another file system.
During the transform operation inside the transformstream. I am doing buffer.toString() for every chunk as I need to manipulate string before pushing them:
myTransformStream._transform = function(chunk, encoding, done) {
var stringTobeManipulated = chunk.toString();
// perform changes on stringTobeManipulated and push it to writestream
}
the size of xmls can be upto 3 mb and I noticed I got divided to ~44 chunks.
Based on the given problem I have questions concerning memory consumption both in v8 heap and system:
1) I understand that buffers are stored outside the v8 memory. When I do a chunk.toString() in my _transform function, does it create a javascript string object inside the v8 memory ? If yes, I am assuming it will be garbage collected after it looses all its references.
2) As the buffers are part of the system memory I believe they are not garbage collected, so when is that memory freed up ?
3) Is the application a good use case of transform stream as I am converting every chunk to string ?
EDITED:
May be I am not explaining myself clearly. Anyways I have been trying to find a ay of removing namespace from the xml tag before converting it to json. Here is the code I ended up with. Took advantage of memoization in javascrtipt. Please let me know if you guys find a better or more memory efficient way. We are using actionhero.js framework
var action = {};
var xml2js = require('xml2js');
var fs = require('fs');
/////////////////////////////////////////////////////////////////////
// metadata
action.name = 'removeNamespace';
action.description = 'I will parse the article related xml and yield the result';
action.inputs = {
'required': [],
'optional': []
};
action.blockedConnectionTypes = [];
action.outputExample = {
articleHeading: 'heading',
articleBody: 'body'
};
/////////////////////////////////////////////////////////////////////
// functional
action.run = function(api, connection, next) {
var stream = require('stream');
var removeNamespace = new stream.Transform();
var util = require('util');
var output = fs.createWriteStream('output.xml');
removeNamespace._transform = function(chunk, encoding, done) {
removeNamespace._transform.brokenTag = removeNamespace._transform.brokenTag || false;
removeNamespace._transform.brokenString = removeNamespace._transform.brokenString || '';
var convertedString = chunk.toString().trim();
if (removeNamespace._transform.brokenTag === true){
convertedString = removeNamespace._transform.brokenString + convertedString ;
}
removeNamespace._transform.brokenTag = false;
removeNamespace._transform.brokenString = '' ;
if (convertedString.lastIndexOf('<') > convertedString.lastIndexOf('>') ){
removeNamespace._transform.brokenString =convertedString.substring(convertedString.lastIndexOf('<'),convertedString.length+1);
convertedString = convertedString.substring(0,convertedString.lastIndexOf('<')) ;
removeNamespace._transform.brokenTag = true ;
}
convertedString = convertedString.replace(/>\s+</g, '><')
.replace.replace(/<[A-Za-z]+:/gi, '<')
.replace(/<\/[A-Za-z]+:/gi, '</');
done(null, convertedString);
};
var Writable = stream.Writable;
function xmlInMemory(keyToXml, options) {
Writable.call(this, options);
this[keyToXml] = new Buffer('');
this._write = function(chunk, encoding, callback) {
chunk = (Buffer.isBuffer(chunk)) ? chunk : new Buffer(chunk, encoding);
this[keyToXml] = Buffer.concat([this[keyToXml], chunk]);
callback();
};
}
util.inherits(xmlInMemory, Writable);
var source = fs.createReadStream('path/to/your/xmlfile');
var target = new xmlInMemory('keyToXml');
source.pipe(removeNamespace).pipe(target);
target.on('finish', function() {
var parser = new xml2js.Parser();
connection.response.xml2js = target.keyToXml.toString();
next(connection, true);
parser.parseString(target.keyToXml.toString(), function(err, result) {
connection.response.xml2js = result;
next(connection, true);
});
});
};
/////////////////////////////////////////////////////////////////////
// exports
exports.action = action;

Resources