function running that's not explicitly called - node.js

Below is virtually all of the code for a node.js app that lets you get playlists for artists if you run the command simply followed by an artists name
simplay the Beatles
From the output in the terminal, I know that the code in the ._flush method (added to the prototype of UrlsForNamesTransform) is getting run but it's never explicitly called. UrlsForNamesTransform extends the Transform stream in node.js, which I mention because I've seen it in other code before where a function is running without explicitly getting called (at least that I can see). Is it something about Transform or what is happening to make the code in ._flush run?
This is the github repo for the code https://github.com/thlorenz/simplay
'use strict';
var urlschema = 'http://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={{artist}}&api_key={{apikey}}&format=json';
var hyperquest = require('hyperquest')
, table = require('text-table')
, colors = require('ansicolors')
, styles = require('ansistyles')
var stream = require('stream');
var util = require('util');
var Transform = stream.Transform;
util.inherits(UrlsForNamesTransform, Transform);
function UrlsForNamesTransform (opts) {
if (!(this instanceof UrlsForNamesTransform)) return new UrlsForNamesTransform(opts);
opts = opts || {};
Transform.call(this, opts);
this._writableState.decodeStrings = false;
this.artist = opts.artist;
this.json = '';
}
UrlsForNamesTransform.prototype._transform = function (chunk, encoding, cb) {
this.json += chunk.toString();
cb();
};
UrlsForNamesTransform.prototype._flush = function (cb) {
var records = [];
try {
var o = JSON.parse(this.json);
var artists = o.similarartists.artist;
if (!Array.isArray(artists)) {
this.push('Sorry, no records for "' + this.artist + '" where found, please correct your spelling and/or try another artist.');
return cb();
}
artists.forEach(function (node) {
var youtubeurl = 'http://www.youtube.com/results?search_query={{artist}},playlist'.replace('{{artist}}', node.name);
var rdiourl = 'http://www.rdio.com/search/{{artist}}/artists/'.replace('{{artist}}', node.name);
var lastfmurl = 'http://www.last.fm/music/{{artist}}'.replace('{{artist}}', node.name);
var lastfmRadioUrl = 'http://www.last.fm/listen/artist/{{artist}}'.replace('{{artist}}', node.name);
var urls = [
''
, colors.white(' youtube: ') + styles.underline(colors.brightBlue(encodeURI(youtubeurl)))
, colors.blue (' rdio: ') + styles.underline(colors.brightBlue(encodeURI(rdiourl)))
, colors.brightRed (' last.fm: ') + styles.underline(colors.brightBlue(encodeURI(lastfmurl)))
, colors.red (' last.fm radio: ') + styles.underline(colors.brightBlue(encodeURI(lastfmRadioUrl)))
, ''
, ''].join('\n');
records.push([ '\n' + colors.brightYellow(node.name), colors.cyan(node.match), urls ]);
})
this.push(table(records.reverse()));
cb();
} catch (err) {
cb(err);
}
}
var go = module.exports =
/**
* Retrieves similar artists for the given artist from last.fm using the apikey.
* Then it converts the information to display youtube.com, last.fm, rdio playlist/artist urls for each artist.
*
* #name simplay
* #function
* #param {String} artist the artist to find similar artists for
* #param {String} apikey the api key to be used with last.fm
* #return {ReadableStream} that will push the url information
*/
function simplay(artist, apikey) {
if (!artist) throw new Error('Please provid the artist that you like to get similar artist links for');
if (!apikey) throw new Error('Please set LASTFM_API env variable to your API key: http://www.last.fm/api/account/create');
var url = urlschema
.replace('{{artist}}', artist)
.replace('{{apikey}}', apikey);
return hyperquest(url)
.on('error', console.error)
.pipe(new UrlsForNamesTransform({ artist: artist }))
.on('error', console.error)
};

The important line is this one:
util.inherits(UrlsForNamesTransform, Transform);
What this means is that UrlsForNamesTransform is a subclass of Transform. There is very good documentation on subclassing Transform, which can be found on the node.js api site.
Essentially, a subclass of Transform must implement _transform and can implement _flush, but is expected to never call either of those functions. Methods in Transform will call them based on events on the incoming stream.

Related

Call multiple API in for loop and get one by one response using node js

I have to call multiple API using promise all in for loop but I am getting a response in unsequential format. e.g. I have 3 rows but getting a response in 1,2,3 sequence. First getting the first-row response than 3rd row and then getting 2nd row but I need to get a response in a sequential format like (1 row,2 rows,3 rows).
result = '{"error_code":0,"err_desc":null,"data":[{"":"","name":"OTT, KATHRYN M","address":"3110 Horseshoe Trl, Glenmoore, PA","email":"","phone1":"(410) 599-2212","phone2":"(610) 827-9107","phone3":"(610) 308-4566","phone4":"(610) 506-1121","phone5":"(610) 469-0737","phone6":"(610) 942-4347","phone7":"323-7898","phone8":"(814) 371-6133","phone9":""},{"":"","name":"BELTRANTE, SUSAN E","address":"3 Rhoads Ave, Moorestown, NJ\"","email":"SUSAN.BELTRANTE#AOL.COM, JOE.BARGER#YAHOO.COM,","phone1":"(856) 266-0381","phone2":"(856) 273-0869","phone3":"(609) 266-0381","phone4":"(856) 235-3933","phone5":"","phone6":"","phone7":"","phone8":"","phone9":""},{"":"","name":"Manish","address":"4895 E American Beauty Dr, Tucson, AZ 85756","email":"abc#gmail.com","phone1":"(857) 266-0381","phone2":"(857) 273-0869","phone3":"(610) 266-0381","phone4":"(857) 235-3933","phone5":"","phone6":"","phone7":"","phone8":"","phone9":""}]}';
var i;
for (i = 0; i < result.length; i++)
//for (i = 0; i <= 0; i++)
{
var phone = result[i].phone9;
var name = result[i].name;
var address = result[i].address;
var email = result[i].email;
var phone1 = result[i].phone1;
var phone2 = result[i].phone2;
var phone3 = result[i].phone3;
var phone4 = result[i].phone4;
var phone5 = result[i].phone5;
var phone6 = result[i].phone6;
var phone7 = result[i].phone7;
var phone8 = result[i].phone8;
var addressinfo = address.split(',');
var street = addressinfo[0];
var city = addressinfo[1];
var state = addressinfo[2];
var zip = addressinfo[3];
Promise.all([
fetch('https://backend.mioym.properties/api/247/eppraisal?street='+street+'&zip='+zip),
fetch('https://backend.mioym.properties/api/247/zillow?street='+street+'&zip='+zip),
fetch('https://backend.mioym.properties/api/247/pennymac?address='+address),
fetch('https://backend.mioym.properties/api/247/chase?address='+address),
fetch('https://backend.mioym.properties/api/247/realtor?address='+address)
]).then(function (responses) {
// Get a JSON object from each of the responses
return Promise.all(responses.map(function (response) {
console.log("here"+i);
//console.log(response.json());
console.log(response.url);
return response.json();
}));
}).then(function (data) {
console.log("success"+i);
// Log the data to the console
// You would do something with both sets of data here
console.log(data);
}).catch(function (error) {
console.log("error"+i);
// if there's an error, log it
console.log(error);
});
}
So please anyone suggest me solution.
The second Promise.all inside your then block is not necessary, as responses will already contain the resolved values. Note that Promise.all processes the requests in parallel but the resolved responses will be in order. So you can simply do:
Promise.all([
fetch('https://backend.mioym.properties/api/247/eppraisal?street=' + street + '&zip=' + zip),
fetch('https://backend.mioym.properties/api/247/zillow?street=' + street + '&zip=' + zip),
fetch('https://backend.mioym.properties/api/247/pennymac?address=' + address),
fetch('https://backend.mioym.properties/api/247/chase?address=' + address),
fetch('https://backend.mioym.properties/api/247/realtor?address=' + address)
]).then(function (responses) {
// Get a JSON object from each of the responses
return responses.map(function (response) {
console.log(response.url);
return response.json();
});
});

Using knex SELECT query results for another SELECT query

I am trying to run a PostgreSQL query with Knex and then use the results to run another query.
exports.buildBuoyFeaturesJSON = function (conditionA, conditionB) {
var query = null;
var selectedFields = knex.select
(
knex.raw('t_record.id AS id'),
...
knex.raw('t_record.latitude AS latitude'),
knex.raw('t_record.longitude AS longitude')
)
.from('t_record')
.then(function (response) {
var geometry_array = [];
var rows = response.rows;
var keys = [];
for (var key = 0; key <= rows.length - 1; key++) {
var geometry =
{
"id" : rows[key].id,
"type" : "Feature",
"geometry" : rows[key].geometry,
"properties" : {
...
"sensors" : []
}
};
keys.push(rows[key].id);
geometry_array.push(geometry);
}
getMeasurementsAndSensors(keys, geometry_array);
});
};
The latter function uses some of the results from the previous function. Due to asynchronous nature of Knex, I need to call the second function from inside the first function's .then() statement:
function getMeasurementsAndSensors (keys, geometry_array) {
var query = knex
.select
(
't_record_id',
'i_sensor_id',
'description',
'i_measurement_id',
't_sensor_name',
't_measurement_name',
'value',
'units'
)
.from('i_record')
...
.whereRaw('i_record.t_record_id IN (' + keys + ')')
.orderByRaw('t_record_id, i_sensor_id ASC')
.then(function (response) {
var rows = response.rows;
var t_record_id = 0;
var i_sensor_id = 0;
var record_counter = -1;
var sensor_counter = -1;
for (var records = 0; records <= rows.length -1; records++) {
if (t_record_id !== rows[records].t_record_id) {
t_record_id = rows[records].t_record_id;
record_counter++;
sensor_counter = -1;
}
if (i_sensor_id !== rows[records].i_sensor_id) {
i_sensor_id = rows[records].i_sensor_id;
geometry_array[record_counter].properties.sensors[++sensor_counter] =
{
'i_sensor_id' : rows[records].i_sensor_id,
't_sensor_name' : rows[records].t_sensor_name,
'description' : rows[records].description,
'measurements' : []
};
}
geometry_array[record_counter].properties.sensors[sensor_counter].measurements.push
({
'i_measurement_id': rows[records].i_measurement_id,
'measurement_name': rows[records].t_measurement_name,
'value': rows[records].value,
'units': rows[records].units
});
}
//wrapping features with metadata.
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
JSONToFile(feature_collection, 'buoy_features');
});
}
Currently I save end result to a JSON file because I couldn't get Promises to work. JSON is later used to power a small OpenLayers application, hence the JSON-ification after getting results.
I am quite sure that getting the data from a database, saving it to file, then accessing it from another process and using it for OpenLayers is a very redundant way to do it, but so far, it is the only one that works.
I know there are a lot of ways to make these functions work better, but I am new to promises and don't know how to work with them outside of most basic functions. Any suggestions how to make this code better are welcome.
All you appear to be missing is a bunch of returns.
Here are skeletonized versions of the two functions, including the necessary returns :
exports.buildBuoyFeaturesJSON = function(conditionA, conditionB) {
return knex.select (...)
^^^^^^
.from(...)
.then(function(response) {
// synchronous stuff
// synchronous stuff
return getMeasurementsAndSensors(keys, geometry_array);
^^^^^^
}).then(function(geometry_array) {
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
return feature_collection;
^^^^^^
});
};
function getMeasurementsAndSensors(keys, geometry_array) {
return knex.select(...)
^^^^^^
.from(...)
...
.whereRaw(...)
.orderByRaw(...)
.then(function(response) {
// heaps of synchronous stuff
// heaps of synchronous stuff
// heaps of synchronous stuff
return geometry_array;
^^^^^^^^^^^^^^^^^^^^^
});
}
I moved the feature_collection collection part into buildBuoyFeaturesJSON() on the basis that it seems to sit there more logically. If not, then it would be very simple to move it back into getMeasurementsAndSensors().
I have not tried to fix the additional issue highlighted by #vitaly-t.

Cannot store value inside .get method from npm library 'Sqlite3'

Problem: I cannot get a variable into the correct scope, using the .get function from the nodejs library 'sqlite3'.
Explanation: Even though I have declared the variable 'image_path' in a scope from which it can be returned out of the function, when I assign it a value within the .get method, the scope of image_path appears to change; It is no longer able to be returned from the function 'returnImagePath' as expected.
Aim: I simply want to create a function with which I can query an Sqlite3 database using nodejs and return a value. If there is a better way of doing that then what I am currently trying, I will accept it as an answer.
Example:
var sqlite3 = require('sqlite3');
var dataBaseFile = 'test.db';
var dataBase = new sqlite3.Database(dataBaseFile);
function returnImagePath(id){
var image_path;
dataBase.serialize(function(){
var statementReturnImagePath = dataBase.prepare(`SELECT image_path FROM images WHERE id = ${id}`);
statementReturnImagePath.run();
statementReturnImagePath.finalize();
dataBase.get(`SELECT image_path FROM images WHERE id = 1`, function(error, row){
image_path = row.image_path;
console.log(image_path); //image_path is defined here, and logs as expected
});
dataBase.close();
});
console.log(image_path); //image_path is undefined here
return image_path;
}
var asdf = returnImagePath(1);
console.log(asdf); //this, therefore doesn't work
In the node.js world, you have to understand what is asynchronous function.
If a process requires IO operations, you cannot just return the data from that process, instead you have to "wait" for the data.
We usually achieve this with a callback function.
Here is the example for you,
var sqlite3 = require('sqlite3');
var dataBaseFile = 'test.db';
var dataBase = new sqlite3.Database(dataBaseFile);
//callback is a function which accept (err, data)
function returnImagePath(id, callback){
dataBase.serialize(function(){
dataBase.get('SELECT image_path FROM images WHERE id = ' + id, function(error, row){
if (error) {
//stop and return error if error occurs
return callback(error, null);
}
image_path = row.image_path;
console.log(image_path); //image_path is defined here, and logs as expected
//now you get the image path and return it with callback
callback(null, image_path);
dataBase.close();
});
});
}
returnImagePath(1, function(err, image_path) {
if (err) { //capture error(s)
return console.error(err);
}
console.log(image_path); //now you get the image path
});

Memory consideration using buffer and buffer.toString() in node.js streams

I am performing readstream.pipe(transformstream).pipe(writeStream). The application is read xml from a file system using readstream --> do some manipulation based on different tags using transform stream --> write it to another file system.
During the transform operation inside the transformstream. I am doing buffer.toString() for every chunk as I need to manipulate string before pushing them:
myTransformStream._transform = function(chunk, encoding, done) {
var stringTobeManipulated = chunk.toString();
// perform changes on stringTobeManipulated and push it to writestream
}
the size of xmls can be upto 3 mb and I noticed I got divided to ~44 chunks.
Based on the given problem I have questions concerning memory consumption both in v8 heap and system:
1) I understand that buffers are stored outside the v8 memory. When I do a chunk.toString() in my _transform function, does it create a javascript string object inside the v8 memory ? If yes, I am assuming it will be garbage collected after it looses all its references.
2) As the buffers are part of the system memory I believe they are not garbage collected, so when is that memory freed up ?
3) Is the application a good use case of transform stream as I am converting every chunk to string ?
EDITED:
May be I am not explaining myself clearly. Anyways I have been trying to find a ay of removing namespace from the xml tag before converting it to json. Here is the code I ended up with. Took advantage of memoization in javascrtipt. Please let me know if you guys find a better or more memory efficient way. We are using actionhero.js framework
var action = {};
var xml2js = require('xml2js');
var fs = require('fs');
/////////////////////////////////////////////////////////////////////
// metadata
action.name = 'removeNamespace';
action.description = 'I will parse the article related xml and yield the result';
action.inputs = {
'required': [],
'optional': []
};
action.blockedConnectionTypes = [];
action.outputExample = {
articleHeading: 'heading',
articleBody: 'body'
};
/////////////////////////////////////////////////////////////////////
// functional
action.run = function(api, connection, next) {
var stream = require('stream');
var removeNamespace = new stream.Transform();
var util = require('util');
var output = fs.createWriteStream('output.xml');
removeNamespace._transform = function(chunk, encoding, done) {
removeNamespace._transform.brokenTag = removeNamespace._transform.brokenTag || false;
removeNamespace._transform.brokenString = removeNamespace._transform.brokenString || '';
var convertedString = chunk.toString().trim();
if (removeNamespace._transform.brokenTag === true){
convertedString = removeNamespace._transform.brokenString + convertedString ;
}
removeNamespace._transform.brokenTag = false;
removeNamespace._transform.brokenString = '' ;
if (convertedString.lastIndexOf('<') > convertedString.lastIndexOf('>') ){
removeNamespace._transform.brokenString =convertedString.substring(convertedString.lastIndexOf('<'),convertedString.length+1);
convertedString = convertedString.substring(0,convertedString.lastIndexOf('<')) ;
removeNamespace._transform.brokenTag = true ;
}
convertedString = convertedString.replace(/>\s+</g, '><')
.replace.replace(/<[A-Za-z]+:/gi, '<')
.replace(/<\/[A-Za-z]+:/gi, '</');
done(null, convertedString);
};
var Writable = stream.Writable;
function xmlInMemory(keyToXml, options) {
Writable.call(this, options);
this[keyToXml] = new Buffer('');
this._write = function(chunk, encoding, callback) {
chunk = (Buffer.isBuffer(chunk)) ? chunk : new Buffer(chunk, encoding);
this[keyToXml] = Buffer.concat([this[keyToXml], chunk]);
callback();
};
}
util.inherits(xmlInMemory, Writable);
var source = fs.createReadStream('path/to/your/xmlfile');
var target = new xmlInMemory('keyToXml');
source.pipe(removeNamespace).pipe(target);
target.on('finish', function() {
var parser = new xml2js.Parser();
connection.response.xml2js = target.keyToXml.toString();
next(connection, true);
parser.parseString(target.keyToXml.toString(), function(err, result) {
connection.response.xml2js = result;
next(connection, true);
});
});
};
/////////////////////////////////////////////////////////////////////
// exports
exports.action = action;

I can't receive data from custom module in node.js

I wrote a module called accountManager.js
var sqlite3 = require('sqlite3');
var db = new sqlite3.Database("./users.db");
exports.userExists = function userExists(nickName) {
var stmt = 'SELECT * FROM users WHERE login="' + nickName + '"';
db.each(stmt,function(err,row) {
if(row) {
if(row.login==nickName) return true;
else return false;
}
});
}
In my main app.js file I've got
var accountManager = require('./lib/accountManager');
console.log(accountManager.userExists('user1'));
This app says 'undefined' in console... I checked that module is working fine, I guess it's problem with callback? Please, give me some help, I don't understand what is wrong with this code...
You need to understand how asynchronous functions and callbacks work.
Basically you cannot return anything inside the callback but need to invoke another callback which you pass to userExists.
var sqlite3 = require('sqlite3');
var db = new sqlite3.Database("./users.db");
exports.userExists = function userExists(nickName, cb) {
var stmt = 'SELECT * FROM users WHERE login="' + nickName + '"';
db.each(stmt,function(err,row) {
if(row) {
cb(row.login == nickName);
}
});
}
To use it:
accountManager.userExists('user1', function(found) {
console.log(found);
});
Besides that, your code has a gaping SQL injection hole and might not do what you intend to do. Here's a fixed version of the userExists function:
exports.userExists = function userExists(nickName, cb) {
var stmt = 'SELECT COUNT(*) AS cnt FROM users WHERE login = ?';
db.get(stmt, nickName, function(err, row) {
cb(row.cnt > 0);
});
};
Why is this better?
You do not interpolate the value in the SQL string (which is bad, you would have to escape stuff to avoid SQL injection). Passing it separately is much cleaner and better
You just want to know if a user exists. So retrieve the count (which will be exactly one row). If it's not zero the user exists.
Now the callback is always invoked. In the first example that is more closely based on your code it would only be invoked in case a user has been found - most likely not what you wanted.
You're returning a value from within the callback from db.each. However, this value is not returned by the outer function (userExists), which may return before the function passed to db.each is ever called.
You may want to provide a callback to the userExists function, like so:
exports.userExists = function (nickName, cb) {
var stmt = 'SELECT * FROM users WHERE login="' + nickName + '"';
var found=false;
db.each(stmt,function(err,row) {
if(row) {
if(row.login==nickName) {
found=true;
cb(true);
}
}
}, function () {
if (!found) {
cb(false);
}
});
}
Then, call it like:
var accountManager = require('./lib/accountManager');
accountManager.userExists('user1', function (found) {
console.log(found);
});

Resources