How to get synchronous SerialPort Read in NodeJS - node.js

Actually working on an automaton project (based on Arduino <-> Serial <-> Raspberry), I seem to be kind of stuck on the serial interface.
Basically, I'm using Express to read/write on some parameter on the arduino by URL in that kind of syntax (http://localhost:3000/parameter/6 or http://localhost:3000/parameter/6/set?value=10).
Since I'm willing to get a simple/short result (as an API), I decided to render a simple json object on each request, there is also no way for me to use post-rendering scripts/frameworks like Socket.io/jQuery/... for this purpose (I'd probably often call those URLs from curl/wget/[other html parsers]).
Now the thing is that there is no problem to write on the port, but for the reading, I'd like to wait for the buffer to be returned from serialport.on('data', function(buffer) { ... }) before rendering the page.
As serialport.on('data', ... ) does seem to be called only once per request, the only way I found for the moment is to redirect to the same page until the buffer has been read, which seem kind of a nasty way of getting things done, also it is incomplete ...
Here's a bit of code to get an idea of the actual logic :
Library :
// -> lib/serial.js - LIBRARY
var SerialPort = require("serialport");
var data = {'state': 0};
var serialPort = new SerialPort(port, {
baudrate: 115200,
parser: SerialPort.parsers.readline('\r\n'),
});
serialPort.on('open', function() {
console.log('Serial port Open');
serialPort.on('data', function(buffer) { data.buffer = buffer, data.state = 1 });
});
function readConfig(cmd, paramNb) {
var cmd = String.fromCharCode(cmd);
var param = String.fromCharCode(paramNb);
if (serialPort.isOpen() == true) {
serialPort.write(cmd + param + '\n', function(err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log('message written');
});
}
return data;
};
function writeConfig(cmd, paramNb, value) {
var cmd = String.fromCharCode(cmd);
var param = String.fromCharCode(paramNb);
var value = String.fromCharCode(value);
if (serialPort.isOpen() == true) {
serialPort.write(cmd + param + value + '\n', function(err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log('message written');
});
}
};
exports.readConfig = readConfig;
exports.writeConfig = writeConfig;
Route :
// -> parameter.js - ROUTE
var express = require('express');
var router = express.Router();
var serial = require('../../lib/serial');
var sleep = require('sleep');
/* SET Parameter. */
router.get('/:id/set', function(req, res) {
var id = req.params.id;
var value = req.query.value;
serial.writeConfig('b', id, value);
res.json({'result': value, 'id': id});
});
/* GET Parameter. */
router.get('/:id', function(req, res) {
var id = req.params.id;
var buffer = serial.readConfig('a', id);
if (buffer.state != 0) {
res.json({'result': buffer});
}
else {
sleep.usleep(100000);
res.redirect('/api/parameter/' + id); // <- Nasty Bulls**t
}
});
module.exports = router;
The first idea I came up with was to find a synchronous way to read the port so I can call something like this (pseudo-code) :
while (buffer == '') { var buffer = serial.read; }
res.json({buffer});
Anyway thanks everyone.

Related

Stop node-simplecrawler instance when creating a new one (make it behave like a singleton)

Heyllo, everyone!
I am making a scraper which uses node-simplecrawler. Everything runs fine, but what I can't figure out is how to stop one instance when creating a new one (I want to have only one running at a time). I am using express for this and all the scraping logic is in one route. In order to cancel the crawling right now, I need to stop the node process and run the app again.
Here is the part of the code that concerns running the crawler (note: I've simplified the code a little bit, so it's shorter):
module.exports = function(socket) {
var express = require('express');
var router = express.Router();
[... requires continue...]
/* GET scaning page. */
router.post('/', function(req, res, next) {
res.render('scanning'); // Load the socket.io host page
var render = {};
var pages = [];
var timer = new Date();
// Helper func to log the requests.
function log(message) {
var now = new Date();
console.log(now - timer + 'ms', message);
timer = now;
}
// Ensure URL format, parse URL
// Check if URL exist
request(url.href, function (error, response, body) {
if (!error && response.statusCode == 200) {
// URL exists, so let's scan it
// Exclude links to the following extensions:
var exclude = ['gif', 'jpg', 'jpeg', 'png', 'ico', 'bmp', 'ogg', 'webp',
'mp4', 'webm', 'mp3', 'ttf', 'woff', 'json', 'rss', 'atom', 'gz', 'zip',
'rar', '7z', 'css', 'js', 'gzip', 'exe', 'xml', 'svg'];
var exts = exclude.join('|');
var regexReject = new RegExp('\.(' + exts + ')', 'i');
var rootURL = url.protocol + '//' + url.host + '/';
// Crawler configuration
var crawler = new Crawler(url.host);
crawler.initialPort = 80;
crawler.initialPath = url.pathname;
crawler.maxConcurrency = 1;
crawler.ignoreWWWDomain = false; // This is a little suspicious...
crawler.filterByDomain = true; // Only URLs from the current domain
crawler.scanSubdomains = true;
crawler.downloadUnsupported = false;
crawler.parseHTMLComments = false;
crawler.parseScriptTags = false;
crawler.acceptCookies = false;
// crawler.maxDepth = 1 // Debug only!
/*
* Fetch Conditions
*/
// Get only URLs, ignore feeds, only from this host
crawler.addFetchCondition(function (parsedURL) {
return (
!parsedURL.path.match(regexReject) && // Only links
(parsedURL.path.search('/feed') === -1) && // Igrnore feeds
(parsedURL.host === url.host) // Page is from this domain
);
});
// Should we only include subpages?
if(onlySubpages) {
crawler.addFetchCondition(function(parsedURL) {
// return parsedURL.path.search(url.pathname) > -1;
return parsedURL.path.startsWith(url.pathname);
// console.log(url, parsedURL);
});
}
// Exclude urls with fragments?
if(excludeUrls.length >= 1 ) {
crawler.addFetchCondition(function(parsedURL) {
var urlFragmentsOk = true;
excludeUrlFragments.forEach(function(fragment) {
if(parsedURL.path.search('/'+fragment) > -1) {
urlFragmentsOk = false;
}
});
return urlFragmentsOk;
});
}
// Include only URLs with fragments
if(includeOnlyUrls.length >= 1) {
crawler.addFetchCondition(function(parsedURL) {
var urlFragmentsOk = false;
var includeUrlFragments = includeOnlyUrls.replace(/\s/, '').split(',');
includeUrlFragments.forEach(function(fragment) {
if(parsedURL.path.search('/'+fragment) !== -1) {
urlFragmentsOk = true;
}
});
return urlFragmentsOk;
});
}
// Run the crawler
crawler.start();
// Execute for each URL, on fetchcomplete
crawler.on('fetchcomplete', function(item, responseBuffer, response) {
[Do stuff with the scraped page]
});
// Completed crawling. Now let's get to work!
crawler.on('complete', function() {
[Get all scraped pages and do something with them]
});
// Error handling
crawler.on('queueerror', function(errorData, URLData) {
console.log('Queue error:', errorData, URLData);
});
crawler.on('fetchdataerror', function(queueitem, response) {
console.log('Fetch error:', queueitem, response);
});
crawler.on('fetchtimeout', function(queueItem, crawlerTimeoutValue) {
console.log('Fetch timeout:', queueItem, crawlerTimeoutValue);
});
crawler.on('fetchclienterror', function(queueItem, errorData) {
console.log('Fetch local error:', queueItem, errorData);
});
crawler.on('fetchtimeout', function(queueItem, crawlerTimeoutValue) {
console.log('Crawler timeout:', queueItem, crawlerTimeoutValue);
});
} else if(error) {
console.log(error);
}
});
});
return router;
}
Every simplecrawler instance has a stop method that can be called to prevent the crawler from making any further requests (requests won't be stopped in flight, however).
I would probably store the crawler instance in a scope outside of the route handler, check if its defined first thing in the route handler, in that case call the stop method and then construct a new scraper.
I stripped out a lot of the meat of your code, but something like this is what I had in mind:
module.exports = function(socket) {
var express = require('express');
var router = express.Router();
var Crawler = requrie('simplecrawler');
var crawler;
router.post('/', function(req, res, next) {
// Check if URL exist
request(url.href, function (error, response, body) {
if (!error && response.statusCode == 200) {
// Stop any crawler that's already running
if (crawler instanceof Crawler) {
crawler.stop();
}
// Crawler configuration
crawler = new Crawler(url.host);
crawler.initialPort = 80;
crawler.initialPath = url.pathname;
// Run the crawler
crawler.start();
// Execute for each URL, on fetchcomplete
crawler.on('fetchcomplete', function(item, responseBuffer, response) {
// [Do stuff with the scraped page]
});
// Completed crawling. Now let's get to work!
crawler.on('complete', function() {
// [Get all scraped pages and do something with them]
});
} else if(error) {
console.log(error);
}
});
});
return router;
}

Why does this Bottleneck-ed app stall?

I have the app below and it stalls (code below). And I have no idea why. I suspect I might be using the Bottleneck module the wrong way.
Disclaimer: I am trying to learn programming and NodeJS myself using this project. Please help.
Intro
The point of the app is to fetch data missing from documents in a DB by requesting a webpage and parsing it jQuery-style. Then saving the returned data to new keys in the document. The database consists of ~92 000 documents. The app uses the bottleneck, cheerio and request modules. I run the app on OS X.
The problem
If I set a limit to the number of requests, such as
var limiter = new bottleneck(5, 0);
The app stalls after the first batch (5 in this case). But why? I suspect something might be wrong with Bottleneck and how it expects my program to work. Something to do with callbacks per Bottleneck "Gotchas" maybe?
If I set no limit, the app kind-of-works. It fetches webpages and writes to the DB. However with a lot of errors due to resources being overloaded and thus slowly. This is how I tell bottleneck not to limit:
var limiter = new bottleneck(0, 0);
These are the kind of errors I get:
{ [Error: getaddrinfo ENOTFOUND www.vestnikverejnychzakazek.cz www.vestnikverejnychzakazek.cz:443]
code: 'ENOTFOUND',
errno: 'ENOTFOUND',
syscall: 'getaddrinfo',
hostname: 'www.vestnikverejnychzakazek.cz',
host: 'www.vestnikverejnychzakazek.cz',
port: 443 }
{ [Error: connect EMFILE 65.52.146.11:443 - Local (undefined:undefined)]
code: 'EMFILE',
errno: 'EMFILE',
syscall: 'connect',
address: '65.52.146.11',
port: 443 }
App code
'use strict';
var express = require('express');
var router = express.Router();
var assert = require('assert');
var mongo = require('mongoskin');
var path = require('path');
var ObjectID = require('mongodb').ObjectID;
var db = mongo.db("mongodb://localhost:27017/zak", {
native_parser: true
});
var database = db.collection("zakazky");
var cheerio = require("cheerio");
var request = require("request");
var fs = require("fs");
var toJs = (path.join(__dirname, '../public/javascripts', 'jquery.min.js'));
var jquery = fs.readFileSync(toJs).toString();
var bottleneck = require("bottleneck");
var limiter = new bottleneck(5, 0);
/* GET home page. */
router.get('/', function(req, res) {
var cursor = database.find();
cursor.each(function(err, data) {
assert.equal(err, null);
if (data != null) {
var vvz = "vestnikverejnychzakazek";
var praha = "zakazky.praha.eu";
var id = data["_id"];
var zdroj = data["zdroj"];
if (zdroj.indexOf(vvz) > -1) {
if ((data["cpv"] == null) || (data["predpokladana_hodnota"] == null)) {
limiter.submit(getCPV, id, zdroj, null);
// getCPV(id, zdroj);
} else {
// console.log("we're good");
return
}
} else if (zdroj.indexOf(praha) > -1) {
// console.log("pha");
}
} else {
// callback();
}
});
var getCPV = function(id, zdroj, callback) {
console.log("CPV started");
var zdroj = zdroj.replace("http://", "https://");
console.log("zdroj: " + zdroj);
var cpv = [];
var retryWrapper = function(retries) {
var retries; // I added this
if (retries === 3) {
return;
} else if (retries === undefined) {
retries = 0;
} else if (retries > 0) {
console.log("trying again");
}
request(zdroj, function(err, resp, data) {
if (err) {
console.log(err);
return retryWrapper(retries + 1);
}
var $ = cheerio.load(data);
var predpokladnaHodnota = $("[id*='Hodnota1_']").first().attr("value");
$("[id*='HlavniSlovnik']").each(function() {
cpv.push(this.attribs.value);
});
// let's check what we've got is actual data
if (cpv.length === 0) {
return
} else {
// send it off
writeCPV(id, "cpv", cpv)
}
if (predpokladnaHodnota == undefined || predpokladnaHodnota == null) {
return
} else {
// send it off
writeCPV(id, "predpokladana_hodnota", predpokladnaHodnota)
}
callback();
});
}; // end of retryWrapper
retryWrapper();
};
var writeCPV = function(id, key, value) {
id = ObjectID(id);
(function() {
console.log("starting DB write 1");
database.update({
"_id": id
}, {
$set: {
[key]: value
}
}, function(err, results) {
if (err) {
console.log("error in Mongo DB: \n------------------------\n" + err);
}
console.log("Mongo success!:\n ----------------------\n" + results);
// callback();
});
})();
};
// send the browser we're done
res.sendStatus(200);
});
// ---------------------
module.exports = router;
Here is a sample document from the DB including the fetched keys:
{
"_id": ObjectId("568d91396912101c1007ab4e"),
"cena": 1636363,
"cena_celkem": 1500000,
"cena_dopocitano": false,
"created": "2015-04-07T13:45:10.420739",
"datum_zadani": "2015-02-16",
"dodavatel": "/api/v1/dodavatel/381836/",
"druh_rizeni": "/api/v1/druh_rizeni/1116/",
"id": 1312587,
"modified": "2015-04-18T14:22:10.765733",
"nazev": "Pohostinství",
"pocet_nabidek": 2,
"podporeno_eu": true,
"popis": "Kurzy v oblasti pohostinství (formou profesní kvalifikace)",
"ramcova_smlouva": true,
"resource_uri": "/api/v1/zakazka/1312587/",
"skupina": "490648-ISVZUS_2011",
"typ_zakazky": "/api/v1/typ_zakazky/193/",
"zadavatel": "/api/v1/zadavatel/131528/",
"zdroj": "http://www.vestnikverejnychzakazek.cz/en/Form/Display/568547",
"zdroj_nazev": "isvzus.cz",
"cpv": ["80000000-4", "80400000-8", "", "", ""],
"predpokladana_hodnota": "1 500 000,00"
}
Sample URL being requested:
http://www.vestnikverejnychzakazek.cz/en/Form/Display/568547
This has been up here for a bit but in case anyone else stumbles upon this hopefully this will help someone!
The limiter is calling getCPV and it does call the callback at the end of the sequence, however, there are some conditional statements in retryWrapper that would allow an early return and as a result, never call the callback. The limiter will get piled up until it fires, so always make sure the callback will get fired in all of the scenarios.

How can i write a mocha test for the following function?

I want to write a test for this node.js funcion,This has two arguments request and response. I set the request variable . But dont know how to set response variable.
function addCustomerData(request, response) {
common.getCustomerByMobile(request.param('mobile_phone'), function (customerdata) {
if (!customerdata) {
var areaInterest = request.param('area_interest');
var customerInfo = {userType: request.param('userType'),
firstName : request.param('first_name'),
middleName : request.param('middle_name'),
lastName : request.param('last_name'),
email : request.param('email'),
mobilePhone : request.param('mobile_phone'),
uniqueName : request.param('user_name'),
company : request.param('company')
};
if(customerInfo.email){
customerInfo.email = customerInfo.email.toLowerCase();
}
if(customerInfo.uniqueName){
customerInfo.uniqueName = customerInfo.uniqueName.toLowerCase();
}
if(areaInterest) {
customerInfo.areaInterest = '{' + areaInterest + '}';
}else
areaInterest = null;
addCustomer(request, response, customerInfo, function (data) {
request.session.userId = data;
return response.send({success: true, message: 'Inserted successfully'});
}
);
} else {
return response.send({success: false, message: 'User with this mobile number already exists'});
}
});
}
I wrote the test as follows
describe('signup', function(){
describe('#addCustomer()', function(){
before(function (done) {
request = {};
request.data = {};
request.session = {};
request.data['userType'] = '3';
request.data['first_name'] = 'Shiji';
request.data['middle_name'] = '';
request.data['last_name'] = 'George';
request.data['email'] = 'shiji#lastplot.com';
request.data['mobile_phone'] = '5544332333';
request.data['user_name'] = 'shiji';
request.session['imageArray'] = [];
request.param=function(key){
// Look up key in data
return this.data[key];
};
request1 = {};
request1.data = {};
request1.session = {};
request1.data['area_interest']=["aluva","ernakulam"];
request1.data['userType'] = '1';
request1.data['first_name'] = 'Hari';
request1.data['middle_name'] = 'G';
request1.data['last_name'] = 'Ganesh';
request1.data['email'] = 'hari#lastplot.com';
request1.data['mobile_phone'] = '5544332321';
request1.data['user_name'] = 'hariganesh';
request1.data['company'] = 'HG Realestate';
request1.session['imageArray'] = [];
request1.param=function(key){
// Look up key in data
return this.data[key];
};
done();
});
it('It should list the matching properties', function(done){
async.parallel([
function(callback) {
signup.addCustomerData(request, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
},
function(callback) {
signup.addCustomerData(request1, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
}],function(){
done();
});
});
But i got the error as response has no method send()
Thanks in Advance.
Your addCustomerData function does not have a callback, it just calls respond.send(). You need to mock the response object, as well as the send method, and put your tests inside of it, but you won't be able to use async.parallel() as, like I already mentioned, your function does not have a callback parameter. If you're testing request/response functions, I suggest you look into Supertest https://github.com/visionmedia/supertest which is widely used for cases like this.

angularjs error on server callback

I'm making a call to the server using resource and when I go to the base URL of
/viewEvent
It works fine. I receive all the database entries. However, when I go to
/viewEvent/1234
where 1234 is the eventID
I get a undefined is not a function and this is a crash from within angular. Stack trace is
TypeError: undefined is not a function
at copy (http://localhost:8000/js/lib/angular/angular.js:593:21)
at http://localhost:8000/js/lib/angular/angular-resource.js:410:19
at wrappedCallback (http://localhost:8000/js/lib/angular/angular.js:6846:59)
at http://localhost:8000/js/lib/angular/angular.js:6883:26
at Object.Scope.$eval (http://localhost:8000/js/lib/angular/angular.js:8057:28)
at Object.Scope.$digest (http://localhost:8000/js/lib/angular/angular.js:7922:25)
at Object.Scope.$apply (http://localhost:8000/js/lib/angular/angular.js:8143:24)
at done (http://localhost:8000/js/lib/angular/angular.js:9170:20)
at completeRequest (http://localhost:8000/js/lib/angular/angular.js:9333:7)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8000/js/lib/angular/angular.js:9303:11) angular.js:575
When I examine the server, the request was made correctly. I can see that it got 1234 and it pulls the correct entry from the mongo database.
This is the controller logic
.controller("viewEventsController", ["$scope", 'EventService', '$location', function($scope, EventService, $location){
var path = $location.path().split('/');
var pathSize = path.length;
$scope.events = [];
if(pathSize === 2){
console.log("No event ID");
$scope.events = EventService.query();
}
else{
console.log("Event ID specified");
EventService.get({"eventID": path[pathSize - 1]}, function(data){
//$scope.events.push(data);
console.log(data);
}, function(error){
console.log(error);
});
}
}]);
and the service logic
service.factory('EventService', function($resource){
return $resource('api/viewEvent/:eventID');
});
It never makes it back to the controller so I'm "confident" it's not that. (watch it be that)
Not sure if the best way, but I got it working by doing
In service:
service.factory('EventService', function($resource){
return $resource('api/viewEvent/:eventID',
{eventID:"#eventID"},
{
'getSingleEvent': {
url: "api/viewEvent/:eventID",
method: "GET",
isArray: true
}
}
);
controller
var path = $location.path().split('/');
var pathSize = path.length;
EventService.getSingleEvent({"eventID":path[pathSize - 1]}, function(result){
$scope.updateEvent();
});
Server
routes = require('./routes')
var router = express.Router();
router.get('/api/viewEvent/:eventID', routes.viewEvent);
and in the routes directory I have a js file with
var mongoose = require('mongoose');
var db = mongoose.createConnection('localhost', 'eventApp');
var eventSchema = require('../models/createEvent.js').eventSchema;
var event = db.model('events', eventSchema);
exports.viewEvent = function(req, res){
console.log(req.params.eventID);
if(req.params.eventID) {
event.find({"_id": req.params.eventID}, function (error, events) {
console.log(events);
res.send(events);
});
}
else{
event.find({}, function (error, events) {
console.log(events);
res.send(events);
})
}
};

regarding foodme project in github

hello i have a question regarding the foodme express example over github:
code:
var express = require('express');
var fs = require('fs');
var open = require('open');
var RestaurantRecord = require('./model').Restaurant;
var MemoryStorage = require('./storage').Memory;
var API_URL = '/api/restaurant';
var API_URL_ID = API_URL + '/:id';
var API_URL_ORDER = '/api/order';
var removeMenuItems = function(restaurant) {
var clone = {};
Object.getOwnPropertyNames(restaurant).forEach(function(key) {
if (key !== 'menuItems') {
clone[key] = restaurant[key];
}
});
return clone;
};
exports.start = function(PORT, STATIC_DIR, DATA_FILE, TEST_DIR) {
var app = express();
var storage = new MemoryStorage();
// log requests
app.use(express.logger('dev'));
// serve static files for demo client
app.use(express.static(STATIC_DIR));
// parse body into req.body
app.use(express.bodyParser());
// API
app.get(API_URL, function(req, res, next) {
res.send(200, storage.getAll().map(removeMenuItems));
});
i don't understand where is the api folder. it doesn't exist and i don't understand how information is going in and out from there. i can't find it.
can someone please explain this to me?
another question:
there is a resource for the restaurant
foodMeApp.factory('Restaurant', function($resource) {
return $resource('/api/restaurant/:id', {id: '#id'});
});
and in the restaurant controller there is a query:
var allRestaurants = Restaurant.query(filterAndSortRestaurants);
and the following lines:
$scope.$watch('filter', filterAndSortRestaurants, true);
function filterAndSortRestaurants() {
$scope.restaurants = [];
// filter
angular.forEach(allRestaurants, function(item, key) {
if (filter.price && filter.price !== item.price) {
return;
}
if (filter.rating && filter.rating !== item.rating) {
return;
}
if (filter.cuisine.length && filter.cuisine.indexOf(item.cuisine) === -1) {
return;
}
$scope.restaurants.push(item);
});
// sort
$scope.restaurants.sort(function(a, b) {
if (a[filter.sortBy] > b[filter.sortBy]) {
return filter.sortAsc ? 1 : -1;
}
if (a[filter.sortBy] < b[filter.sortBy]) {
return filter.sortAsc ? -1 : 1;
}
return 0;
});
};
the things that isn't clear to me is:
how is that we are giving the query just a function without even activating it.
as i understand we should have passed the query somthing like:
{id: $routeParams.restaurantId}
but we only passed a reference to a function. that doesn't make any sense.
could someone elaborate on this?
thanks again.
var API_URL = '/api/restaurant';
var API_URL_ID = API_URL + '/:id';
var API_URL_ORDER = '/api/order';
These lines are just defining string constants that are plugged into Express further down. They're not a folder.
app.get(API_URL, function(req, res, next) {
res.send(200, storage.getAll().map(removeMenuItems));
});
So this function call to app.get(API_URL... is telling Express "Look out for GET requests that are pointed at the URL (your app's domain)/api/restaurant, and execute this function to handle such a request."
"api" is not a folder.
Every requests will pass through the app.get method.
This method will respond to the routes /api/restaurant as defined in the API_URL variable.

Resources