With incomming data like STX(0x02)..Data..ETX(0x03)
I can process data by byte sequence parser:
var SerialPort = require('serialport');
var port = new SerialPort('/dev/tty-usbserial1', {
parser: SerialPort.parsers.byteDelimiter([3])
});
port.on('data', function (data) {
console.log('Data: ' + data);
});
But my actual incomming data is STX(0x02)..Data..ETX(0x03)..XX(plus 2 characters to validate data)
How can I get appropriate data?
Thanks!
Since version 2 or 3 of node-serialport, parsers have to inherit the Stream.Tansform class. In your example, that would become a new class.
Create a file called CustomParser.js :
class CustomParser extends Transform {
constructor() {
super();
this.incommingData = Buffer.alloc(0);
}
_transform(chunk, encoding, cb) {
// chunk is the incoming buffer here
this.incommingData = Buffer.concat([this.incommingData, chunk]);
if (this.incommingData.length > 3 && this.incommingData[this.incommingData.length - 3] == 3) {
this.push(this.incommingData); // this replaces emitter.emit("data", incomingData);
this.incommingData = Buffer.alloc(0);
}
cb();
}
_flush(cb) {
this.push(this.incommingData);
this.incommingData = Buffer.alloc(0);
cb();
}
}
module.exports = CustomParser;
Them use your parser like this:
var SerialPort = require('serialport');
var CustomParser = require('./CustomParser ');
var port = new SerialPort('COM1');
var customParser = new CustomParser();
port.pipe(customParser);
customParser.on('data', function(data) {
console.log(data);
});
Solved!
I write my own parser:
var SerialPort = require('serialport');
var incommingData = new Buffer(0);
var myParser = function(emitter, buffer) {
incommingData = Buffer.concat([incommingData, buffer]);
if (incommingData.length > 3 && incommingData[incommingData.length - 3] == 3) {
emitter.emit("data", incommingData);
incommingData = new Buffer(0);
}
};
var port = new SerialPort('COM1', {parser: myParser});
port.on('data', function(data) {
console.log(data);
});
Related
I am new to nodejs. Need your help. From the nodejs terminal, i want to download an excel file and convert it to csv (say, mocha online.js). Note: i don't want to do this via a browser.
Below is a script i am working on to download and convert to csv. There is no error nor the expected result:
online.js
if (typeof require !== 'undefined') XLSX = require('xlsx');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
/* set up XMLHttpRequest */
var url = "http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx";
var xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
xhr.responseType = "arraybuffer";
describe('suite', function () {
it('case', function () {
var arraybuffer = xhr.response;
/* convert data to binary string */
var data = new Uint8Array(arraybuffer);
var arr = new Array();
for (var i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
var bstr = arr.join("");
/* Call XLSX */
var sheetName = 'Database';
var workbook = XLSX.read(bstr, { type: "binary" });
var worksheet = workbook.Sheets[sheetName];
var csv = XLSX.utils.sheet_to_csv(worksheet);
console.log(csv);
xhr.send();
//.... perform validations here using the csv data
});
})}
I tried myself with this code, and it seems it is working, the only thing is that I spent 15 minutes trying to understand why my open office would not open the file, I eventually understood that they were sending a zip file ... here is the full code, the doc of the http get function is here http.get
You could have used the request module, but it isn't native, request is easier though.
enjoy!
const url = 'http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx'
const http = require('http')
const fs = require('fs')
http.get(url, (res) => {
debugger
const {
statusCode
} = res;
const contentType = res.headers['content-type'];
console.log(`The type of the file is : ${contentType}`)
let error;
if (statusCode !== 200) {
error = new Error(`Request Failed.\n` +
`Status Code: ${statusCode}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
res.setEncoding('binary');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
try {
const parsedData = xlsxToCSVFunction(rawData);
// And / Or just put it in a file
fs.writeFileSync('fileName.zip', rawData, 'binary')
// console.log(parsedData);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
function xlsxToCSVFunction(rawData) {
return rawData //you should return the csv file here whatever your tools are
}
I actually encountered the same problem 3 months ago : here is what I did!
I did not find any nodeJS module that was exactly as I wanted, so I used in2csv (a python shell program) to transform the data; the t option is to use tabulation as the delimiter
1) Step 1: transforming the xlsx file into csv using in2csv
This code takes all the xlsx files in the current directory, transform them into csv files and put them in another directory
var shelljs = require('shelljs/global')
var dir = pwd().stdout.split('/')
dir = dir[dir.length - 1].replace(/\s/g, '\\ ')
mkdir('../'+ dir + 'CSV')
ls('*.xlsx').forEach(function(file) {
// below are the two lines you need
let string = 'in2csv -t ' + file.replace(/\s/g, '\\ ') + ' > ../'+ dir + 'CSV/' + file.replace('xlsx','csv').replace(/\s/g, '\\ ')
exec(string, {silent:true}, function(code, stdout, stderr){
console.log('new file : ' + file.replace('xlsx','csv'))
if(stderr){
console.log(string)
console.log('Program stderr:', stderr)
}
})
});
Step 2: loading the data in a nodejs program:
my script is very long but the main two lines are :
const args = fileContent.split('\n')[0].split(',')
const content = fileContent.split('\n').slice(1).map(e => e.split(','))
And for the benefit of seekers like me...here is a solution using mocha, request and xlsx
var request = require('request');
var XLSX = require('xlsx');
describe('suite', function () {
it('case', function (done) {
var url = "http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx";
var options = {
url: url,
headers: {
'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
},
encoding: null
};
request.get(options, function (err, res, body){
var arraybuffer = body;
/* convert data to binary string */
var data = arraybuffer;
//var data = new Uint8Array(arraybuffer);
var arr = new Array();
for (var i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
var bstr = arr.join("");
/* Call XLSX */
var sheetName = 'Database';
var workbook = XLSX.read(bstr, { type: "binary" });
var worksheet = workbook.Sheets[sheetName];
var csv = XLSX.utils.sheet_to_csv(worksheet);
console.log(csv);
done();
});
});
});
Actually working on an automaton project (based on Arduino <-> Serial <-> Raspberry), I seem to be kind of stuck on the serial interface.
Basically, I'm using Express to read/write on some parameter on the arduino by URL in that kind of syntax (http://localhost:3000/parameter/6 or http://localhost:3000/parameter/6/set?value=10).
Since I'm willing to get a simple/short result (as an API), I decided to render a simple json object on each request, there is also no way for me to use post-rendering scripts/frameworks like Socket.io/jQuery/... for this purpose (I'd probably often call those URLs from curl/wget/[other html parsers]).
Now the thing is that there is no problem to write on the port, but for the reading, I'd like to wait for the buffer to be returned from serialport.on('data', function(buffer) { ... }) before rendering the page.
As serialport.on('data', ... ) does seem to be called only once per request, the only way I found for the moment is to redirect to the same page until the buffer has been read, which seem kind of a nasty way of getting things done, also it is incomplete ...
Here's a bit of code to get an idea of the actual logic :
Library :
// -> lib/serial.js - LIBRARY
var SerialPort = require("serialport");
var data = {'state': 0};
var serialPort = new SerialPort(port, {
baudrate: 115200,
parser: SerialPort.parsers.readline('\r\n'),
});
serialPort.on('open', function() {
console.log('Serial port Open');
serialPort.on('data', function(buffer) { data.buffer = buffer, data.state = 1 });
});
function readConfig(cmd, paramNb) {
var cmd = String.fromCharCode(cmd);
var param = String.fromCharCode(paramNb);
if (serialPort.isOpen() == true) {
serialPort.write(cmd + param + '\n', function(err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log('message written');
});
}
return data;
};
function writeConfig(cmd, paramNb, value) {
var cmd = String.fromCharCode(cmd);
var param = String.fromCharCode(paramNb);
var value = String.fromCharCode(value);
if (serialPort.isOpen() == true) {
serialPort.write(cmd + param + value + '\n', function(err) {
if (err) {
return console.log('Error on write: ', err.message);
}
console.log('message written');
});
}
};
exports.readConfig = readConfig;
exports.writeConfig = writeConfig;
Route :
// -> parameter.js - ROUTE
var express = require('express');
var router = express.Router();
var serial = require('../../lib/serial');
var sleep = require('sleep');
/* SET Parameter. */
router.get('/:id/set', function(req, res) {
var id = req.params.id;
var value = req.query.value;
serial.writeConfig('b', id, value);
res.json({'result': value, 'id': id});
});
/* GET Parameter. */
router.get('/:id', function(req, res) {
var id = req.params.id;
var buffer = serial.readConfig('a', id);
if (buffer.state != 0) {
res.json({'result': buffer});
}
else {
sleep.usleep(100000);
res.redirect('/api/parameter/' + id); // <- Nasty Bulls**t
}
});
module.exports = router;
The first idea I came up with was to find a synchronous way to read the port so I can call something like this (pseudo-code) :
while (buffer == '') { var buffer = serial.read; }
res.json({buffer});
Anyway thanks everyone.
When writing the tests for my entry file, index.js I run into the problem that the command mocha test passes test as an argument to index.js as it uses process.argv to receive parameters to run on a development environment. I had thought that by using something like minimist to name the parameters would fix this, however this problem still remains when running the tests. In this way my tests do not use the object provided in my test suits, as shown in the following code.
How do I get around this, so that when running my tests, it uses the event object I provide in my test set-up and not the command mocha test?
index.js
'use strict';
var _ = require("underscore");
var async = require('async');
var argv = require("minimist")(process.argv.slice(2));
var getprotocol = require("./getProtocol");
var _getprotocol = getprotocol.getProtocol;
var S3rs = require("./S3resizer");
var s3resizer = S3rs.rs;
var objCr = require("./objectCreator");
var createObj = objCr.creator;
var fileRs = require("./fileResizer");
var fileResizer = fileRs.rs;
var configs = require("./configs.json");
var mkDir = require("./makeDir");
var makeDir = mkDir.handler;
exports.imageRs = function (event, context) {
var _path = argv.x || event.path; //argv.x used to be process.argv[2]
console.log("Path, %s", _path);
var _dir = argv.y; // used to be process.argv[3]
console.log(_dir);
var parts = _getprotocol(_path);
var imgName = parts.pathname.split("/").pop();
console.log("imgName: %s", imgName);
var s3Bucket = parts.hostname;
var s3Key = imgName;
var _protocol = parts.protocol;
console.log(_protocol);
// RegExp to check for image type
var imageTypeRegExp = /(?:(jpg)|(png)|(jpeg))$/;
var sizesConfigs = configs.sizes;
var obj = createObj(_path);
// Check if file has a supported image extension
var imgExt = imageTypeRegExp.exec(s3Key);
if (imgExt === null) {
console.error('unable to infer the image type for key %s', s3Key);
context.done(new Error('unable to infer the image type for key %s' + s3Key));
return;
}
var imageType = imgExt[1] || imgExt[2];
// Do more stuff here
};
if (!process.env.LAMBDA_TASK_ROOT) {
exports.imageRs();
}
test.js
describe("imgeRs", function () {
var getprotocol = require("../getProtocol");
var S3rs = require("../S3resizer");
var objCr = require("../objectCreator");
var mkDir = require("../makeDir");
var fileResizer = require("../fileResizer");
describe("Calling S3", function () {
describe("Success call", function () {
var testedModule, eventObj, contextDoneSpy, S3resizerStub, objCreatorStub, getProtocolStub, fakeResults, mkDirStub, fileResizerStub;
before(function (done) {
contextDoneSpy = sinon.spy();
S3resizerStub = sinon.stub(S3rs, "rs");
objCreatorStub = sinon.stub(objCr, 'creator');
getProtocolStub = sinon.stub(getprotocol, "getProtocol");
mkDirStub = sinon.stub(mkDir, "handler");
fileResizerStub = sinon.stub(fileResizer, "rs");
eventObj = {"path": "s3://theBucket/image.jpeg"};
fakeResults = ["resized"];
testedModule = proxyquire("../index", {
'./getProtocol': {
'getProtocol': getProtocolStub
},
'./S3resizer': {
'rs': S3resizerStub
},
'./objectCreator': {
'creator': objCreatorStub
},
'./makeDir': {
'handler': mkDirStub
},
'./fileResizer': {
'rs': fileResizerStub
}
});
S3resizerStub.callsArgWith(5, null, fakeResults);
testedModule.imageRs(eventObj, {done: function (error) {
contextDoneSpy.apply(null, arguments);
done();
}});
});
after(function () {
S3rs.rs.restore();
objCr.creator.restore();
getprotocol.getProtocol.restore();
mkDir.handler.restore();
fileResizer.rs.restore();
});
it("calls context.done with no error", function () {
expect(contextDoneSpy).has.been.called;
});
});
});
});
I'm working on the nodeschool stream adventure and got to the duplex redux challenge.
I solved the challenge using the duplexer module but wanted to solve it using native stream functionality.
Here is the solution using duplexer:
var duplexer = require('duplexer');
var through = require('through');
module.exports = function (counter) {
var counts = {};
var input = through(write, end);
return duplexer(input, counter);
function write (row) {
counts[row.country] = (counts[row.country] || 0) + 1;
}
function end () { counter.setCounts(counts) }
};
So, it gets a readable stream in, and is returned a duplex stream. I'm trying to solve this again with:
var duplexer = require('duplexer');
var stream = require('stream');
module.exports = function (counter) {
var counts = {};
var ts = stream.Transform({ objectMode: true })
ts._transform = function(obj, enc, next) {
var country = obj.country;
var count = counts[country] || 0;
counts[obj.country] = count + 1;
next();
}
ts.on('finish', function() {
counter.setCounts(counts);
});
counter.pipe(ts)
return ts;
};
When run as is, it results in no output at all, so I changed next() to next(null, JSON.stringify(counts)). The stringify is because the next stream is not set to object mode. I get output, but its wrong and it throws stream.push() after EOF.
Is there a setsockopt/getsockopt-like socket options manipulation functionality in node.js?
I'm expanding on the comment left by socketpair which shows getsockopt. You can accomplish this by using ffi and ref. I've reformatted this to allow it to be easily manipulated.
I edited my comment because I had to make some changes to make the code work on both Linux and Win32. I had to create a node library for Windows to get the socket handle and pass it to setsockopt. Be aware that Linux and Windows may have different values for socket options
Edit: Here's a cleaned up piece of of production code I'm using:
var net = require("net");
var ffi = require("ffi");
var ref = require("ref");
var getSocketHandleAddress;
var SOL_SOCKET = 0x1;
var SO_OOBINLINE = 0xA;
var _setsockopt;
if (process.platform == "win32") {
SOL_SOCKET = 0xffff;
SO_OOBINLINE = 0x0100;
}
var setSocketOption = function (handle, level, option, value) {
if (!_setsockopt) {
var library;
var paramTypes;
if (process.platform === "win32") {
library = "ws2_32.dll";
paramTypes = [
ref.types.int,
ref.types.int,
ref.types.int,
ref.refType(ref.types.void),
ref.types.int
];
} else {
paramTypes = [
ref.types.int,
ref.types.int,
ref.types.int,
ref.refType(ref.types.void),
ref.refType(ref.types.int)
];
}
var lib = new ffi.DynamicLibrary(library);
_setsockopt = ffi.ForeignFunction(
lib.get("setsockopt"),
ref.types.int,
paramTypes);
}
var refType;
var length;
if (typeof value === "boolean") {
refType = ref.types.bool;
} else {
refType = ref.types.int;
}
if (process.platform !== "win32") {
return _setsockopt(
handle.fd,
level,
option,
ref.alloc(refType, value),
ref.alloc(ref.types.int, refType.size)
);
}
if (!getSocketHandleAddress) {
getSocketHandleAddress = require("getsockethandleaddress");
}
return _setsockopt(
getSocketHandleAddress.getAddress(handle),
level,
option,
ref.alloc(refType, value),
refType.size
);
};
var tcpserver = net.createServer(function (socket) {
var ret = setSocketOption(socket._handle, SOL_SOCKET, SO_OOBINLINE, true);
if (ret !== 0) {
console.error("OOB Inline socket option failed: " + ret);
}
});
This is my getsockopt:
var ffi = require('ffi');
var net = require('net');
var StructType = require('ref-struct');
var ref = require('ref');
var current = ffi.Library(null, {
'getsockopt': [ 'int', [ 'int', 'int', 'int', 'pointer', 'pointer']],
'ntohs': ['uint16', ['uint16']],
// const char *inet_ntop(int af, const void *src, char *dst, socklen_t size);
});
var SOL_IP = 0;
var SO_ORIGINAL_DST = 80;
var AF_INET = 2;
var sockaddr_in = StructType([
['int16', 'sin_family'],
['uint16', 'sin_port'],
['uint32', 'sin_addr'],
['uint32', 'trash1'],
['uint32', 'trash2'],
]);
function get_original_dst(client) {
var dst = new sockaddr_in;
var dstlen = ref.alloc(ref.types.int, sockaddr_in.size);
var r = current.getsockopt(client._handle.fd, SOL_IP, SO_ORIGINAL_DST, dst.ref(), dstlen);
if (r === -1)
throw new Error("getsockopt(SO_ORIGINAL_DST) error");
if (dst.sin_family !== AF_INET)
throw new Error("getsockopt(SO_ORIGINAL_DST) returns unknown family: " + dst.sin_family );
// TODO: inet_ntop. inet_ntoa is _UNSAFE_
var ipaddr = dst.ref(); ipaddr = ipaddr[4] + "." + ipaddr[5] + "." + ipaddr[6] + "." + ipaddr[7];
return [ipaddr, current.ntohs(dst.sin_port)];
}
module.exports.get_original_dst = get_original_dst;
Kind of late but here is a pacakge on npm https://www.npmjs.com/package/net-keepalive
Provides high-level access to socket options like TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT
var Net = require('net')
, NetKeepAlive = require('net-keepalive')
;
// Create a TCP Server
var srv = Net.createServer(function(s){>
console.log('Connected %j', s.address())
// Doesn't matter what it does
s.pipe(s)
});
// Start on some port
srv.listen(1337, function(){
console.log('Listening on %j', srv.address())
});
// Connect to that server
var s = Net.createConnection({port:1337}, function(){
console.log('Connected to %j', s.address())
//IMPORTANT: KeepAlive must be enabled for this to work
s.setKeepAlive(true, 1000)
// Set TCP_KEEPINTVL for this specific socket
NetKeepAlive.setKeepAliveInterval(s, 1000)
// and TCP_KEEPCNT
NetKeepAlive.setKeepAliveProbes(s, 1)
});