How to read non-byte-aligned integers with Node.js? - node.js

I am trying to read a binary SWF file using Node.JS. As the specification mentions at the bottom of the 17th page, some integers are encoded using variable-length bit fields, and by definition most of them are not byte-aligned.
The problem is that Node.js's Buffer only provides functions for reading byte-aligned integers. So I tried to write a wrapper object that would read bit by bit. However, it seems really hacky. Below is the object prototype I wrote:
/*jslint node:true, bitwise:true */
'use strict';
var util = require('util');
var maxBits = 32;
function BitBuffer() {
Buffer.apply(this, arguments);
this.cursor = 0;
this.bitCursor = 0;
}
util.inherits(BitBuffer, Buffer);
module.exports = BitBuffer;
function padBits(bits, length, bit) {
var leftPad = '', repeatLength, i;
bits = bits.toString(2);
length = length || 8;
bit = bit || '0';
if (bits.length >= length) {
return bits;
} else {
repeatLength = length - bits.length;
for (i = 0; i < repeatLength; i += 1) {
leftPad += bit;
}
return leftPad + bits;
}
}
BitBuffer.prototype.move = function (bits) {
var bytes = Math.floor((this.bitCursor + bits) / 8);
this.bitCursor += bits;
if (this.bitCursor > 8) {
this.cursor += bytes;
this.bitCursor -= bytes * 8;
}
if (this.cursor >= this.length) {
this.rewind();
return false;
}
return true;
};
BitBuffer.prototype.align = function () {
if (this.bitCursor > 0) {
this.bitCursor = 0;
this.cursor += 1;
}
};
BitBuffer.prototype.rewind = function () {
this.cursor = this.bitCursor = 0;
};
BitBuffer.prototype.readBits = function (bits) {
var bytes = Math.ceil((this.bitCursor + bits) / 8), result = 0,
length, buffer, i;
if (bits > maxBits) {
throw new RangeError('Cannot read more than ' + maxBits + ' bits');
}
buffer = this.slice(this.cursor, this.cursor + bytes);
result = padBits(buffer[0]).substr(this.bitCursor);
length = buffer.length;
for (i = 1; i < length; i += 1) {
result += padBits(buffer[i]);
}
result = result.substr(0, bits);
return (this.move(bits)) ? parseInt(result, 2) : false;
};
BitBuffer.prototype.readUB = BitBuffer.prototype.readBits;
BitBuffer.prototype.readSB = function (bits) {
var readBits = this.readBits(bits),
stringBits = readBits.toString(2);
if (readBits === false) {
return false;
}
// add automatically removed zeros
if (stringBits.length < bits) {
stringBits = padBits(stringBits, bits);
}
// negative, pad to 32 bits then invert bits
if (stringBits[0] === '1') {
return -~parseInt(padBits(stringBits, maxBits, '1'), 2) - 1;
} else {
return readBits;
}
};
BitBuffer.prototype.readFB = function (bits) {
var highBits, lowBits, result;
if (bits < 17) {
throw new RangeError('Should be at least 17 bits long');
}
highBits = this.readSB(bits - 16);
lowBits = this.readUB(16);
lowBits *= Math.pow(10, -lowBits.toString(10).length);
return (highBits >= 0) ?
highBits + lowBits : highBits - lowBits;
};
// wrap read functions
(function () {
var nativeReadFunctions = {
'readInt8': 8,
'readInt16LE': 16,
'readInt16BE': 16,
'readInt32LE': 32,
'readInt32BE': 32,
'readUInt8': 8,
'readUInt16LE': 16,
'readUInt16BE': 16,
'readUInt32LE': 32,
'readUInt32BE': 32,
'readDoubleLE': 64,
'readDoubleBE': 64,
'readFloatLE': 32,
'readFloatBE': 32
}, method;
function wrapNativeRead(method, length) {
return function (noAssert) {
var cursor;
this.align();
cursor = this.cursor;
this.move(length);
return Buffer.prototype[method].call(
this,
this.cursor,
noAssert
);
};
}
for (method in nativeReadFunctions) {
if (nativeReadFunctions.hasOwnProperty(method)) {
BitBuffer.prototype[method] =
wrapNativeRead(method, nativeReadFunctions[method]);
}
}
}());
Is writing my own object the good way?

Beside your implementation is pretty good, you should understand that in your implementation this.buffer === this and you should change the following lines:
// inside BitBuffer prototype constructor
this.buffer = new Buffer(param1, param2);
// change to
Buffer.call(this, param1, param2); // The right way to call the super constructor
and
// inside BitBuffer.prototype.readBits
buffer = this.buffer.slice(this.cursor, this.cursor + bytes);
// change to
buffer = this.slice(this.cursor, this.cursor + bytes);

Related

BIP32 derivePath different privatekey in nodejs and dart(flutter) (same mnemonic)

Hello i'm have some problem with BIP32
i use BIP32 library in nodejs and dart(flutter)
in dart(flutter) BIP32 derivePath show different privatekey but not of all mnemonic is just incorrect some mnemonic
in 1000 mnemonic different privatekey 1-2 mnemonic
below is dart derivePath,,
what is inccorect in this code:
Help me pls.
BIP32 derivePath(String path) {
final regex = new RegExp(r"^(m\/)?(\d+'?\/)*\d+'?$");
if (!regex.hasMatch(path)) throw new ArgumentError("Expected BIP32 Path");
List<String> splitPath = path.split("/");
if (splitPath[0] == "m") {
if (parentFingerprint != 0) throw new ArgumentError("Expected master, got child");
splitPath = splitPath.sublist(1);
print("splitPath: "+ splitPath);
}
print("splitPath: "+ splitPath);
return splitPath.fold(this, (BIP32 prevHd,String indexStr) {
int index;
if (indexStr.substring(indexStr.length - 1) == "'") {
index = int.parse(indexStr.substring(0, indexStr.length - 1));
return prevHd.deriveHardened(index);
} else {
index = int.parse(indexStr);
return prevHd.derive(index);
}
});
}
BIP32 derive(int index) {
if (index > UINT32_MAX || index < 0) throw new ArgumentError("Expected UInt32");
final isHardened = index >= HIGHEST_BIT;
Uint8List data = new Uint8List(37);
if (isHardened) {
if (isNeutered()) {
throw new ArgumentError("Missing private key for hardened child key");
}
data[0] = 0x00;
data.setRange(1, 33, privateKey);
data.buffer.asByteData().setUint32(33, index);
} else {
data.setRange(0, 33, publicKey);
data.buffer.asByteData().setUint32(33, index);
}
final I = hmacSHA512(chainCode, data);
final IL = I.sublist(0, 32);
final IR = I.sublist(32);
if (!ecc.isPrivate(IL)) {
return derive(index + 1);
}
BIP32 hd;
if (!isNeutered()) {
final ki = ecc.privateAdd(privateKey, IL);
if (ki == null) return derive(index + 1);
hd = BIP32.fromPrivateKey(ki, IR, network);
} else {
final ki = ecc.pointAddScalar(publicKey, IL, true);
if (ki == null) return derive(index + 1);
hd = BIP32.fromPublicKey(ki, IR, network);
}
hd.depth = depth + 1;
hd.index = index;
hd.parentFingerprint = fingerprint.buffer.asByteData().getUint32(0);
return hd;
}
BIP32 deriveHardened(int index) {
if (index > UINT31_MAX || index < 0) throw new ArgumentError("Expected UInt31");
return this.derive(index + HIGHEST_BIT);
}
OK can solved this problem problem occur in the ecuve function bytes is less than 32 bytes just padding 0 in the left equal 32 bytes just solved it !

How do I reverse a scanline using the jpeg-js module/node JS buffer?

I've been fiddling around with the jpeg-js module and Node JS Buffer, and attempting to create a small command line program that modifies the decoded JPEG buffer data and creates a pattern of X number of reversed scanlines and X number of normal scanlines before saving a new JPEG. In other words, I'm looking to flip portions of the image, but not the entire image itself (plenty of modules that do such a thing, of course, but not the specific use case I have).
To create the reversed/normal line patterns, I've been reading/writing line by line, and saving a slice of that line to a variable, then starting at the end of scanline and incrementally going down by slices of 4 bytes (the alloc for an RGBA value) until I'm at the beginning of the line. Code for the program:
'use strict';
const fs = require('fs');
const jpeg = require('jpeg-js');
const getPixels = require('get-pixels');
let a = fs.readFileSync('./IMG_0006_2.jpg');
let d = Buffer.allocUnsafe(a.width * a.height * 4);
let c = jpeg.decode(a);
let val = false; // track whether normal or reversed scanlines
let lineWidth = b.width * 4;
let lineCount = 0;
let track = 0;
let track2 = 0;
let track3 = 0;
let curr, currLine; // storage for writing/reading scnalines, respectively
let limit = {
one: Math.floor(Math.random() * 141),
two: Math.floor(Math.random() * 151),
three: Math.floor(Math.random() * 121)
};
if (limit.one < 30) {
limit.one = 30;
}
if (limit.two < 40) {
limit.two = 40;
}
if (limit.two < 20) {
limit.two = 20;
}
let calc = {};
calc.floor = 0;
calc.ceil = 0 + lineWidth;
d.forEach(function(item, i) {
if (i % lineWidth === 0) {
lineCount++;
/* // alternate scanline type, currently disabled to figure out how to succesfully reverse image
if (lineCount > 1 && lineCount % limit.one === 0) {
// val = !val;
}
*/
if (lineCount === 1) {
val = !val; // setting alt scanline check to true initially
} else if (calc.floor + lineWidth < b.data.length - 1) {
calc.floor += lineWidth;
calc.ceil += lineWidth;
}
currLine = c.data.slice(calc.floor, calc.ceil); // current line
track = val ? lineWidth : 0; // tracking variable for reading from scanline
track2 = val ? 4 : 0; // tracking variable for writing from scanline
}
//check if reversed and writing variable has written 4 bytes for RGBA
//if so, set writing source to 4 bytes at end of line and read from there incrementally
if (val && track2 === 4) {
track2 = 0; // reset writing count
curr = currLine.slice(track - 4, track); // store 4 previous bytes as writing source
if (lineCount === 1 && lineWidth - track < 30) console.log(curr); //debug
} else {
curr = currLine; //set normal scanline
}
d[i] = curr[track2];
// check if there is no match between data source and decoded image
if (d[i] !== curr[track2]) {
if (track3 < 50) {
console.log(i);
}
track3++;
}
track2++; //update tracking variable
track = val ? track - 1 : track + 1; //update tracking variable
});
var rawImageData = {
data: d,
width: b.width,
height: b.height
};
console.log(b.data.length);
console.log('errors\t', track3);
var jpegImageData = jpeg.encode(rawImageData, 100);
fs.writeFile('foo2223.jpg', jpegImageData.data);
Alas, the reversed scanline code I've written does not properly. Unfortunately, I've only been able successfully reverse the red channel of my test image (see below left), with the blue and green channels just turning into vague blurs. The color scheme should look something like the right image.
What am I doing wrong here?
For reversed lines, you stored slices of 4 bytes(4 bytes = 1 pixel), then write the first value of the pixel(red) correctly.
But in the next iteration, you overwrite the slice curr with currLine, rest of channels gets wrong values.
if (val && track2 === 4) {
track2 = 0; // reset writing count
curr = currLine.slice(track - 4, track); // store 4 previous bytes as writing source
if (lineCount === 1 && lineWidth - track < 30) console.log(curr); //debug
} else {
curr = currLine; //set normal scanline
}
Iteration 0: val == true, track2 == 4, set curr to next pixel, write red channel.
Iteration 1: val == true, track2 == 1, (val && track2 === 4) == false, set curr to currLine, write green channel.
You can move track2 === 4 branch to avoid this:
if (val) {
if (track2 === 4) {
track2 = 0; // reset writing count
curr = currLine.slice(track - 4, track); // store 4 previous bytes as writing source
if (lineCount === 1 && lineWidth - track < 30) console.log(curr); //debug
}
} else {
curr = currLine; //set normal scanline
}
Fixed code should look like this:
function flipAlt(input, output) {
const fs = require('fs');
const jpeg = require('jpeg-js');
let a = fs.readFileSync(input);
let b = jpeg.decode(a);
let d = Buffer.allocUnsafe(b.width * b.height * 4);
let val = false; // track whether normal or reversed scanlines
let lineWidth = b.width * 4;
let lineCount = 0;
let track = 0;
let track2 = 0;
let track3 = 0;
let curr, currLine; // storage for writing/reading scnalines, respectively
let limit = {
one: Math.floor(Math.random() * 141),
two: Math.floor(Math.random() * 151),
three: Math.floor(Math.random() * 121)
};
if (limit.one < 30) {
limit.one = 30;
}
if (limit.two < 40) {
limit.two = 40;
}
if (limit.two < 20) {
limit.two = 20;
}
let calc = {};
calc.floor = 0;
calc.ceil = 0 + lineWidth;
d.forEach(function(item, i) {
if (i % lineWidth === 0) {
lineCount++;
if (lineCount > 1) {
val = !val;
}
if (lineCount === 1) {
val = !val; // setting alt scanline check to true initially
} else if (calc.floor + lineWidth < b.data.length - 1) {
calc.floor += lineWidth;
calc.ceil += lineWidth;
}
currLine = b.data.slice(calc.floor, calc.ceil); // current line
track = val ? lineWidth : 0; // tracking variable for reading from scanline
track2 = val ? 4 : 0; // tracking variable for writing from scanline
}
//check if reversed and writing variable has written 4 bytes for RGBA
//if so, set writing source to 4 bytes at end of line and read from there incrementally
if (val) {
if (track2 === 4) {
track2 = 0; // reset writing count
curr = currLine.slice(track - 4, track); // store 4 previous bytes as writing source
if (lineCount === 1 && lineWidth - track < 30) console.log(curr); //debug
}
} else {
curr = currLine; //set normal scanline
}
d[i] = curr[track2];
// check if there is no match between data source and decoded image
if (d[i] !== curr[track2]) {
if (track3 < 50) {
console.log(i);
}
track3++;
}
track2++; //update tracking variable
track = val ? track - 1 : track + 1; //update tracking variable
});
var rawImageData = {
data: d,
width: b.width,
height: b.height
};
console.log(b.data.length);
console.log('errors\t', track3);
var jpegImageData = jpeg.encode(rawImageData, 100);
fs.writeFile(output, jpegImageData.data);
}
flipAlt('input.jpg', 'output.jpg');
Instead of tracking array indices, you can use utility library like lodash, it should make things easier:
function flipAlt(input, output) {
const fs = require('fs');
const jpeg = require('jpeg-js');
const _ = require('lodash');
const image = jpeg.decode(fs.readFileSync(input));
const lines = _.chunk(image.data, image.width*4);
const flipped = _.flatten(lines.map((line, index) => {
if (index % 2 != 0) {
return line;
}
const pixels = _.chunk(line, 4);
return _.flatten(pixels.reverse());
}));
const imageData = jpeg.encode({
width: image.width,
height: image.height,
data: new Buffer(flipped)
}, 100).data;
fs.writeFile(output, imageData);
}
flipAlt('input.jpg', 'output.jpg');

How to create sourcemaps for concatenated files

I want to concatenate a bunch of different files of a single type into one large file. For example, many javascript files into one large file, many css files down to one etc. I want to create a sourcemap of the files pre concatenation, but I do not know where to start. I am working in Node, but I am also open to solutions in other environments.
I know there are tools that can do this, but they seem to be on a language by language basis (uglifyjs, cssmin or whatever its called these days), but I want a tool that is not language specific.
Also, I would like to define how the files are bound. For example, in javascript I want to give each file its own closure with an IIFE. Such as:
(function () {
// File
}());
I can also think of other wrappers I would like to implement for different files.
Here are my options as I see it right now. However, I don't know which is best or how to start any of them.
Find a module that does this (I'm working in a Node.js environment)
Create an algorithm with Mozilla's source-map module. For that I also see a couple options.
Only map each line to the new line location
Map every single character to the new location
Map every word to its new location (this options seems way out of scope)
Don't even worry about source maps
What do you guys think about these options. I've already tried options 2.1 and 2.2, but the solution seemed way too complicated for a concatenation algorithm and it did not perform perfectly in the Google Chrome browser tools.
I implemented code without any dependencies like this:
export interface SourceMap {
version: number; // always 3
file?: string;
sourceRoot?: string;
sources: string[];
sourcesContent?: string[];
names?: string[];
mappings: string | Buffer;
}
const emptySourceMap: SourceMap = { version: 3, sources: [], mappings: new Buffer(0) }
var charToInteger = new Buffer(256);
var integerToChar = new Buffer(64);
charToInteger.fill(255);
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='.split('').forEach((char, i) => {
charToInteger[char.charCodeAt(0)] = i;
integerToChar[i] = char.charCodeAt(0);
});
class DynamicBuffer {
buffer: Buffer;
size: number;
constructor() {
this.buffer = new Buffer(512);
this.size = 0;
}
ensureCapacity(capacity: number) {
if (this.buffer.length >= capacity)
return;
let oldBuffer = this.buffer;
this.buffer = new Buffer(Math.max(oldBuffer.length * 2, capacity));
oldBuffer.copy(this.buffer);
}
addByte(b: number) {
this.ensureCapacity(this.size + 1);
this.buffer[this.size++] = b;
}
addVLQ(num: number) {
var clamped: number;
if (num < 0) {
num = (-num << 1) | 1;
} else {
num <<= 1;
}
do {
clamped = num & 31;
num >>= 5;
if (num > 0) {
clamped |= 32;
}
this.addByte(integerToChar[clamped]);
} while (num > 0);
}
addString(s: string) {
let l = Buffer.byteLength(s);
this.ensureCapacity(this.size + l);
this.buffer.write(s, this.size);
this.size += l;
}
addBuffer(b: Buffer) {
this.ensureCapacity(this.size + b.length);
b.copy(this.buffer, this.size);
this.size += b.length;
}
toBuffer(): Buffer {
return this.buffer.slice(0, this.size);
}
}
function countNL(b: Buffer): number {
let res = 0;
for (let i = 0; i < b.length; i++) {
if (b[i] === 10) res++;
}
return res;
}
export class SourceMapBuilder {
outputBuffer: DynamicBuffer;
sources: string[];
mappings: DynamicBuffer;
lastSourceIndex = 0;
lastSourceLine = 0;
lastSourceCol = 0;
constructor() {
this.outputBuffer = new DynamicBuffer();
this.mappings = new DynamicBuffer();
this.sources = [];
}
addLine(text: string) {
this.outputBuffer.addString(text);
this.outputBuffer.addByte(10);
this.mappings.addByte(59); // ;
}
addSource(content: Buffer, sourceMap?: SourceMap) {
if (sourceMap == null) sourceMap = emptySourceMap;
this.outputBuffer.addBuffer(content);
let sourceLines = countNL(content);
if (content.length > 0 && content[content.length - 1] !== 10) {
sourceLines++;
this.outputBuffer.addByte(10);
}
let sourceRemap = [];
sourceMap.sources.forEach((v) => {
let pos = this.sources.indexOf(v);
if (pos < 0) {
pos = this.sources.length;
this.sources.push(v);
}
sourceRemap.push(pos);
});
let lastOutputCol = 0;
let inputMappings = (typeof sourceMap.mappings === "string") ? new Buffer(<string>sourceMap.mappings) : <Buffer>sourceMap.mappings;
let outputLine = 0;
let ip = 0;
let inOutputCol = 0;
let inSourceIndex = 0;
let inSourceLine = 0;
let inSourceCol = 0;
let shift = 0;
let value = 0;
let valpos = 0;
const commit = () => {
if (valpos === 0) return;
this.mappings.addVLQ(inOutputCol - lastOutputCol);
lastOutputCol = inOutputCol;
if (valpos === 1) {
valpos = 0;
return;
}
let outSourceIndex = sourceRemap[inSourceIndex];
this.mappings.addVLQ(outSourceIndex - this.lastSourceIndex);
this.lastSourceIndex = outSourceIndex;
this.mappings.addVLQ(inSourceLine - this.lastSourceLine);
this.lastSourceLine = inSourceLine;
this.mappings.addVLQ(inSourceCol - this.lastSourceCol);
this.lastSourceCol = inSourceCol;
valpos = 0;
}
while (ip < inputMappings.length) {
let b = inputMappings[ip++];
if (b === 59) { // ;
commit();
this.mappings.addByte(59);
inOutputCol = 0;
lastOutputCol = 0;
outputLine++;
} else if (b === 44) { // ,
commit();
this.mappings.addByte(44);
} else {
b = charToInteger[b];
if (b === 255) throw new Error("Invalid sourceMap");
value += (b & 31) << shift;
if (b & 32) {
shift += 5;
} else {
let shouldNegate = value & 1;
value >>= 1;
if (shouldNegate) value = -value;
switch (valpos) {
case 0: inOutputCol += value; break;
case 1: inSourceIndex += value; break;
case 2: inSourceLine += value; break;
case 3: inSourceCol += value; break;
}
valpos++;
value = shift = 0;
}
}
}
commit();
while (outputLine < sourceLines) {
this.mappings.addByte(59);
outputLine++;
}
}
toContent(): Buffer {
return this.outputBuffer.toBuffer();
}
toSourceMap(sourceRoot?: string): Buffer {
return new Buffer(JSON.stringify({ version: 3, sourceRoot, sources: this.sources, mappings: this.mappings.toBuffer().toString() }));
}
}
I, at first, implemented "index map" from that spec, only to find out that it is not supported by any browser.
Another project that could be useful to look at is magic string.

Function failing to return a value

So I'm making a simple steganography tool (encrypting messages within images) and exposing it as a web service via Node.js. I am very new to Javascript and Node.js in particular. The app first converts a text string into a binary string by changing each character into an 8-bit ASCII encoding, resulting in one large binary string. I then encrypt the message within the pixels. Even values of pixels represent 0s from the binary, and odd values represent 1s. The end of the string is marked as 3 pixels of value 100 in a row (this is temporary, until I figure out a better way to mark the end). I'm using a node.js library called 'pngjs' that gives me pixel-level access to png images.
So I have a problem with the decodeMessage function. It builds up the string message, and is then meant to return it, however the return call at the end results in undefined.
How can I fix it?
Thanks in advance for the help!
function encodeMessage(image, mes) {
var message = mes;
var fs = require('fs'),
PNG = require('pngjs').PNG;
fs.createReadStream(image)
.pipe(new PNG({
filterType: 4
}))
.on('parsed', function() {
for (var y = 0; y < this.height; y++) {
for (var x = 0; x < this.width; x++) {
var idx = (this.width * y + x);// << 2;
//console.log(idx);
if (idx < message.length) {
var item = message.charAt(idx);
/* if the character in the encoded string is 0 */
if (item == 0) {
/* if the pixel is odd, we want it to be even */
if (this.data[idx] % 2 == 1) {
/* if the pixel is 0, add 1 to it */
if (this.data[idx] == 0) {
this.data[idx] = this.data[idx] + 1;
} else {
/* else substract 1 */
this.data[idx] = this.data[idx] - 1;
}
}
} else {
/* if the character in the encoded string is 1 */
if (this.data[idx] % 2 == 0) {
if (this.data[idx] == 0) {
this.data[idx] = this.data[idx] + 1;
} else {
this.data[idx] = this.data[idx] - 1;
}
}
}
//console.log(this.data[idx]);
} else if (idx === message.length) {
/* do something to the first pixel following the end of the string */
this.data[idx] = 100;
this.data[idx+1] = 100;
this.data[idx+2] = 100;
//console.log(this.data[idx]);
} else {
/* do something to the remaining pixels */
}
}
}
this.pack().pipe(fs.createWriteStream('encoded_' + image));
});
}
function decodeMessage(image) {
var message = "";
var fs = require('fs'),
PNG = require('pngjs').PNG;
fs.createReadStream(image)
.pipe(new PNG({
filterType: 4
}))
.on('parsed', function() {
dance:
for (var y = 0; y < this.height; y++) {
for (var x = 0; x < this.width; x++) {
var idx = (this.width * y + x);// << 2;
if (this.data[idx] == 100 && this.data[idx+1] == 100 && this.data[idx+2] == 100) {
break dance;
} else {
if (this.data[idx] % 2 == 0) {
message += "0";
} else {
message += "1";
}
}
}
}
/* the message outputs correctly over here */
console.log(message);
//return message;
});
/* but the return of the variable here doesn't work */
return message;
}
exports.encodeMessage = encodeMessage;
exports.decodeMessage = decodeMessage;
The parsed event is fired asynchronously, so you cannot return a value from decodeMessage.
function decodeMessage(image, cb) {
// Code
.on('parsed', function() {
// Code
console.log(message);
cb(message);
});
}
Then you must pass a callback to your decodeMessage function.
decodeMessage(image, function(decoded){
// Here is the decoded data.
});
The same is true for your encodeMessage function. The function will return before encoding has finished. If you want to know when it is done, you need to pass a callback the same way.

Issue in fetching float attributes in Sphinx using Node.js

I have been trying to fetch sphinx data using Node.js and limestone. I got everything from sphinx instead the float value. In my Sphinx index Height is defined as float value ("sql_attr_float = Height") and but the Node.js returning some integer value.
Eg: if Height value in sphinx is 172.72, then i got "1127004242" from the Node.js
Please help me on this.
Below is the function used in limestone file which reads sphinx data,
proto.toReader = function toReader() {
var offset = 0, length = this.length, buffer = this;
return {
empty: function empty() {
return offset >= length;
},
int64: function shiftInt64() {
var hi_low_pair = buffer.int64Read(offset);
offset += 8;
return hi_low_pair;
},
int32: function shiftInt32() {
var number = buffer.int32Read(offset);
offset += 4;
return number;
},
int16: function shiftInt16() {
var number = buffer.int16Read(offset);
offset += 2;
return number;
},
buffer: function shiftBuffer(length) {
var b = buffer.slice(offset, offset + length);
offset += length;
return b;
},
string: function shiftString(length) {
var string = buffer.toString('utf8', offset, offset + length);
offset += length;
return string;
},
cstring: function shiftCstring() {
var end = offset;
while (buffer[end] > 0 && end < length) { end++; }
var string = buffer.toString('utf8', offset, end);
offset = end + 1;
return string;
},
lstring: function shiftLString() {
var length = buffer.int32Read(offset);
offset += 4;
if(!isNaN(length) && !isNaN(offset)){
length = length;
var string = buffer.toString('utf8', offset, offset + length);
}else{
var string = "";
}
offset += length;
return string;
},
multicstring: function shiftMulticstring() {
var fields = [];
while (buffer[offset] > 0) {
fields.push(this.cstring());
}
offset++;
return fields;
},
hash: function shiftHash() {
var hash = {};
while (buffer[offset] > 0) {
hash[this.cstring()] = this.cstring();
}
offset++;
return hash;
}
};
};
Hmm, line 715, of
https://github.com/kurokikaze/limestone/blob/master/limestone.js
has
// FLOAT size attributes (32 bits)
if (output.attributes[attribute].type == Sphinx.attribute.FLOAT) {
attr_value = response.int32();
match.attrs[output.attributes[attribute].name] = attr_value;
continue;
}
So its just reading the float as int32!
And as you say toReader/makeWriter is missing float methods.
So you need to figure out how to decode that value yourself, and either patch limestone to directly handle floats, or just do it yourself in the application.
Checking SphinxAPI code, this is how it done in php:
// handle floats
if ( $type==SPH_ATTR_FLOAT )
{
list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4;
list(,$fval) = unpack ( "f*", pack ( "L", $uval ) );
$attrvals[$attr] = $fval;
continue;
}
java can do it somewhat nativly
/* handle floats */
if ( type==SPH_ATTR_FLOAT )
{
docInfo.attrValues.add ( attrNumber, new Float ( in.readFloat() ) );
continue;
}
For ports of pack/unpack:
pack / unpack functions for node.js

Resources