In BE server, I send binarystring to FE server. Binarystring is in responseText.
//So. I do this.\
xhr2.open('GET', url, true);
xhr2.onload = function()
{
routeResponse = Buffer.from(xhr2.responseText, 'binary');
//init
Buf = '';
//byte by byte output
for (let i = 0; i < routeResponse.length; i++) {
Buf += routeResponse.readUInt8(i).toString(16).toUpperCase();
Buf += ' ';
}
console.log(Buf);
}
But It's different binary data in log and original cgi file.
left : console.log(Buf) / Right : Hexadecimal value in original file. ex) getRoute.cgi
Oddly, only certain values are output as 'FD'. Actual data is '8B', '8C' and so on.
cgi file fomat : binary / Little Endian.
Why certain data are replaced to 'FD'
Please answer for me.
thank you.
Related
How to write a single file while reading from multiple input streams of the exact same file from diffrent locations with NodeJS.
As its still not Clear Maybe?
I want to use more performance for the download lets say we have 2 locations for the same file each can perform only 10mb down stream so i want to download a part from the first location and the secund in parallel. to get it with 20mb.
so both streams need to get joined some how and both streams need to know the range they are downloading.
i have 2 examples
var http = require('http')
var fs = require('fs')
// will write to disk __dirname/file1.zip
function writeFile(fileStream){
//...
}
// This example assums downloading from 2 http locations
http.request('http://location1/file1.zip').pipe(writeFile)
http.request('http://location2/file1.zip').pipe(writeFile)
var fs = require('fs')
// will write to disk __dirname/file1.zip
function writeFile(fileStream){
//...
}
// this example is reading the same file from 2 diffrent disks
fs.readfFile('/mount/volume1/file1.zip').pipe(writeFile)
fs.readfFile('/mount/volume2/file1.zip').pipe(writeFile)
How i think that it would work
ReadStream needs to check if a defined content range is already writen befor rereading the next chunk from each file and maybe they should start in on a random location in the file to read.
if the total file content length is X we will divide it into smaller chunks and create a map where each entry has a fixed content length so we know what parts we got and what parts we are downloading in total.
Trying to answer this question my self
We can try to simply optimistic raise Read
let SIZE = 64; // 64 byte intervals
let buffers = []
let bytesRead = 0
function readParallel(filepath,callback){
fs.open(filepath, 'r', function(err, fd) {
fs.fstat(fd, function(err, stats) {
let bufferSize = stats.size;
while (bytesRead < bufferSize) {
let size = Math.min(SIZE, bufferSize - bytesRead);
let buffer = new Buffer(size),
let position = bytesRead
let length = size
let offset = bytesRead
let read = fs.readSync(fd, buffer, offset, length, position);
buffers.push(buffer);
bytesRead += read;
}
});
});
}
// At the End: buffers.concat() ==== "File Content"
fs.createReadStream() has an option you can pass it to specify the start
let f = fs.createReadStream("myfile.txt", {start: 1000});
You could also open a normal file descriptor with fs.open(), then fs.read() one byte from a position right before where you want the stream to be positioned using the position argument to fs.read() and then you can pass that file descriptor into fs.createReadStream() as an option and the stream will start with that file descriptor and position (though obviously the start option to fs.createReadStream() is a bit simpler).
Using csv-parse with csv-stringify from the CSV Project.
const fs = require('fs');
const parse = require('csv-parse');
const stringify = require('csv-stringify')
const stringifier = stringify();
const writeFile = fs.createWriteStream('out.csv');
fs.createReadStream('file1.csv').pipe(parse()).pipe(stringifier).pipe(writeFile);
fs.createReadStream('file2.csv').pipe(parse()).pipe(stringifier).pipe(writeFile);
Here I parse each file separately (using a different parse stream for each source), then pipe both to the same stringify stream which concatenates them, then write to destination.
Range Locking
The Answer is Advisory Locking it is as simple as Torrent does it
assign the whole file or a part of it to multiple smaller parts
lock the file range and fetch that range from a list of sources.
use the file created in part 1 as driver for a FIFO Queue it contains all meta
To get a File from Multiple Sources a JS Implementation would look like
if we assume all files are only i put no error handling in here
const queue = [];
const sources = ['https://example.com/file','https://example1.com/file'];
const fileSize = fetch({sources[0],{method: 'HEAD'}).then(({ headers })=>headers['Content-Size']);
const targetBuffer = new UInt8Array(fileSize);
const charset = 'x-user-defined';
// Maps to the UTF Private Address Space Area so you can get bits as chars
const binaryRawEnablingHeader = `text/plain; charset=${charset}`;
const requestDefaults = {
headers: {
'Content-Type': binaryRawEnablingHeader,
'range': 'bytes=2-5,10-13'
}
}
const downloadPlan = /* some logic that puts that bytes into the target WiP */
// use response.text() and then convert that to byte via
// UNICODE Private Area 0xF700-0xF7ff.
const convertToAbyte = (chars) =>
new Array(chars.length)
.map((_abyte,offset) =>
chars.charCodeAt(offset) & 0xff);
I'm trying to separate the 4 channel that are in a Buffer i receive from a 4-mic Array ReSpeaker. I'm using nodejs and currently i use a spawn command like:
spawn('arecord -r16000 -fS16_LE -traw -c4 -Dac108')
and then pipe the output in a transformer where i split the Buffer in the 4 channels and save them into separate file for check the result
const stream = require("stream");
const fs = require('fs');
class ChannelTransformer extends stream.Transform {
constructor(options) {
var write_1 = fs.createWriteStream('ch1', {encoding: 'binary'});
var write_2 = fs.createWriteStream('ch2', {encoding: 'binary'});
var write_3 = fs.createWriteStream('ch3', {encoding: 'binary'});
var write_4 = fs.createWriteStream('ch4', {encoding: 'binary'});
options.readableObjectMode = true;
options.writableObjectMode = true;
options.highWaterMark = 20000;
options.transform = (chunk, encoding, callback) => {
let channels = [[],[],[],[]];
for(let i=0; i<source.length;i++ ){
channels[i%4].push(chunk[i])
}
write_1.write(new Uint8Array(channels[0]));
write_2.write(new Uint8Array(channels[1]));
write_3.write(new Uint8Array(channels[2]));
write_4.write(new Uint8Array(channels[3]));
callback();
};
super(options);
}
}
As result from this code i get 4 file and if I import them with Audacity i find out that ch2 and ch4 files have been correctly separated, while ch1 and ch3 are corrupted and result in a white noise file.
Am i missing something on the separation? i thought that audio was stored on the pattern :
[[ch1_0],[ch2_0],[ch3_0],[ch4_0],[ch1_1],[ch2_1],...]
Also i dont get why, if the pattern i follow is not correct, 2 of the channels where separate succesfully.
I've also tried to cast the chunk into something else like:
let source = new Int8Array(chunk);
and then in the for cicle:
channels[i%4].push(source[i])
with different Type like Float32Array, Uint8Array, Uint16Array, Int16Array
but the result are the same.
I've already tested that the 4mic is working correctly by using the command:
arecord -r16000 -fS16_LE -traw -c4 -Dac108 -I ch1 ch2 ch3 ch4
which produce 4 file as expected containing each channel.
For every test, I block with mi finger on mic each couple of seconds while speaking so i can tell the difference between every channel.
Can anyone help me? or have some hints?
Thanks!
OK i figured out what was the problem.
Basically i was recording with bitWidth = 16, and the Buffer object in Node is an instance of Uint8Array so the pattern i was following was indeed correct, but due to the 8bitArray i had to assign 2 element per channel. Cause the format of the 8bit Array is:
[[ch1],[ch1],[ch2],[ch2],[ch3],[ch3],[ch4],[ch4],...]
Also casting the Array was useless because the produced 16bitArray didn't merge 2 8bit element to create a 16bit element but instead each 8bit element create the last 8bit of a 16bit element where the first 8bit are 0 like this:
8bitArray=[[11111111],[11111111],[2222222],[22222222],...]
16bitArray Casted= [[0000000011111111],[0000000011111111],[0000000022222222],...]
So i have created a method that merge the 8bitArray into the correct Array to be handle correctly based on the bitWidth of which your are recording:
var bitMultipler = bitWidth/8; //so i can handle any bitWidth with the same code
let channelsMap = new Map<number, Array<any>>();
for (let channel: number = 0; channel < totalChannels; channel++) {
channelsMap.set(channel, new Array())
}
/**
For each channel i push as many element as needed based on the bitMultipler
*/
let i = 0
while (i < chunk.length) {
for (let channel = 0; channel < totalChannels; channel++) {
for (let indexMultipler = 0; indexMultipler < bitMultipler; indexMultipler++) {
channelsMap.get(channel).push(chunk[i]);
i++;
}
}
}
I have a BLE (Bluetooth 4.0)pedometer device from which I want to read the characteristic data using Evothings plugin (which uses Javascript API to interact with the device)
I am using following code -
callback when device is connected =
function deviceConnected(device) {
console.log('Connected to device: ' + device.name)
console.log('Reading services... Have patience!')
device.readServices(
null, // null means "read all services".
readAllServicesCharacteristicsAndNotifications,
// listAllServicesCharacteristicsDescriptors,
readServicesError)
}
callback code for readAllServicesCharacteristicsAndNotifications -
function readAllServicesCharacteristicsAndNotifications(device) {
// Get data for each service
console.log('number of services ' + device.__services.length)
console.log('number of characteristic ' + device.__services[0].__characteristics.length)
//var characteristic = service.__characteristics[characteristicUUID]
device.readCharacteristic(
device.__services[0].__characteristics[8]['uuid'],
function (data) {
var test = new Uint8Array(data);
var result = "";
for (var i = 0; i < test.length; i++) {
console.log('data element is ' + test[i]);
console.log('data element string is ' + String.fromCharCode(parseInt(test[i], 2)));
result += String.fromCharCode(parseInt(test[i], 2));
}
console.log('data is ' + result);
},
function (errorCode) {
console.log('BLE readCharacteristic error: ' + errorCode);
});
}
I think BLE data formatsuggests that data is exchanged as binary code bytes.
I see the value in the byte array but I don't know how to interpret it.
Has anyone used Evothings Javascript to interact with BLE device?
H i, the data received in JavaScript over BLE is a byte buffer.
See if you can find the documentation for the pedometer. Lookup at the format of the data sent from the pedometer. Then access the data in the buffer accordingly.
Here are some examples.
Access byte values in the data buffer:
var buf = new Uint8Array(data);
var value1 = buf[0];
var value2 = buf[1];
To get a 16-bit value use bitwise OR, for example:
var value = buf[0] | (buf[1] << 8);
Depending on data being little endian or big endian, the bitwise OR needs to be done in different ways.
Here is a library you can use that has functions for accessing values in a byte buffer: https://github.com/evothings/evothings-examples/blob/master/resources/libs/evothings/util/util.js
Here is an example:
// Get the signed int value in the buffer thats starts at
// byte index 2 (the int value represented by byte 2 and 3).
var value = evothings.util.littleEndianToInt16(data, 2)
Hope this helps!
Is there a limit the length of console.log output in Node.js? The following prints numbers up to 56462, then stops. This came up because we were returning datasets from MySQL and the output would just quit after 327k characters.
var out = "";
for (i = 0; i < 100000; i++) {
out += " " + i;
}
console.log(out);
The string itself seems fine, as this returns the last few numbers up to 99999:
console.log(out.substring(out.length - 23));
Returns:
99996 99997 99998 99999
This is using Node v0.6.14.
Have you tried writing that much on a machine with more memory?
According to Node source code console is writing into a stream: https://github.com/joyent/node/blob/cfcb1de130867197cbc9c6012b7e84e08e53d032/lib/console.js#L55
And streams may buffer the data into memory: http://nodejs.org/api/stream.html#stream_writable_write_chunk_encoding_callback
So if you put reeeaally a lot of data into a stream, you may hit the memory ceiling.
I'd recommend you split up your data and feed it into process.stdout.write method, here's an example: http://nodejs.org/api/stream.html#stream_event_drain
I would recommend using output to file when using node > 6.0
const output = fs.createWriteStream('./stdout.log');
const errorOutput = fs.createWriteStream('./stderr.log');
// custom simple logger
const logger = new Console(output, errorOutput);
// use it like console
var count = 5;
logger.log('count: %d', count);
// in stdout.log: count 5
Is it possible to write non-blocking response.write? I've written a simple test to see if other clients can connect while one downloads a file:
var connect = require('connect');
var longString = 'a';
for (var i = 0; i < 29; i++) { // 512 MiB
longString += longString;
}
console.log(longString.length)
function download(request, response) {
response.setHeader("Content-Length", longString.length);
response.setHeader("Content-Type", "application/force-download");
response.setHeader("Content-Disposition", 'attachment; filename="file"');
response.write(longString);
response.end();
}
var app = connect().use(download);
connect.createServer(app).listen(80);
And it seems like write is blocking!
Am I doing something wrong?
Update So, it doesn't block and it blocks in the same time. It doesn't block in the sense that two files can be downloaded simultaneously. And it blocks in the sense that creating a buffer is a long operation.
Any processing done strictly in JavaScript will block. response.write(), at least as of v0.8, is no exception to this:
The first time response.write() is called, it will send the buffered header information and the first body to the client. The second time response.write() is called, Node assumes you're going to be streaming data, and sends that separately. That is, the response is buffered up to the first chunk of body.
Returns true if the entire data was flushed successfully to the kernel buffer. Returns false if all or part of the data was queued in user memory. 'drain' will be emitted when the buffer is again free.
What may save some time is to convert longString to Buffer before attempting to write() it, since the conversion will occur anyways:
var longString = 'a';
for (...) { ... }
longString = new Buffer(longString);
But, it would probably be better to stream the various chunks of longString rather than all-at-once (Note: Streams are changing in v0.10):
var longString = 'a',
chunkCount = Math.pow(2, 29),
bufferSize = Buffer.byteLength(longString),
longBuffer = new Buffer(longString);
function download(request, response) {
var current = 0;
response.setHeader("Content-Length", bufferSize * chunkCount);
response.setHeader("Content-Type", "application/force-download");
response.setHeader("Content-Disposition", 'attachment; filename="file"');
function writeChunk() {
if (current < chunkCount) {
current++;
if (response.write(longBuffer)) {
process.nextTick(writeChunk);
} else {
response.once('drain', writeChunk);
}
} else {
response.end();
}
}
writeChunk();
}
And, if the eventual goal is to stream a file from disk, this can be even easier with fs.createReadStream() and stream.pipe():
function download(request, response) {
// response.setHeader(...)
// ...
fs.createReadStream('./file-on-disk').pipe(response);
}
Nope, it does not block, I tried one from IE and other from firefox. I did IE first but still could download file from firefox first.
I tried for 1 MB (i < 20) it works the same just faster.
You should know that whatever longString you create requires memory allocation. Try to do it for i < 30 (on windows 7) and it will throw FATAL ERROR: JS Allocation failed - process out of memory.
It takes time for memory allocation/copying nothing else. Since it is a huge file, the response is time taking and your download looks like blocking. Try it yourself for smaller values (i < 20 or something)