I wrote the following code to define a SynthDef that records a sound into the buffer passed as one of the parameters.
(
SynthDef(\recordTone, { |freq, bufnum, duration|
var w = SinOsc.ar(freq) * XLine.ar(101,1,duration,add: -1) / 100;
RecordBuf.ar(w!2,bufnum,loop: 0,doneAction: 2);
}).add;
)
I also have the below code that invokes a Synth for the above SynthDef and tried to write the buffer into a file.
({
var recordfn = { |freq, duration, fileName|
var server = Server.local;
var buf = Buffer.alloc(server,server.sampleRate * duration,2);
Synth(\recordTone,[\freq, 440, \bufnum, buf.bufnum, \duration, duration]);
buf.write(
"/Users/minerva/Temp/snd/" ++ fileName ++ ".wav",
"WAVE",
"int16",
completionMessage: ["b_free", buf.bufnum]
);
};
recordfn.value(440,0.5,"test");
}.value)
The output file is being created, but does not contain any audible sound. What am I doing wrong? I've looked through all the SuperCollider documentation I could find, but nothing seems to work! Any pointers is greatly appreciated.
Based on what Dan S answer, I made a few changes to get this working:
(
SynthDef(\playTone, { |freq, duration|
var w = SinOsc.ar(freq) * XLine.ar(1001,1,duration,add: -1,doneAction:2) / 1000;
Out.ar(0,w!2);
}).add;
)
(
SynthDef(\recordTone, { |buffer|
RecordBuf.ar(In.ar(0,2), buffer, loop: 0, doneAction: 2);
}).add;
)
(Routine({
var recordfn = { |freq, duration|
var server = Server.local;
var buffer = Buffer.alloc(server, server.sampleRate * duration, 2);
server.sync;
server.makeBundle(func: {
var player = Synth(\playTone, [\freq, freq, \duration, duration]);
var recorder = Synth.after(player, \recordTone, [\buffer, buffer]);
});
duration.wait;
buffer.write(
"/Users/minerva/Temp/snd/test.wav",
"WAVE",
"int16",
completionMessage: ["/b_free", buffer]
);
};
recordfn.value(440,0.1);
}).next)
Your main problem is that in your recordfn function you're instantiating the SynthDef (i.e. STARTING it recording) and writing the Buffer to disk at the same time. Obviously, at the point you START recording, there's no sound in the Buffer, so SuperCollider is doing exactly as you ask and writing the empty silent Buffer out as a file.
Solutions:
The most basic solution is to invoke one function to start recording, and a separate function when it's time to write to disk.
OR if you want it all in one, consider launching a Task within your function in order to wait until the Buffer is ready to be written to disk.
OR instead of RecordBuf use DiskOut which is for directly "spooling" to disk.
A secondary thing: I can't remember right now but I think it might be "WAV" not "WAVE".
Related
I have an external program that is streaming data to a csv file every now and then (but quit a lot).
I want to collect every 10 seconds all the changed data and do some processing on it.
means I want to process only lines I didn't processed before.
this is the basic code:
function myFunction() {
var loop = setInterval(
() =>
{
var instream = fs.createReadStream("rawData.csv"); //should somehow include only new data since last cycle
var outstream = fs.createWriteStream("afterProcessing.csv");
someProcessing(instream, outstream);
outstream.on('finish', () => {
sendBackResults("afterProcessing.csv");
});
//will exit the loop when 'run' flag will change to false
if(!run) ? clearInterval(loop) : console.log(`\nStill Running...\n`) ;
} , 10000 );
}
Now, I tried to work with chokidar and fs.watch but I couldn't figure out how to use them in this case.
fs.createReadStream can take a start parameter
options can include start and end values to read a range of bytes from
the file instead of the entire file. Both start and end are inclusive
and start counting at 0
So you need to save the last position read, and use it on start.
You can get that using: instream.bytesRead.
let bytesRead = 0;
instream.on('end', () => {
bytesRead = instream.bytesRead;
});
I'm trying to separate the 4 channel that are in a Buffer i receive from a 4-mic Array ReSpeaker. I'm using nodejs and currently i use a spawn command like:
spawn('arecord -r16000 -fS16_LE -traw -c4 -Dac108')
and then pipe the output in a transformer where i split the Buffer in the 4 channels and save them into separate file for check the result
const stream = require("stream");
const fs = require('fs');
class ChannelTransformer extends stream.Transform {
constructor(options) {
var write_1 = fs.createWriteStream('ch1', {encoding: 'binary'});
var write_2 = fs.createWriteStream('ch2', {encoding: 'binary'});
var write_3 = fs.createWriteStream('ch3', {encoding: 'binary'});
var write_4 = fs.createWriteStream('ch4', {encoding: 'binary'});
options.readableObjectMode = true;
options.writableObjectMode = true;
options.highWaterMark = 20000;
options.transform = (chunk, encoding, callback) => {
let channels = [[],[],[],[]];
for(let i=0; i<source.length;i++ ){
channels[i%4].push(chunk[i])
}
write_1.write(new Uint8Array(channels[0]));
write_2.write(new Uint8Array(channels[1]));
write_3.write(new Uint8Array(channels[2]));
write_4.write(new Uint8Array(channels[3]));
callback();
};
super(options);
}
}
As result from this code i get 4 file and if I import them with Audacity i find out that ch2 and ch4 files have been correctly separated, while ch1 and ch3 are corrupted and result in a white noise file.
Am i missing something on the separation? i thought that audio was stored on the pattern :
[[ch1_0],[ch2_0],[ch3_0],[ch4_0],[ch1_1],[ch2_1],...]
Also i dont get why, if the pattern i follow is not correct, 2 of the channels where separate succesfully.
I've also tried to cast the chunk into something else like:
let source = new Int8Array(chunk);
and then in the for cicle:
channels[i%4].push(source[i])
with different Type like Float32Array, Uint8Array, Uint16Array, Int16Array
but the result are the same.
I've already tested that the 4mic is working correctly by using the command:
arecord -r16000 -fS16_LE -traw -c4 -Dac108 -I ch1 ch2 ch3 ch4
which produce 4 file as expected containing each channel.
For every test, I block with mi finger on mic each couple of seconds while speaking so i can tell the difference between every channel.
Can anyone help me? or have some hints?
Thanks!
OK i figured out what was the problem.
Basically i was recording with bitWidth = 16, and the Buffer object in Node is an instance of Uint8Array so the pattern i was following was indeed correct, but due to the 8bitArray i had to assign 2 element per channel. Cause the format of the 8bit Array is:
[[ch1],[ch1],[ch2],[ch2],[ch3],[ch3],[ch4],[ch4],...]
Also casting the Array was useless because the produced 16bitArray didn't merge 2 8bit element to create a 16bit element but instead each 8bit element create the last 8bit of a 16bit element where the first 8bit are 0 like this:
8bitArray=[[11111111],[11111111],[2222222],[22222222],...]
16bitArray Casted= [[0000000011111111],[0000000011111111],[0000000022222222],...]
So i have created a method that merge the 8bitArray into the correct Array to be handle correctly based on the bitWidth of which your are recording:
var bitMultipler = bitWidth/8; //so i can handle any bitWidth with the same code
let channelsMap = new Map<number, Array<any>>();
for (let channel: number = 0; channel < totalChannels; channel++) {
channelsMap.set(channel, new Array())
}
/**
For each channel i push as many element as needed based on the bitMultipler
*/
let i = 0
while (i < chunk.length) {
for (let channel = 0; channel < totalChannels; channel++) {
for (let indexMultipler = 0; indexMultipler < bitMultipler; indexMultipler++) {
channelsMap.get(channel).push(chunk[i]);
i++;
}
}
}
My Node.js program - which is an ordinary command line program that by and large doesn't do anything operationally unusual, nothing system-specific or asynchronous or anything like that - needs to write messages to a file from time to time, and then it will be interrupted with ^C and it needs the contents of the file to still be there.
I've tried using fs.createWriteStream but that just ends up with a 0-byte file. (The file does contain text if the program ends by running off the end of the main file, but that's not the scenario I have.)
I've tried using winston but that ends up not creating the file at all. (The file does contain text if the program ends by running off the end of the main file, but that's not the scenario I have.)
And fs.writeFile works perfectly when you have all the text you want to write up front, but doesn't seem to support appending a line at a time.
What is the recommended way to do this?
Edit: specific code I've tried:
var fs = require('fs')
var log = fs.createWriteStream('test.log')
for (var i = 0; i < 1000000; i++) {
console.log(i)
log.write(i + '\n')
}
Run for a few seconds, hit ^C, leaves a 0-byte file.
Turns out Node provides a lower level file I/O API that seems to work fine!
var fs = require('fs')
var log = fs.openSync('test.log', 'w')
for (var i = 0; i < 100000; i++) {
console.log(i)
fs.writeSync(log, i + '\n')
}
NodeJS doesn't work in the traditional way. It uses a single thread, so by running a large loop and doing I/O inside, you aren't giving it a chance (i.e. releasing the CPU) to do other async operations for eg: flushing memory buffer to actual file.
The logic must be - do one write, then pass your function (which invokes the write) as a callback to process.nextTick or as callback to the write stream's drain event (if buffer was full during last write).
Here's a quick and dirty version which does what you need. Notice that there are no long-running loops or other CPU blockage, instead I schedule my subsequent writes for future and return quickly, momentarily freeing up the CPU for other things.
var fs = require('fs')
var log = fs.createWriteStream('test.log');
var i = 0;
function my_write() {
if (i++ < 1000000)
{
var res = log.write("" + i + "\r\n");
if (!res) {
log.on('drain',my_write);
} else {
process.nextTick(my_write);
}
console.log("Done" + i + " " + res + "\r\n");
}
}
my_write();
This function might also be helpful.
/**
* Write `data` to a `stream`. if the buffer is full will block
* until it's flushed and ready to be written again.
* [see](https://nodejs.org/api/stream.html#stream_writable_write_chunk_encoding_callback)
*/
export function write(data, stream) {
return new Promise((resolve, reject) => {
if (stream.write(data)) {
process.nextTick(resolve);
} else {
stream.once("drain", () => {
stream.off("error", reject);
resolve();
});
stream.once("error", reject);
}
});
}
You are writing into file using for loop which is bad but that's other case. First of all createWriteStream doesn't close the file automatically you should call close.
If you call close immediately after for loop it will close without writing because it's async.
For more info read here: https://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options
Problem is async function inside for loop.
Is there any chance to copy large files with Node.js with progress infos and fast?
Solution 1 : fs.createReadStream().pipe(...) = useless, up to 5 slower than native cp
See: Fastest way to copy file in node.js, progress information is possible (with npm package 'progress-stream' ):
fs = require('fs');
fs.createReadStream('test.log').pipe(fs.createWriteStream('newLog.log'));
The only problem with that way is that it takes easily 5 times longer compared "cp source dest". See also the appendix below for the full test code.
Solution 2 : rsync ---info=progress2 = same slow as solution 1 = useless
Solution 3 : My last resort, write a native module for node.js, using "CoreUtils" (linux sources for cp and others) or other functions as shown in Fast file copy with progress
Does anyone knows better than solution 3? I'd like to avoid native code but it seems the best fit.
thanks! any package recommendations or hints (tried all fs**) are welcome!
Appendix:
test code, using pipe and progress:
var path = require('path');
var progress = require('progress-stream');
var fs = require('fs');
var _source = path.resolve('../inc/big.avi');// 1.5GB
var _target= '/tmp/a.avi';
var stat = fs.statSync(_source);
var str = progress({
length: stat.size,
time: 100
});
str.on('progress', function(progress) {
console.log(progress.percentage);
});
function copyFile(source, target, cb) {
var cbCalled = false;
var rd = fs.createReadStream(source);
rd.on("error", function(err) {
done(err);
});
var wr = fs.createWriteStream(target);
wr.on("error", function(err) {
done(err);
});
wr.on("close", function(ex) {
done();
});
rd.pipe(str).pipe(wr);
function done(err) {
if (!cbCalled) {
console.log('done');
cb && cb(err);
cbCalled = true;
}
}
}
copyFile(_source,_target);
update: a fast (with detailed progress!) C version is implemented here: https://github.com/MidnightCommander/mc/blob/master/src/filemanager/file.c#L1480. Seems the best place to go from :-)
One aspect that may slow down the process is related to console.log. Take a look into this code:
const fs = require('fs');
const sourceFile = 'large.exe'
const destFile = 'large_copy.exe'
console.time('copying')
fs.stat(sourceFile, function(err, stat){
const filesize = stat.size
let bytesCopied = 0
const readStream = fs.createReadStream(sourceFile)
readStream.on('data', function(buffer){
bytesCopied+= buffer.length
let porcentage = ((bytesCopied/filesize)*100).toFixed(2)
console.log(porcentage+'%') // run once with this and later with this line commented
})
readStream.on('end', function(){
console.timeEnd('copying')
})
readStream.pipe(fs.createWriteStream(destFile));
})
Here are the execution times copying a 400mb file:
with console.log: 692.950ms
without console.log: 382.540ms
cpy and cp-file both support progress reporting
I have the same issue. I want to copy large files as fast as possible and want progress information. I created a test utility that tests the different copy methods:
https://www.npmjs.com/package/copy-speed-test
You can run it simply with:
npx copy-speed-test --source someFile.zip --destination someNonExistentFolder
It does a native copy using child_process.exec(), a copy file using fs.copyFile and it uses createReadStream with a variety of different buffer sizes (you can change buffer sizes by passing them on the command line. run npx copy-speed-test -h for more info.
Some things I learnt:
fs.copyFile is just as fast as native
you can get quite inconsistent results on all these methods, particularly when copying from and to the same disc and with SSDs
if using a large buffer then createReadStream is nearly as good as the other methods
if you use a very large buffer then the progress is not very accurate.
The last point is because the progress is based on the read stream, not the write stream. if copying a 1.5GB file and your buffer is 1GB then the progress immediately jumps to 66% then jumps to 100% and you then have to wait whilst the write stream finishes writing. I don't think that you can display the progress of the write stream.
If you have the same issue I would recommend that you run these tests with similar file sizes to what you will be dealing with and across similar media. My end use case is copying a file from an SD card plugged into a raspberry pi and copied across a network to a NAS so that's what I was the scenario that I ran the tests for.
I hope someone other than me finds it useful!
I solved a similar problem (using Node v8 or v10) by changing the buffer size. I think the default buffer size is around 16kb, which fills and empties quickly but requires a full cycle around the event loop for each operation. I changed the buffer to 1MB and writing a 2GB image fell from taking around 30 minutes to 5, which sounds similar to what you are seeing. My image was also decompressed on the fly, which possibly exacerbated the problem. Documentation on stream buffering has been in the manual since at least Node v6: https://nodejs.org/api/stream.html#stream_buffering
Here are the key code components you can use:
let gzSize = 1; // do not initialize divisors to 0
const hwm = { highWaterMark: 1024 * 1024 }
const inStream = fs.createReadStream( filepath, hwm );
// Capture the filesize for showing percentages
inStream.on( 'open', function fileOpen( fdin ) {
inStream.pause(); // wait for fstat before starting
fs.fstat( fdin, function( err, stats ) {
gzSize = stats.size;
// openTargetDevice does a complicated fopen() for the output.
// This could simply be inStream.resume()
openTargetDevice( gzSize, targetDeviceOpened );
});
});
inStream.on( 'data', function shaData( data ) {
const bytesRead = data.length;
offset += bytesRead;
console.log( `Read ${offset} of ${gzSize} bytes, ${Math.floor( offset * 100 / gzSize )}% ...` );
// Write to the output file, etc.
});
// Once the target is open, I convert the fd to a stream and resume the input.
// For the purpose of example, note only that the output has the same buffer size.
function targetDeviceOpened( error, fd, device ) {
if( error ) return exitOnError( error );
const writeOpts = Object.assign( { fd }, hwm );
outStream = fs.createWriteStream( undefined, writeOpts );
outStream.on( 'open', function fileOpen( fdin ) {
// In a simpler structure, this is in the fstat() callback.
inStream.resume(); // we have the _input_ size, resume read
});
// [...]
}
I have not made any attempt to optimize these further; the result is similar to what I get on the commandline using 'dd' which is my benchmark.
I left in converting a file descriptor to a stream and using the pause/resume logic so you can see how these might be useful in more complicated situations than the simple fs.statSync() in your original post. Otherwise, this is simply adding the highWaterMark option to Tulio's answer.
Here is what I'm trying to use now, it copies 1 file with progress:
String.prototype.toHHMMSS = function () {
var sec_num = parseInt(this, 10); // don't forget the second param
var hours = Math.floor(sec_num / 3600);
var minutes = Math.floor((sec_num - (hours * 3600)) / 60);
var seconds = sec_num - (hours * 3600) - (minutes * 60);
if (hours < 10) {hours = "0"+hours;}
if (minutes < 10) {minutes = "0"+minutes;}
if (seconds < 10) {seconds = "0"+seconds;}
return hours+':'+minutes+':'+seconds;
}
var purefile="20200811140938_0002.MP4";
var filename="/sourceDir"+purefile;
var output="/destinationDir"+purefile;
var progress = require('progress-stream');
var fs = require('fs');
const convertBytes = function(bytes) {
const sizes = ["Bytes", "KB", "MB", "GB", "TB"]
if (bytes == 0) {
return "n/a"
}
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)))
if (i == 0) {
return bytes + " " + sizes[i]
}
return (bytes / Math.pow(1024, i)).toFixed(1) + " " + sizes[i]
}
var copiedFileSize = fs.statSync(filename).size;
var str = progress({
length: copiedFileSize, // length(integer) - If you already know the length of the stream, then you can set it. Defaults to 0.
time: 200, // time(integer) - Sets how often progress events are emitted in ms. If omitted then the default is to do so every time a chunk is received.
speed: 1, // speed(integer) - Sets how long the speedometer needs to calculate the speed. Defaults to 5 sec.
// drain: true // drain(boolean) - In case you don't want to include a readstream after progress-stream, set to true to drain automatically. Defaults to false.
// transferred: false// transferred(integer) - If you want to set the size of previously downloaded data. Useful for a resumed download.
});
/*
{
percentage: 9.05,
transferred: 949624,
length: 10485760,
remaining: 9536136,
eta: 42,
runtime: 3,
delta: 295396,
speed: 949624
}
*/
str.on('progress', function(progress) {
console.log(progress.percentage+'%');
console.log('eltelt: '+progress.runtime.toString().toHHMMSS() + 's / hátra: ' + progress.eta.toString().toHHMMSS()+'s');
console.log(convertBytes(progress.speed)+"/s"+' '+progress.speed);
});
//const hwm = { highWaterMark: 1024 * 1024 } ;
var hrstart = process.hrtime(); // measure the copy time
var rs=fs.createReadStream(filename)
.pipe(str)
.pipe(fs.createWriteStream(output, {emitClose: true}).on("close", () => {
var hrend = process.hrtime(hrstart);
var timeInMs = (hrend[0]* 1000000000 + hrend[1]) / 1000000000;
var finalSpeed=convertBytes(copiedFileSize/timeInMs);
console.log('Done: file copy: '+ finalSpeed+"/s");
console.info('Execution time (hr): %ds %dms', hrend[0], hrend[1] / 1000000);
}) );
Refer to https://www.npmjs.com/package/fsprogress.
With that package, you can track progress while you are copying or moving files. The progress tracking is event and method call based so its very convenient to use.
You can provide options to do a lot of things. eg. total number of file for concurrent operation, chunk size to read from a file at a time.
It was tested for single file upto 17GB and directories up to i dont really remember but it was pretty large. And also :D, it is safe to use for large file(s).
So, go ahead and have a look at it whether it matches your expectations or if it is what you are looking for :D
I'm trying to play some mp3 files in node.js. The thing is that I manage to play them one by one, or even, as I want in parallel. But what I also want is to be able to control the amplitude (gain) to be able to create a crossfade in the end. Could anyone help me understand what it is I need to do? (I want to use it in node-webkit so I need a solution that is node.js based with no external dependencies.)
This is what I've got so far:
var lame = require('lame'), Speaker = require('speaker'), fs = require('fs');
var audioOptions = {channels: 2, bitDepth: 16, sampleRate: 44100};
var decoder = lame.Decoder();
var stream = fs.createReadStream("music/ge.mp3", audioOptions).pipe(decoder).on("format", function (format) {
this.pipe(new Speaker(format))
}).on("data", function (data) {
console.log(data)
})
I customized the npm package pcm-volume to do that. To crossfade, provide two pcm audio buffers (output of your decoders). Pipe the result to your Speaker object.
Here is the main part of the modifications. In this case the crossfade happens at the scale of the provided buffer, but you can change that.
var l = buf.length;
var out = new Buffer(l);
for (var i=0; i < l; i+=2) {
volumeSunrise = 0.5*this.volume*(1-Math.cos(pi*i/l));
volumeSunset = 0.5*this.volume*(1+Math.cos(pi*i/l));
uint = Math.round(volumeSunrise*buf.readInt16LE(i) + volumeSunset*this.sunsetBuffer.readInt16LE(i));
// you may want to ensure that -32767 <= uint <= 32768 here, in case you use a volume higher than 1
out.writeInt16LE(uint, i);
}
this.push(out);
callback()