new MediaRecorder(stream[, options]) stream can living modify? - mediarecorder

new MediaRecorder(stream[, options]);
I want record the user camera and audio
I need mixing the song.mp3 to the audio track in recording.
and result export a video file to download by link.
But the MediaRecorder first params stream can living modify ?
But When I use recoder.stop()
It tips error: Failed to execute 'stop' on 'MediaRecorder': The MediaRecorder's state is 'inactive'.
My code:
function getFileBuffer(filepath) {
return fetch(filepath, {method: 'GET'}).then(response => response.arrayBuffer())
}
function mp3play() {
getFileBuffer('song.mp3')
.then(buffer => context.decodeAudioData(buffer))
.then(buffer => {
console.log(buffer)
const source = context.createBufferSource()
source.buffer = buffer
let volume = context.createGain()
volume.gain.value = 1
source.connect(volume)
dest = context.createMediaStreamDestination()
volume.connect(dest)
// volume.connect(context.destination)
source.start(0)
const _audioTrack = stream.getAudioTracks();
if (_audioTrack.length > 0) {
_audioTrack[0].stop();
stream.removeTrack(_audioTrack[0]);
}
//
// console.log(dest.stream)
// console.log(dest.stream.getAudioTracks()[0])
// stream.addTrack(dest.stream.getAudioTracks()[0])
})
}
function startRecording() {
recorder = new MediaRecorder(stream, {
mimeType: 'video/webm'
})
recorder.start()
stopBtn.removeAttribute('disabled')
startBtn.disabled = true
}

No we still can't record a MediaStream whose tracks are changed after the recording began, doing so will stop() the MediaRecorder. Here is a very related Q/A which was more about recording video.
What can be done though is to create a kind of merger MediaStream.
It's way easier with audio, moreover since you are already using the WebAudio API: all you need to do is to create an other MediaStreamDestination node, and connect / disconnect the different sources.
const base = "https://upload.wikimedia.org/wikipedia/en/d/";
const urls = [
"d3/Beach_Boys_-_Good_Vibrations.ogg",
"dc/Strawberry_Fields_Forever_%28Beatles_song_-_sample%29.ogg"
].map( url => base + url );
const context = new AudioContext();
const button = document.querySelector( 'button' );
button.onclick = async () => {
button.disabled = true;
context.resume();
const audiobuffers = await Promise.all( urls.map( fetchAsAudioBuffer ) );
button.remove();
const streamNode = context.createMediaStreamDestination();
const stream = streamNode.stream;
const recorder = new MediaRecorder( stream );
const chunks = [];
recorder.ondataavailable = evt => chunks.push( evt.data );
recorder.onstop = evt => exportAudio( new Blob( chunks ) );
document.getElementById( 'record-stopper' ).onclick = evt => {
recorder.stop();
current_source.stop( 0 );
};
let current_index = 0;
let current_source = null;
document.getElementById( 'switcher' ).onclick = switchAudioSource;
switchAudioSource();
recorder.start();
function switchAudioSource() {
if( current_source ) {
current_source.stop( 0 );
}
current_index = (current_index + 1) % audiobuffers.length;
current_source = context.createBufferSource();
current_source.buffer = audiobuffers[ current_index ];
current_source.loop = true;
current_source.connect( streamNode );
current_source.connect( context.destination );
current_source.start( 0 );
}
};
function exportAudio( blob ) {
const aud = new Audio( URL.createObjectURL( blob ) );
aud.controls = true;
document.body.prepend( aud );
}
async function fetchAsAudioBuffer( url ) {
const buf = await fetchAsBuffer( url );
return context.decodeAudioData( buf );
}
async function fetchAsBuffer( url ) {
const resp = await fetch( url );
return resp.arrayBuffer();
}
button+.recording-controls,
audio+.recording-controls {
display: none;
}
<button>begin</button>
<div class="recording-controls">
<label>Recording...</label>
<button id="switcher">Switch Audio Sources</button>
<button id="record-stopper">Stop Recording</button>
</div>
For video that would imply recording a CanvasMediaStreamTrack and drawing the different video streams on the source <canvas>, but we generally loose a lot of quality doing so...

Related

Puppeteer create PDF files from HTML data hangs Windows 10 system

I created an App that processes students results by extracting data from multiple excel workbooks. The problem is that using Puppeteer to generate the PDF files, throws the system into a loop till it hangs the system.
Actually, I have tested same codes below using PhantomJs which is bundled as pdf-creator-node, and was able to generate 150 PDF files comfortably in 3 minutes. The only challenge I dumped PhantomJs is that all the styling in the CSS file was not included, even when I inserted it as an inline style in the header, suing replace function of JS. Another, is that PhantomJs is no longer in active development. I searched the web, and found out that only Puppeteer is the valid solution with active development and support too.
I tried using page.close() at the end of pdfCreator() which is in a loop, and browser.close() at the end of pdfGenerator(). What I am doing wrong?
Here below are the codes in the server.js and PdfGenerator.js files, with a sample of the ERROR, and screenshot of my Task Manager after the system crawled out of hanging state. For HTML generation, I used Mustache. I excluded some lines of codes in server.js because the total character count was over 60k.
server.js
// [codes were removed here]
if(getCode == 'compute-result') {
// declare variable
let setData = null;
let setTitle = 'Results Computation...';
let setArgs = getArgs;
// dataFromFile = ReadFile(pathCodeTextFile);
// setArgs = Number(dataFromFile);
setCode = 'compute-result';
let setView = [];
let setNext = true;
let countTerms = [];
// if(getArg > 0) {
// Final Result computation
const getJson = ReadFile(pathJsonResults);
// const getCtrl = ReadFile(pathJsonCtrl);
const getResultObject = JSON.parse(getJson);
getResult = getResultObject;
const totalResults = getResult.firstTerm.length + getResult.secondTerm.length + getResult.thirdTerm.length;
if(setView.length < 1 && getResult != null) {
setData = 'PDFs for Students Results initiating...';
setView.unshift('Reading saved data...');
client.emit('query', {data: setData, title: setTitle, code: setCode, next: setNext, args: null, view: JSON.stringify(setView)});
}
Sleep(2000).then(() => {
if(getResult != null) {
setData = 'Students Results will be ready in a moment';
client.emit('query', {data: setData, title: setTitle, code: setCode, next: setNext, args: setArgs, view: JSON.stringify(setView)});
}
const wacthFiles = (file, className, termName, sessionName, completed, pdfList) => {
try {
if(typeof file == 'string' && !FileExists(pathJsonPdfList)) {
if(pdfList.length < 2){
setData = 'Saving PDFs to downladable files...';
}
if(className != null && termName != null && sessionName != null) {
setTitle = `${pdfList.length} Result PDF${pdfList.length > 1?'s':''}...`;
setView.unshift(file);
if(!countTerms.includes(termName)) {
countTerms.push(termName)
}
// setCode = -1000 - pdfList.length;
// console.log('PDF PROGRESS: ', `${pdfList.length} Result PDF${pdfList.length > 1?'s':''}... ${setCode}`);
// when all PDFs are created
if(completed) {
setTitle = setTitle.replace('...', ' [completed]');
setData = 'Result Download button is Active. You may click it now.';
setView.unshift('=== PDF GENERATION COMPLETED ===');
setView.unshift(`A total of ${pdfList.length} students' Results were generated`);
WriteFile(pathJsonPdfList, JSON.stringify(pdfList));
// set donwload button active
setCode = Number(codeTextFilePdfCompleted);
setNext = false;
getResult = null;
let termString = countTerms.toString();
termString = ReplaceAll(termString, '-term', '');
termString = ReplaceAll(termString, ',', '-');
const addTxt = `${className} _${termString} Term${countTerms.length>1?'s':''} (${sessionName})`;
WriteFile(pathCodeTextFile, addTxt);
// console.log('======== PDF GENERATION ENDS ================');
} else {
setCode = -1 * pdfList.length;
}
client.emit('query', {data: setData, title: setTitle, code: setCode, next: setNext, args: setArgs, view: JSON.stringify(setView)});
}
}
} catch (error) {
console.log('ERROR ON WATCHER: ', error);
}
}
if(!FileExists(pathJsonPdfList) && getResult !== null) {
PdfGenerator(getResult, wacthFiles);
}
// Watcher(pathWatchResults, setCode, wacthDir, 10000);
});
// }
}
}
} catch (error) {
})
client.on('disconnect', () => {
console.log('SERVER: Disconnected');
});
server.listen(portApi, () =>{
console.log('Server listens on port 8881')
});
// serve static files
app.use(express.static(pathPublic));
// [codes were removed here]
PdfGenerator.js
The problem lies in these functions: PdfGenerator & createPdf
'use strict';
process.setMaxListeners(Infinity) // fix for Puppeteer MaxListenerExceededWarning
const Puppeteer = require('puppeteer')
const {HtmlGenerator} = require('../components/HtmlGenerator')
const {WriteFile, FileExists, RandomNumber, RoundNumber, IsNumberFraction, ReadFile} = require('../components/Functions')
if (process.env.NODE_ENV !== 'production') {
require('dotenv').config();
}
const pathFirstTermResults = process.env.DIR_FIRST_TERM_RESULTS;
const pathSecondTermResults = process.env.DIR_SECOND_TERM_RESULTS;
const pathThirdTermResults = process.env.DIR_THIRD_TERM_RESULTS;
const publicDir = process.env.DIR_PUBLIC;
const cssFile = process.env.PATH_CSS_FILENAME;
const pathCssRaw = __dirname + '\\' + publicDir + '\\' + cssFile;
const pathCss = pathCssRaw.replace(`\\uploads`, '');
const tagCssReplace = process.env.TAG_CSS_REPLACE;
let jsonDir = process.env.PATH_JSON;
jsonDir = jsonDir.split('/').pop();
let htmlDir = process.env.DIR_HTML;
htmlDir = __dirname + '\\' + htmlDir.split('/').pop();
const htmlType1 = htmlDir + '\\' + process.env.HTML_TYPE1;
const htmlType2 = htmlDir + '\\' + process.env.HTML_TYPE2;
const htmlType3 = htmlDir + '\\' + process.env.HTML_TYPE3;
const pathJsonPdfList = './' + jsonDir + '/' + process.env.JSON_PDF_LIST_FILENAME;
const pathJsonPdfContent = __dirname + '\\' + jsonDir + '\\' + process.env.JSON_PDF_CONTENT;
const firstTermDir = 'first-term';
const secondTermDir = 'second-term';
const thirdTermDir = 'third-term';
let cumulativeFirstTermTotalList = {};
let cumulativeSecondTermTotalList = {};
let firstTermOnce = true;
let secondTermOnce = true;
let thirdTermOnce = true;
let isActive = false;
const getPath = (p, f) => {
let dir = pathFirstTermResults;
switch (p) {
case firstTermDir:
dir = pathFirstTermResults;
break;
case secondTermDir:
dir = pathSecondTermResults;
break;
case thirdTermDir:
dir = pathThirdTermResults;
break;
default:
break;
}
return dir + f
}
const resolution = {
x: 1920,
y: 1080
}
const args = [
'--disable-gpu',
`--window-size=${resolution.x},${resolution.y}`,
'--no-sandbox',
]
const createPdf = (page, content, templateType, filename, className, term, sessionName, isProcessActive, pdfFileList, cb) => {
let path, document, options;
path = getPath(term, filename);
if(path != null) {
let options = {
path: path,
format: 'A4',
printBackground: true,
margin: {
left: '0px',
top: '0px',
right: '0px',
bottom: '0px'
}
}
let templateData = '';
switch (templateType) {
case '1':
templateData = ReadFile(htmlType1);
break;
case '2':
templateData = ReadFile(htmlType2);
break;
case '3':
templateData = ReadFile(htmlType3);
break;
default:
templateData = ReadFile(htmlType1);
break;
}
(async() => {
const html = HtmlGenerator(content, templateData);
if(html != undefined && html !== '' && html != null) {
// create PDF file
cb(filename, className, term, sessionName, isProcessActive, pdfFileList);
// get style from .css & replace
const css = ReadFile(pathCss);
await page.setContent(html, { waitUntil: 'networkidle0'});
await page.addStyleTag(css);
await page.pdf(options);
page.close();
}
})()
}
}
const pdfGenerator = (json, cb) => {
let data = {};
let pdfFileList = [];
if(typeof json == 'string') {
data = JSON.parse(json)
} else {
data = json;
}
try {
// declare defaults
let filename = 'Student' + '.pdf';
let termName = firstTermDir;
const templateType = data.keys.templateType;
const session = data.classInfo.Session;
const sessionName = session.replace('/', '-');
const students = data.students;
const className = data.classInfo.Class_Name;
const recordFirstTerm = data.firstTerm;
const recordSecondTerm = data.secondTerm;
const recordThirdTerm = data.thirdTerm;
let pdfCreatedList = [];
let isReset = false;
let totalResultsExpected = Object.keys(recordFirstTerm).length + Object.keys(recordSecondTerm).length + Object.keys(recordThirdTerm).length;
let totalResultsCount = 0;
let jsonForPdf = {};
let record = {};
let sRecord, path, id, fName, lName;
// get each student
let logEndOnce = true;
let logBeforeOnce = true;
logBeforeOnce && console.log('============== *** ================');
logBeforeOnce && console.log('======== PDF GENERATION BEGINS ================');
const computeResult = (page, setTerm, setRecord, setReset) => {
const termName = setTerm;
const record = setRecord;
let isReset = setReset;
logBeforeOnce && console.log(`====== ${termName} RESULTS BEGINS ======`);
for(let elem of students){
id = elem.id;
fName = elem.firstName;
lName = elem.lastName;
filename = `${lName} ${fName} _${termName} ${sessionName}.pdf`;
// sRecord = record.filter(function (entry) { return entry[id] !== undefined; });
sRecord = record[id];
path = getPath(termName, filename);
// create pdf
if(!FileExists(path) && !FileExists(pathJsonPdfList)){
// generate final JSON for the student
// isReset = (pdfCreatedList.includes(id))? false: true;
jsonForPdf = finalJson(elem, sRecord, data, termName);
(pdfFileList.length < 1) && WriteFile(pathJsonPdfContent, JSON.stringify(jsonForPdf));
pdfFileList.push({
'term': termName,
'file': filename
});
totalResultsCount = pdfFileList.length;
const pdfDate = new Date();
console.log(`${filename} (${totalResultsCount}/${totalResultsExpected}) at ${pdfDate.getHours()}hr${pdfDate.getHours()>1?'s':''} - ${pdfDate.getMinutes()}min${pdfDate.getMinutes()>1?'s':''} - ${pdfDate.getSeconds()}sec${pdfDate.getSeconds()>1?'s':''}`);
isActive = (totalResultsExpected === totalResultsCount)? true: false;
logEndOnce = false;
// cb(filename, className, termName, sessionName, isActive, pdfFileList);
// WriteFile(path, null);
isReset = true;
createPdf(page, jsonForPdf, templateType, filename, className, termName, sessionName, isActive, pdfFileList, cb);
}
}
logBeforeOnce && console.log(`====== ${termName} RESULTS ENDS ======`);
}
// get each student result for First Term
const computeFirstTerm = (p) => {
return new Promise((resolve) => {
if(data.keys.firstTerm === '1') {
termName = firstTermDir;
record = recordFirstTerm;
pdfCreatedList = [];
isReset = false;
computeResult(p, termName, record, isReset)
}
resolve()
})
}
// get each student result for Second Term
const computeSecondTerm = (p) => {
return new Promise((resolve) => {
if(data.keys.secondTerm === '1') {
termName = secondTermDir;
record = recordSecondTerm;
pdfCreatedList = [];
isReset = false;
computeResult(p, termName, record, isReset)
}
resolve()
})
}
// get each student result for Third Term
const computeThirdTerm = (p) => {
return new Promise((resolve) => {
if(data.keys.thirdTerm === '1') {
termName = thirdTermDir;
record = recordThirdTerm;
pdfCreatedList = [];
isReset = false;
computeResult(p, termName, record, isReset)
}
resolve()
})
}
(async () => {
browser = await Puppeteer.launch({
headless: true,
handleSIGINT: false,
args: args,
});
const page = await browser.newPage();
await page.setViewport({
width: resolution.x,
height: resolution.y,
})
await computeFirstTerm(page);
await computeSecondTerm(page);
await computeThirdTerm(page);
browser.close()
})()
if(totalResultsExpected === totalResultsCount && totalResultsCount !== 0 && !logEndOnce) {
logEndOnce = true;
logBeforeOnce = false;
console.log('======== PDF GENERATION ENDS ================');
}
} catch (error) {
console.log('==== ERROR IN PDF GENERATION: ', error)
}
}
module.exports = {
PdfGenerator: pdfGenerator
}
ERROR
info Visit https://yarnpkg.com/en/docs/cli/run for documentation about this command.
lerna ERR! yarn run start stderr:
<--- Last few GCs --->
[9884:000002D68A73C6B0] 1665171 ms: Scavenge 44.1 (45.8) -> 43.2 (45.8) MB, 223.9 / 0.0 ms (average mu = 0.956, current mu = 0.952) allocation failure
[9884:000002D68A73C6B0] 1684089 ms: Scavenge 44.1 (45.8) -> 43.3 (45.8) MB, 587.3 / 0.0 ms (average mu = 0.956, current mu = 0.952) allocation failure
[9884:000002D68A73C6B0] 1749901 ms: Scavenge 44.2 (45.8) -> 43.3 (45.8) MB, 5099.0 / 0.0 ms (average mu = 0.956, current mu = 0.952) allocation failure
<--- JS stacktrace --->
FATAL ERROR: Committing semi space failed. Allocation failed - JavaScript heap out of memory
1: 00007FF6ED61013F
2: 00007FF6ED59F396
3: 00007FF6ED5A024D
4: 00007FF6EDED19EE
5: 00007FF6EDEBBECD
6: 00007FF6EDD5F61C
7: 00007FF6EDD6933F
8: 00007FF6EDD5BF19
9: 00007FF6EDD5A0D0
10: 00007FF6EDD7EA06
11: 00007FF6EDAB1CD5
12: 00007FF6EDF5F3E1
13: 00007FF6EDF602E9
14: 000002D68C4EF69E
error Command failed with exit code 134.
Screenshot of Task Manager, Chromium running multiple instances of over 50.
I appreciate any help. I hope this can be resolved to give me a smooth PDF generation.
Thank you.
Example solution (limiting parallel browsers)
I created you a PdfPrinter class which you can integrate into your setup. It allows you to limit the amount of parallel pdf generation jobs and allows setting a limit and manages opening/closing the browser for you. The PdfPrinter class is also highly coupled and needed some modification for using it as a general queue. Logicwise this can be modified to be a general queue.
You can try to integrate that into your code. This is a fully working test example with simplified pdfs (without the part of getting the actual data from the excel..)
As far as I understood your code, you do not need to pass the page around all your functions. First create your html + css and then use the pdfPrinter and let it handle page creation + browser launching..
(I like to code stuff like this so I went straight ahead..)
var puppeteer = require('puppeteer')
const defaultPrinterOptions = {
format: 'A4',
printBackground: true,
margin: {
left: '0px',
top: '0px',
right: '0px',
bottom: '0px'
}
}
class PdfPrinter {
maxBrowsers = 2
enqueuedPrintJobs = []
failedJobs = []
browserInstances = 0
// max browser instances in parallel
constructor(maxBrowsers) {
this.maxBrowsers = maxBrowsers
}
/**
*
* #param {*} html the html content to print
* #param {*} css to apply to the page
* #param {*} printOptions options passed to puppeteer
*/
// enqueues a print but the exact end moment cannot be known..
enqueuePrint = (html, css, path, done) => {
// merge custom options with defaultOptions..
const printOptions = {
...defaultPrinterOptions,
// add the path to the options.
path: path
}
// create a function which can be stored in an array
// it will later be grabbed by startPrinter() OR at the time any
// brwoser freed up..
// the function needs to be passed the actual used browser instance!
this.enqueuedPrintJobs.push(async(browser) => {
// catch the error which may be produced when printing something..
try {
// print the document
await this.print(browser, html, css, printOptions)
} catch (err) {
console.error('error when printing document..CLosing browser and starting a new job!!', printOptions.path)
console.error(err)
// store someting so you now what failed and coudl be retried or something..
this.failedJobs.push({ html, css, path: printOptions.path })
// puppeteer can run into erros too!!
// so close the browser and launch a new one!
await this.closeBrowser(browser)
browser = await this.launchBrowser()
}
// after the print, call done() so the promise is resovled in the right moment when
// this particular print has ended.!
done()
// start the next job right now if there are any left.
const job = this.enqueuedPrintJobs.shift()
if (!job) {
console.log('No print jobs available anymore. CLosing this browser instance.. Remaining browsers now:', this.maxBrowsers - this.browserInstances + 1)
await this.closeBrowser(browser)
return
}
// job is actually this function itself! It will be executed
// and automatically grab a new job after completion :)
// we pass the same browser instance to the next job!.
await job(browser)
})
// whenever a print job added make sure to start the printer
// this starts new browser instances if the limit is not exceeded resp. if no browser is instantiated yet,
// and does nothing if maximum browser count is reached..
this.tryStartPrinter()
}
// same as enqueuePrint except it wraps it in a promise so we can now the
// exact end moment and await it..
enqueuePrintPromise(html, css, path) {
return new Promise((resolve, reject) => {
try {
this.enqueuePrint(html, css, path, resolve)
} catch (err) {
console.error('unexpected error when setting up print job..', err)
reject(err)
}
})
}
// If browser instance limit is not reached will isntantiate a new one and run a print job with it.
// a print job will automatically grab a next job with the created browser if there are any left.
tryStartPrinter = async() => {
// Max browser count in use OR no jobs left.
if (this.browserInstances >= this.maxBrowsers || this.enqueuedPrintJobs.length === 0) {
return
}
// browser instances available!
// create a new one
console.log('launching new browser. Available after launch:', this.maxBrowsers - this.browserInstances - 1)
const browser = await this.launchBrowser()
// run job
const job = this.enqueuedPrintJobs.shift()
await job(browser)
}
closeBrowser = async(browser) => {
// decrement browsers in use!
// important to call before closing browser!!
this.browserInstances--
await browser.close()
}
launchBrowser = async() => {
// increment browsers in use!
// important to increase before actualy launching (async stuff..)
this.browserInstances++
// this code you have to adjust according your enviromnemt..
const browser = await puppeteer.launch({ headless: true })
return browser
}
// The actual print function which creates a pdf.
print = async(browser, html, css, printOptions) => {
console.log('Converting page to pdf. path:', printOptions.path)
// Run pdf creation in seperate page.
const page = await browser.newPage()
await page.setContent(html, { waitUntil: 'networkidle0' });
await page.addStyleTag({ content: css });
await page.pdf(printOptions);
await page.close();
}
}
// testing the PDFPrinter with some jobs.
// make sure to run the printer in an `async` function so u can
// use await...
const testPrinterQueue = async() => {
// config
const maxOpenedBrowsers = 5 // amount of browser instances which are allowed to be opened in parallel
const testJobCount = 100 // amount of test pdf jobs to be created
const destDir = 'C:\\somepath' // the directory to store the pdfs in..
// create sample jobs for testing...
const jobs = []
for (let i = 0; i < testJobCount; i++) {
jobs.push({
html: `<h1>job number [${i}]</h1>`,
css: 'h1 { background-color: red; }',
path: require('path').join(destDir, `pdf_${i}.pdf`)
})
}
// track time
const label = 'printed a total of ' + testJobCount + ' pdfs!'
console.time(label)
// run the actual pdf generation..
const printer = new PdfPrinter(maxOpenedBrowsers)
const jobProms = []
for (let job of jobs) {
// run jobs in parallel. Each job wil be runned async and return a Promise therefor
jobProms.push(
printer.enqueuePrintPromise(job.html, job.css, job.path)
)
}
console.log('All jobs enqueued!! Wating for finish now.')
// helper function which awaits all the print jobs, resp. an array of promises.
await Promise.all(jobProms)
console.timeEnd(label)
// failed jobs::
console.log('jobs failed:', printer.failedJobs)
// as file:
await require('fs').promises.writeFile('failed-jobs.json', JSON.stringify(printer.failedJobs))
}
testPrinterQueue().then(() => {
console.log('done with everyting..')
}).catch(err => {
console.error('unexpected error occured while printing all pages...', err)
})
You only need to adjust the destDir / openedBrowsers and testJobCount vars in the beginning of testPrinterQueue() for getting this to work.
What caused the problem in your code
Let's have a look at this piece
(async () => {
browser = await Puppeteer.launch({
headless: true,
handleSIGINT: false,
args: args,
});
const page = await browser.newPage();
await page.setViewport({
width: resolution.x,
height: resolution.y,
})
await computeFirstTerm(page);
await computeSecondTerm(page);
await computeThirdTerm(page);
browser.close()
})()
You created an anonymous function which is executed immediatly. Within the function all the statements are correctly awaited using await. But if you run this whole piece within a synchronious part of your application, the whole function will start immediatly but NOT been awaited before running next code.
Checkout this example:
//utility
function wait(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms)
})
}
const AsyncFunction = async() => {
console.log('Async named function started')
// simulate execution time of 2 seconds
await wait(2000)
console.log('Async named function ended')
};
function SyncFunction() {
console.log('sync function started')
// example of async function execution within a sync function..
AsyncFunction();
// what you have done in your code:
(async() => {
console.log('Async anonymus function started')
await wait(3000)
console.log('Async anonymus function ended')
})()
// what
console.log('sync function ended.')
}
SyncFunction()
console.log('done')
Note the output:
Async named function started
Async anonymus function started
sync function ended. // => sync function already ended
done // sync function ended and code continues execution.
Async named function ended
Async anonymus function ended
To correctly await your async stuff you need to put your whole application in async scope:
//utility
function wait(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms)
})
}
const AsyncFunction = async() => {
console.log('Async named function started')
// simulate execution time of 2 seconds
await wait(2000)
console.log('Async named function ended')
};
// this is now async!!
async function SyncFunction() {
console.log('sync function started')
// example of async function execution within a sync function..
await AsyncFunction();
// what you have done in your code:
await (async() => {
console.log('Async anonymus function started')
await wait(3000)
console.log('Async anonymus function ended')
})()
// what
console.log('sync function ended.')
}
SyncFunction().then(() => {
console.log('done')
}).catch(err => {
console.error('unexpected error occured..')
})
This output is what we want
sync function started
Async named function started
Async named function ended
Async anonymus function started
Async anonymus function ended
sync function ended.
done
Hope this helps you understand.
Feel free to leave a comment.

Upload audio blob after conversion from wav to mp3

function init() {
var cfg = {};
audio = new Recorder(cfg);
}
function toggle( btn ){ // audio && audio.record();
if(audio instanceof Recorder){
var btnLabel = btn.firstChild.nodeValue;
if( btnLabel === 'Record' ){
audio.record();
}else{
audio.stop();
createPreview( 'recordings' );
audio.clear();
}
btn.firstChild.nodeValue = (btnLabel === 'Record') ? 'Stop' : 'Record';
btn.setAttribute('class', (btn.getAttribute('class') === 'btn btn-primary') ? 'btn btn-danger' : 'btn btn-primary');
} else {
init();
toggle( btn );
}
}
function createPreview( containerId ) {
// audio && audio.exportWAV( function(blob) {
var targetContainer = document.getElementById( containerId );
var timestamp = new Date().getTime();
var filename = 'recording_'+ timestamp;
var div = document.createElement('div');
var linkMP3 = document.createElement('a');
linkMP3.setAttribute('id', 'MP3-'+ timestamp);
var iconMP3 = document.createElement('img');
iconMP3.setAttribute('src', 'images/i-mp3.jpeg');
var linkWAV = document.createElement('a');
linkWAV.setAttribute('id', 'WAV-'+ timestamp);
var iconWAV = document.createElement('img');
iconWAV.setAttribute('src', 'images/i-wav.jpeg');
var player = document.createElement('audio');
player.setAttribute('id', 'PLAYER-'+ timestamp);
player.controls = true;
div.appendChild(player);
div.appendChild(linkWAV);
div.appendChild(linkMP3);
targetContainer.appendChild(div);
audio.export( function( mediaObj ) {
if( mediaObj.blob.type == 'audio/mp3' ){
var url = mediaObj.url;
targetLink = document.getElementById( 'MP3-'+ timestamp );
targetLink.href = url;
targetLink.download = filename +'.mp3';
targetLink.innerHTML = targetLink.download;
saveAudio( url, filename );
} else { // 'audio/wav'
var url = URL.createObjectURL( mediaObj.blob );
targetPlayer = document.getElementById( 'PLAYER-'+ timestamp );
targetLink = document.getElementById( 'WAV-'+ timestamp );
targetPlayer.src = url;
targetLink.href = url;
targetLink.download = filename +'.wav';
targetLink.innerHTML = targetLink.download;
}
});
}
function saveAudio( url, filename ){
var firebaseUrl = 'your_firebase_url';
if(firebaseUrl !== 'your_firebase_url'){
console.info('>> saving audio: url');
console.log( url );
ref = new Firebase( firebaseUrl );
ref.set({
filetype: 'audio/mp3',
base64Str: url,
filename: filename +'.mp3'
});
}else{
console.warn('Audio not saved to firebase because firebaseUrl is undefined.');
}
}
I need to record audio in the browser (short clips, spoken voice, mono) and upload it in mp3 format. This by Chris Geirman has almost everything that I need, except that instead of using firebase, I'd like to use jquery to upload audio blobs to a folder on my server. I'm fairly new to all of this, but I'm guessing that I need to replace the saveAudio() function with my own uploadAudio() jquery(?) function (something like this), which will link to a script in /upload.php. So far so good (?), but I can't figure out from Chris's script exactly what it is that I should be uploading / passing to /upload.php. I'm planning to implement the script here.
OK just in case it helps anyone I managed to get it working using this from Soumen Basak.
function uploadAudio( blob ) {
var reader = new FileReader();
reader.onload = function(event){
var fd = {};
fd["fname"] = "test.wav";
fd["data"] = event.target.result;
$.ajax({
type: 'POST',
url: 'upload.php',
data: fd,
dataType: 'text'
}).done(function(data) {
console.log(data);
});
};
reader.readAsDataURL(blob);
}
Replace test.wav with whatever applies - in my case BlahBlah.mp3. Then to reference the blob from Chris Geirman's script, change uploadAudio( blob ); to uploadAudio( mediaObj.blob );.
Be aware that with this set up on localhost, 2 mins of audio took 1'40" to convert from wav to mp3 and move to the uploads directory. Next job, create progress bars, etc!
Upload.php (Thanks again Soumen Basak):
<?
// pull the raw binary data from the POST array
$data = substr($_POST['data'], strpos($_POST['data'], ",") + 1);
// decode it
$decodedData = base64_decode($data);
// print out the raw data,
$filename = $_POST['fname'];
echo $filename;
// write the data out to the file
$fp = fopen($filename, 'wb');
fwrite($fp, $decodedData);
fclose($fp);
?>

Amazon Transcribe streaming with Node.js using websocket

I am working on a whatsapp chatbot where I receive audio file(ogg format) file url from Whatsapp and I get buffer and upload that file on S3(sample.ogg) Now what is want to use AWS Transcribe Streaming so I am creating readStream of file and sending to AWS transcribe I am using websocket but I am receiving Empty response of Sometimes when I Mhm mm mm response. Please can anyone tell what wrong I am doing in my code
const express = require('express')
const app = express()
const fs = require('fs');
const crypto = require('crypto'); // tot sign our pre-signed URL
const v4 = require('./aws-signature-v4'); // to generate our pre-signed URL
const marshaller = require("#aws-sdk/eventstream-marshaller"); // for converting binary event stream messages to and from JSON
const util_utf8_node = require("#aws-sdk/util-utf8-node");
var WebSocket = require('ws') //for opening a web socket
// our converter between binary event streams messages and JSON
const eventStreamMarshaller = new marshaller.EventStreamMarshaller(util_utf8_node.toUtf8, util_utf8_node.fromUtf8);
// our global variables for managing state
let languageCode;
let region = 'ap-south-1';
let sampleRate;
let inputSampleRate;
let transcription = "";
let socket;
let micStream;
let socketError = false;
let transcribeException = false;
// let languageCode = 'en-us'
app.listen(8081, (error, data) => {
if(!error) {
console.log(`running at 8080----->>>>`)
}
})
let handleEventStreamMessage = function (messageJson) {
let results = messageJson.Transcript.Results;
if (results.length > 0) {
if (results[0].Alternatives.length > 0) {
let transcript = results[0].Alternatives[0].Transcript;
// fix encoding for accented characters
transcript = decodeURIComponent(escape(transcript));
console.log(`Transcpted is----->>${transcript}`)
}
}
}
function downsampleBuffer (buffer, inputSampleRate = 44100, outputSampleRate = 16000){
if (outputSampleRate === inputSampleRate) {
return buffer;
}
var sampleRateRatio = inputSampleRate / outputSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Float32Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++ ) {
accum += buffer[i];
count++;
}
result[offsetResult] = accum / count;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result;
}
function pcmEncode(input) {
var offset = 0;
var buffer = new ArrayBuffer(input.length * 2);
var view = new DataView(buffer);
for (var i = 0; i < input.length; i++, offset += 2) {
var s = Math.max(-1, Math.min(1, input[i]));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
}
return buffer;
}
function getAudioEventMessage(buffer) {
// wrap the audio data in a JSON envelope
return {
headers: {
':message-type': {
type: 'string',
value: 'event'
},
':event-type': {
type: 'string',
value: 'AudioEvent'
}
},
body: buffer
};
}
function convertAudioToBinaryMessage(raw) {
if (raw == null)
return;
// downsample and convert the raw audio bytes to PCM
let downsampledBuffer = downsampleBuffer(raw, inputSampleRate);
let pcmEncodedBuffer = pcmEncode(downsampledBuffer);
setTimeout(function() {}, 1);
// add the right JSON headers and structure to the message
let audioEventMessage = getAudioEventMessage(Buffer.from(pcmEncodedBuffer));
//convert the JSON object + headers into a binary event stream message
let binary = eventStreamMarshaller.marshall(audioEventMessage);
return binary;
}
function createPresignedUrl() {
let endpoint = "transcribestreaming." + "us-east-1" + ".amazonaws.com:8443";
// get a preauthenticated URL that we can use to establish our WebSocket
return v4.createPresignedURL(
'GET',
endpoint,
'/stream-transcription-websocket',
'transcribe',
crypto.createHash('sha256').update('', 'utf8').digest('hex'), {
'key': <AWS_KEY>,
'secret': <AWS_SECRET_KEY>,
'protocol': 'wss',
'expires': 15,
'region': 'us-east-1',
'query': "language-code=" + 'en-US' + "&media-encoding=pcm&sample-rate=" + 8000
}
);
}
function showError(message) {
console.log("Error: ",message)
}
app.get('/convert', (req, res) => {
var file = 'recorded.mp3'
const eventStreamMarshaller = new marshaller.EventStreamMarshaller(util_utf8_node.toUtf8, util_utf8_node.fromUtf8);
let url = createPresignedUrl();
let socket = new WebSocket(url);
socket.binaryType = "arraybuffer";
let output = '';
const readStream = fs.createReadStream(file, { highWaterMark: 32 * 256 })
readStream.setEncoding('binary')
//let sampleRate = 0;
let inputSampleRate = 44100
readStream.on('end', function() {
console.log('finished reading----->>>>');
// write to file here.
// Send an empty frame so that Transcribe initiates a closure of the WebSocket after submitting all transcripts
let emptyMessage = getAudioEventMessage(Buffer.from(new Buffer([])));
let emptyBuffer = eventStreamMarshaller.marshall(emptyMessage);
socket.send(emptyBuffer);
})
// when we get audio data from the mic, send it to the WebSocket if possible
socket.onopen = function() {
readStream.on('data', function(chunk) {
let binary = convertAudioToBinaryMessage(chunk);
if (socket.readyState === socket.OPEN) {
console.log(`sending to steaming API------->>>>`)
socket.send(binary);
}
});
// the audio stream is raw audio bytes. Transcribe expects PCM with additional metadata, encoded as binary
}
// the audio stream is raw audio bytes. Transcribe expects PCM with additional metadata, encoded as binary
socket.onerror = function () {
socketError = true;
showError('WebSocket connection error. Try again.');
};
// handle inbound messages from Amazon Transcribe
socket.onmessage = function (message) {
//convert the binary event stream message to JSON
let messageWrapper = eventStreamMarshaller.unmarshall(Buffer(message.data));
//console.log(`messag -->>${JSON.stringify(messageWrapper)}`)
let messageBody = JSON.parse(String.fromCharCode.apply(String, messageWrapper.body));
console.log("results:.. ",JSON.stringify(messageBody))
if (messageWrapper.headers[":message-type"].value === "event") {
handleEventStreamMessage(messageBody);
}
else {
transcribeException = true;
showError(messageBody.Message);
}
}
let closeSocket = function () {
if (socket.OPEN) {
// Send an empty frame so that Transcribe initiates a closure of the WebSocket after submitting all transcripts
let emptyMessage = getAudioEventMessage(Buffer.from(new Buffer([])));
let emptyBuffer = eventStreamMarshaller.marshall(emptyMessage);
socket.send(emptyBuffer);
}
}
})

Adding image dynamically in public folder in reactjs

I am developing an face detection application,for that I need to collect the users image for reference to detect them later.i have successfully uploaded the image in MySQL databse.now I need upload the image in public folder in react to detect the image in camera.i stuck in uploading image in react public folder.help me out get rid of this problem..
This is the React code where image to be detected in the imgUrl variable
detect = async () => {
const videoTag = document.getElementById("videoTag");
const canvas = document.getElementById("myCanvas");
const displaySize = { width: videoTag.width, height: videoTag.height };
faceapi.matchDimensions(canvas, displaySize);
//setInterval starts here for continuous detection
time = setInterval(async () => {
let fullFaceDescriptions = await faceapi
.detectAllFaces(videoTag)
.withFaceLandmarks()
.withFaceExpressions()
.withFaceDescriptors();
const value = fullFaceDescriptions.length;
this.setState({ detection: value });
fullFaceDescriptions = faceapi.resizeResults(
fullFaceDescriptions,
displaySize
);
canvas.getContext("2d").clearRect(0, 0, canvas.width, canvas.height);
//Label Images
var dummy = ["praveen", "vikranth", "Gokul", "Rahul"];
const labels = nameArray1;
// const labels = ["praveen", "vikranth", "Gokul", "Rahul"];
if (no_of_times <= 0) {
if (no_of_times === 0) {
labeledFaceDescriptors = await Promise.all(
labels.map(async (label) => {
// fetch image data from urls and convert blob to HTMLImage element
const imgUrl = `/img/${label}.png`; // for testing purpose
// const imgUrl = testImage;
const img = await faceapi.fetchImage(imgUrl);
const fullFaceDescription = await faceapi
.detectSingleFace(img)
.withFaceLandmarks()
.withFaceExpressions()
.withFaceDescriptor();
if (!fullFaceDescription) {
throw new Error(`no faces detected for ${label}`);
}
const faceDescriptors = [fullFaceDescription.descriptor];
return new faceapi.LabeledFaceDescriptors(label, faceDescriptors);
})
);
// console.log(no_of_times);
}
}
const maxDescriptorDistance = 0.7;
no_of_times++;
const faceMatcher = new faceapi.FaceMatcher(
labeledFaceDescriptors,
maxDescriptorDistance
);
const results = fullFaceDescriptions.map((fd) =>
faceMatcher.findBestMatch(fd.descriptor)
);
result = [];
results.forEach((bestMatch, i) => {
const box = fullFaceDescriptions[i].detection.box;
// console.log(box)
const text = bestMatch.toString(); //this for basMatch name detection
var str = "";
//This is for removing names confidence to map value without duplicate
var val = text.replace(/[0-9]/g, "");
for (let i of val) {
if (i !== " ") {
str += i;
} else {
break;
}
}
if (result.includes(str) === false) result.push(str);
const drawBox = new faceapi.draw.DrawBox(box, { label: text });
drawBox.draw(canvas);
faceapi.draw.drawFaceExpressions(canvas, fullFaceDescriptions, 0.85);
});
for (let i = 0; i < fullFaceDescriptions.length; i++) {
const result1 = fullFaceDescriptions[i].expressions.asSortedArray()[i];
// console.log(result[i]);
// console.log(result1.expression);
this.test(result[i], result1.expression);
}
}, 100);
In the above code i am manually putting image in public folder,this need to be done dynamically when the user uploads image.
this is place i get the images in base64 from nodejs
axios.get("/image").then((res) => {
testImage = res.data;
// console.log("from image" + res.data);
imgback = <img src={`data:image/jpeg;base64,${res.data}`} />;
});
This is nodejs code for the get request from reactjs
app.get("/image", (req, res) => {
connection.query("SELECT * FROM images", (error, row, fields) => {
if (!!error) {
console.log("Error in the query");
} else {
console.log("successful query");
var buffer = new Buffer(row[0].image, "binary");
var bufferBase64 = buffer.toString("base64");
res.send(bufferBase64);
}
});
});
my goal is, in the imgUrl variable in react code i need to specify the image folder for that i need to dynamically add image in folder.
Or is there is any other way to directly give image array in the imgUrl variable.please help me to sort out this problem.

Video stream from Blob NodeJS

I am recording MediaStream on client side in this way:
handleStream(stream) {
const ws = new WebSocket('ws://localhost:5432/binary');
var recorder = new MediaRecorder(stream);
recorder.ondataavailable = function(event) {
ws.send(event.data);
};
recorder.start();
}
This data are accepted on server side like this:
const wss = new WebSocket.Server({ port: 5432 });
wss.on('connection', function connection(ws) {
ws.on('message', function incoming(message) {
writeToDisk(message, 'video.webm');
});
});
function writeToDisk(dataURL, fileName) {
var fileBuffer = new Buffer(dataURL, 'base64');
fs.writeFileSync(fileName, fileBuffer);
}
It works like a charm, but I want to take the Buffer and make video live stream served by server side. Is there any way how to do it?
Thanks for your help.
I have already done this here.
You can use the MediaRecorder class to split the video into chunks and send them to the server for broadcast.
this._mediaRecorder = new MediaRecorder(this._stream, this._streamOptions);
this._mediaRecorder.ondataavailable = e => this._videoStreamer.pushChunk(e.data);
this._mediaRecorder.start();
...
this._mediaRecorder.requestData()
Do not forget to restart recording at intervals, so that new clients should not download all the video to connect to stream. Also, during the change of chunks, you should replace <video> by <image> or update video's poster so that the gluing goes smoothly.
async function imageBitmapToBlob(img) {
return new Promise(res => {
const canvas = document.createElement('canvas');
canvas.width = img.width;
canvas.height = img.height;
canvas.getContext('2d').drawImage(img,0,0);
canvas.toBlob(res);
});
}
...
const stream = document.querySelector('video').captureStream();
if(stream.active==true) {
const track = stream.getVideoTracks()[0];
const capturer = new ImageCapture(track);
const bitmap = await imageBitmapToBlob(await capturer.grabFrame());
URL.revokeObjectURL(this._oldPosterUrl);
this._video.poster = this._oldPosterUrl = URL.createObjectURL(bitmap);
track.stop();
}
You can glue Blob objects through their constructor. In the process of obtaining a new chunk, do not forget to clear the memory for the old video with URL.revokeObjectURL() and update video's current time
_updateVideo = async (newBlob = false) => {
const stream = this._video.captureStream();
if(stream.active==true) {
const track = stream.getVideoTracks()[0];
const capturer = new ImageCapture(track);
const bitmap = await imageBitmapToBlob(await capturer.grabFrame());
URL.revokeObjectURL(this._oldPosterUrl);
this._video.poster = this._oldPosterUrl = URL.createObjectURL(bitmap);
track.stop();
}
let data = null;
if(newBlob === true) {
const index = this._recordedChunks.length - 1;
data = [this._recordedChunks[index]];
} else {
data = this._recordedChunks;
}
const blob = new Blob(data, this._options);
const time = this._video.currentTime;
URL.revokeObjectURL(this._oldVideoUrl);
const url = this._oldVideoUrl = URL.createObjectURL(blob);
if(newBlob === true) {
this._recordedChunks = [blob];
}
this._size = blob.size;
this._video.src = url;
this._video.currentTime = time;
}
You should use two WebSocket for video broadcast and two for listening. One WebSocket transfers only video chunks, the second only new blobs with video headers (restart recording at intervals).
const blobWebSocket = new WebSocket(`ws://127.0.0.1:${blobPort}/`);
blobWebSocket.onmessage = (e) => {
console.log({blob:e.data});
this._videoWorker.pushBlob(e.data);
}
const chunkWebSocket = new WebSocket(`ws://127.0.0.1:${chunkPort}/`);
chunkWebSocket.onmessage = (e) => {
console.log({chunk:e.data});
this._videoWorker.pushChunk(e.data);
}
After connecting, the server sends the client all the current video blob and begins to dynamically send new chunks to the client.
const wss = new WebSocket.Server({ port });
let buffer = new Buffer.alloc(0);
function chunkHandler(buf,isBlob=false) {
console.log({buf,isBlob});
if(isBlob === true) {
//broadcast(wss,buf);
buffer = buf;
} else {
const totalLenght = buffer.length + buf.length;
buffer = Buffer.concat([buffer,buf],totalLenght);
broadcast(wss,buf);
}
}
wss.on('connection', function connection(ws) {
if(buffer.length !== 0) {
ws.send(buffer);
}
});

Resources