screenshotting urls with node - async - node.js

Question

Here is my code:
const fs = require('fs');
const screenshot = require('screenshot-stream');
const urlp = require('url');
var urls=[
'https://archive.org/details/8bitrecs',
'http://hackaday.com/',
'http://techcrunch.com/2012/02/16/auraslate-is-an-open-source-android-tablet-for-hackers/',
'http://www.english.illinois.edu/-people-/faculty/debaron/482/482readings/greenfield.html',
'http://sustain.rca.ac.uk/Sustain-Talks'];
urls.forEach(function(url){
const stream = screenshot(url, '1024x768', {crop: true});
stream.pipe(fs.createWriteStream(urlp.parse(url).hostname + 'test-1024x768.png'));
});
It only screenshots the last item in the url. Rhe others are images with zero bytes. I think I need to do the operation asynchronously so it doesn't overwrite the stream each time.
How would I do this?
UPDATE:
I want the screenshot to work, but catch errors and not block if a url is not accessible
UPDATE:
https://www.npmjs.com/package/screenshot-promise worked better although this code below still slows my computer down a lot!
const screenshotPromise = require('screenshot-promise');
...
urls.forEach(function(url) {
const promise = screenshotPromise(url, '1024x768', {crop: true}).then(buf => {
fs.writeFileSync(urlp.parse(url).hostname + 'test-1024x768.png', buf);
});
promise.then((value) => {
// value is whatever we passed in the resolve(...) function above.
// It doesn't have to be a string, but if it is only a succeed message, it probably will be.
console.log(value);
});

Answer 1

What you failed to add here is the error :
events.js:160
throw er; // Unhandled 'error' event
^
Error: Couldn't load url: http://techcrunch.com/2012/02/16/auraslate-is-an-open-source-android-tablet-for-hackers/
at LineStream.byline.on.data
(e:---\node_modules\screenshot-stream\index.js:77:16)
at emitOne (events.js:96:13)
at LineStream.emit (events.js:188:7)
The issue is that the module screenshot-stream is using PhantomJS, and phantomJS is unable to get to the page that outputs the error.
This error seems related to this issue : https://github.com/ariya/phantomjs/issues/10460.
Techcrunch.com and Aol.com seem to use web fonts (e.g. "BebasNeue-webfont.ttf") which Qt loads as application fonts. Something may be going wrong there.
My suggestion is using Google's Puppeteer that includes a built-in screenshot method : https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagescreenshotoptions

Answer 2

Code I did in the end that worked:
const puppeteer = require('puppeteer');
const urlp = require('url');
var URL = require('url-parse');
var urls = [
'https://archive.org/details/8bitrecs',
'http://hackaday.com/',
'http://techcrunch.com/2012/02/16/auraslate-is-an-open-source-android-tablet-for-hackers/',
'http://www.english.illinois.edu/-people-/faculty/debaron/482/482readings/greenfield.html',
'http://sustain.rca.ac.uk/Sustain-Talks',
'https://www.quintessentially.com/',
'https://www.producthunt.com/tech/ux-project-checklist',
'https://freedom.press/',
'http://issuu.com/search?q=vintage+motorcycle',
'http://www.pocketmod.com/v2/',
'https://www.metamind.io/',
'http://nautil.us/blog/chernobyls-hot-mess-the-elephants-foot-is-still-lethal',
'https://www.instructables.com/id/Tool-Storage-Hacks-or-How-to-Hang-Those-Black-Frid/',
'https://www.zippi.co.uk/framed-photo-print'
];
var getLocation = function(href) {
var l = document.createElement("a");
l.href = href;
return l;
};
(async() => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
page.waitForNavigation({
timeout: 40000
});
for (let i = 0; i < urls.length; i++) {
const url = urls[i];
var url1 = new URL(url);
try {
await page.goto(`${url}`);
await page.screenshot({
path: 'images/' + url1.hostname + '.png'
});
} catch (error) {
console.log(error.message);
// await page.close();
// await browser.close();
// process.exit(1);
continue;
}
}
})();