puppeteer page.authenticate https proxy not work - node.js

Running puppeteer with Proxy authentication failing.
Puppeteer version: 1.8
Platform / OS version:MacOS 10.13.6
Node.js version:v10.9.0
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
headless:false,
ignoreHTTPSErrors:true,
devtools:true,
timeout:3000,
args: ['--no-sandbox','--proxy-server=xxx:xxx']
});
const user='xxx';
const password='xxx';
const page = await browser.newPage();
// await page.setExtraHTTPHeaders({
// 'Proxy-Authorization': 'Basic ' + Buffer.from(`${user}:${password}`).toString('base64'),
// });
await page.authenticate({username:user, password:password});
await page.goto('https://www.apple.com/');
let title = await page.title();
console.log("title:" + title);
await browser.close();
})();
got error:
(node:5858) UnhandledPromiseRejectionWarning:
Error: net::ERR_TUNNEL_CONNECTION_FAILED at https://www.apple.com/
at navigate (/xxx/node_modules/_puppeteer#1.8.0#puppeteer/lib/Page.js:622:37)
at process._tickCallback (internal/process/next_tick.js:68:7)
(node:5858) UnhandledPromiseRejectionWarning:
Unhandled promise rejection. This error originated either by throwing
inside of an async function without a catch block,
or by rejecting a promise which was not handled with .catch().
(rejection id: 1)

const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({
headless:false,
ignoreHTTPSErrors:true,
devtools:true,
timeout:3000,
args: ['--no-sandbox','--proxy-server=https=xxx:xxx']
});
const user='xxx';
const password='xxx';
const page = await browser.newPage();
await page.authenticate({username:user, password:password});
await page.goto('https://www.apple.com/');
let title = await page.title();
console.log("title:" + title);
await browser.close();
})();
For me it worked by adding https between the proxy server argument.

Within your args to launch puppeteer,
You can also try this format:
--proxy-server=http=xxx:xxxx
OR
Have you tried setting proxy like this '--proxy-server=https=MyWorkingProxyIP:Port' according to docs https://www.chromium.org/developers/design-documents/network-settings ?

Related

Invalid chai property: browser when trying to run a test on puppeteer

i'm trying to run a test on a website using puppeteer and i want to use 'expect' assertion from chai but i keep getting this error and that my test failed.
const puppeteer = require('puppeteer')
const expect = require('chai').expect
describe('My First Puppeteer Test ', () => {
it( 'should launch the browser', async function() {
const browser = await puppeteer.launch({
headless: false,
slowMo: 10,
devtools: false,
defaultViewport: null,
args: ['--start-maximized']
})
const page = await browser.newPage()
await page.goto('http://example.com')
const title = await page.title()
const url = page.url()
const text = await page.$eval('h1', Element => Element.textContent)
const count = await page.$$eval('p', Element => Element.length)
expect(title).to.be.a('string', 'Example Domain')
expect(url).to.include('example.com')
expect(text).to.browser.a('string', 'example domain')
expect(count).to.equal(2)
await browser.close()
})
})
what's wrong with it?
do i need to add something in order for this to work?
this is the message i get from terminal
My First Puppeteer Test
1) should launch the browser
0 passing (1s)
1 failing
1) My First Puppeteer Test
should launch the browser:
Error: Invalid Chai property: browser
at Object.proxyGetter [as get] (node_modules\chai\lib\chai\utils\proxify.js:78:17)
at Context.<anonymous> (tests\example.test.js:25:24)
at runNextTicks (node:internal/process/task_queues:61:5)
at listOnTimeout (node:internal/timers:528:9)
at processTimers (node:internal/timers:502:7)

Unable to load script on puppeteer AWS Lambda

Im trying to add an external javascript file to puppeteer on Lambda.
const chromium = require('chrome-aws-lambda');
const browser = await chromium.puppeteer.launch({
....
});
const webPage = await browser.newPage();
await webPage.goto(url, { waitUntil: 'networkidle2', timeout: 0 });
await webPage.addScriptTag({
url: `https://code.jquery.com/jquery-3.6.0.min.js`,
});
But unable to load the script
ERROR Error: Loading script from https://code.jquery.com/jquery-3.6.0.min.js failed
at DOMWorld.addScriptTag (/var/task/node_modules/puppeteer-core/lib/cjs/puppeteer/common/DOMWorld.js:171:23)
at processTicksAndRejections (internal/process/task_queues.js:95:5)
at async I (/var/task/src/functions/preparePages/handler.js:20:3689)
at async Runtime.E [as handler] (/var/task/src/functions/preparePages/handler.js:20:5400)

Node.js puppeteer errors DeprecationWarning and UnhandledPromiseRejectionWarning

I'm new to using node.js and puppeteer and i keep getting three errors and i can't seem to work out why i'm getting the errors or how i can fix them.
I'm trying to scrape the same website over and over again but sometimes end up getting the errors.
Error one:
UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'close' of undefined
Error two:
UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 1)
Error three:
DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
My code:
const puppeteer = require("puppeteer");
const referers = require('./models/referers.json');
const agents = require('./models/agents.json');
const log = console.log;
const waitForDelay = (time) => {
try {
return new Promise((resolve) => {
setTimeout(resolve, time);
});
} catch (error) {
log('delay error', error);
}
};
const runPuppeteer = async() => {
let browser, page;
try {
log('started puppeteer');
const randomAgent = agents[Math.floor(Math.random() * agents.length)];
const randomReferer = referers[Math.floor(Math.random() * referers.length)];
log('randomAgent', randomAgent, '\n randomReferer', randomReferer);
browser = await puppeteer.launch({
headless: true,
ignoreHTTPSErrors: true,
slowMo: 10,
args: [
"--no-sandbox",
"--disable-setuid-sandbox"
]
});
page = await browser.newPage();
await page.setUserAgent(randomAgent);
await page.setJavaScriptEnabled(true);
await page.setExtraHTTPHeaders({ referer: randomReferer, 'Accept-Language': 'en' });
await page.setViewport({ width: 1680, height: 1050 });
await page.goto('https://SiteUrl.com', { waitUntil: 'domcontentloaded', timeout: 60000 });
await page.waitForSelector('.mwButton', { visible: true, timeout: 60000 });
const buttonElement = await page.$('.mwButton');
if (await buttonElement.isIntersectingViewport()) {
await page.click(".mwButton");
log('button clicked');
log(await page.title());
await waitForDelay(10000);
await page.screenshot({ path: 'screenshot.png' });
} else {
log('button not clicked');
}
} catch(error) {
log(error.message);
} finally {
await browser.close();
log('closing browser');
await setTimeout(runPuppeteer, 10000);
}
};
Those are very common and generic errors node throws when you have an exception on an async function.
The problem in your code is the finally clause, it attempts to close the browser, but when your code fails to create a browser object, it remains undefined when it reaches finally.
You could do if(browser) await browser.close() to make sure it exists, if not there is no point in trying to close it...

UnhandledPromiseRejectionWarning: Error: Protocol error (Network.setCookies): Target closed - Can't set cookies

Currently I'm getting an issue where Puppeteer crashes upon using the setCookies method. I'm currently using Puppeteer v 1.4.0 (latest version as of writing this) as well as the Chromium version that is bundled with Puppeteer, and here is the code that is giving me trouble:
const puppeteer = require('puppeteer');
const moment = require('moment');
(async () => {
const browser = await puppeteer.launch(
{
headless: false
}
);
const page = await browser.newPage();
await page.goto('https://google.com');
const currentUrl = await page.url();
await browser.close();
const browser1 = await puppeteer.launch(
{
headless: false
}
);
const page1 = await browser1.newPage();
const cookie = await currentUrl.split("/");
await page1.setCookie({
'name': 'samplename',
'value': cookie[0],
'domain': 'sampledomain',
'path': cookie[0] + '/' + cookie[0] + '/' + cookie[0],
'expires': moment().add(21, 'days').valueOf(),
'httpOnly': false,
'secure': true,
'sameSite': "Lax"
});
await page1.goto(currentUrl);
})();
and here is the error message
(node:64704) UnhandledPromiseRejectionWarning: Error: Protocol error (Network.setCookies): Target closed.
at Promise (/Users/pc/Desktop/Shopify Bot/node_modules/puppeteer/lib/Connection.js:200:56)
at new Promise (<anonymous>)
at CDPSession.send (/Users/pc/Desktop/Shopify Bot/node_modules/puppeteer/lib/Connection.js:199:12)
at Page.setCookie (/Users/pc/Desktop/Shopify Bot/node_modules/puppeteer/lib/Page.js:320:26)
at <anonymous>
at process._tickCallback (internal/process/next_tick.js:182:7)
(node:64704) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:64704) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
I've looked into the issue on my own for a while, and multiple sources seem to say that not actually executing async is the problem, however, I believe I am running everything in async (however, as this is my first time doing anything async on NodeJS, I could be making critical errors in judgement). I've tried verifying my Chromium and uninstalling + reinstalling Puppeteer, but nothing seems to avail.
The reason you're gettning an error is in this line:
'path': cookie[0] + '/' + cookie[0] + '/' + cookie[0],
It resolves into https:/https:/https: which is not a valid value for this property.
Try setting path to '/' or just do not set this property and it's going to work just fine.
You can find more reading on how to use path here and here.

How to open link with using puppeter wsendpoint

I try to connect browser with wsendpoint like this :
(async () => {
const browser = await puppeteer.connect({browserWSEndpoint: 'ws://localhost:9222/devtools/page/eedbdd50-cd25-472d-a701-a0302d9ffe2f'});
const page = await browser.newPage();
await page.goto('https://www.google.com.tr');
})();
this code can open new tab but do not go to link. It give error:
(node:2022) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 1): Error: Protocol error (Performance.enable): 'Performance.enable' wasn't found undefined
(node:2022) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
Try this:
const puppeteer = require('puppeteer');
puppeteer.connect({
browserWSEndpoint: 'your endpoint...'
}).then(async browser => {
const page = await browser.newPage();
await page.goto('https://www.google.com.tr');
});

Resources