Jest - Storyshot - getCustomBrowser - how to manage? - jestjs

Having issue where i want to use a custom browser for storyshot with jest but I'm having a hard time finding any example or docs about managing the browser lifecycle - it's just mentioned offhand. My initStoryshots looks like this
initStoryshots({
suite: 'Image storyshots',
storyKindRegex: /^((?!.*?skipImageSnapshot).)*$/,
test: imageSnapshot({
storybookUrl,
getMatchOptions,
getCustomBrowser: async () => {
let browser = await puppeteer.launch({
args: [
'--no-sandbox ',
'--headless',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-lcd-text',
],
});
return browser
}
}),
});
So I'm not clear where I can add an afterAll or some other way to get the browser and .close() it?
Hoping to find some guidance here. Please let me know what details I can add.

Ok, solved it. Leaving record here for the next person:
Solution was to capture the testFn returned by imageSnapshot and override the afterAll on that.
let browser;
let afterAll = () => {
if (browser) {
browser.close();
}
};
let testFn = imageSnapshot({
storybookUrl,
getMatchOptions,
getCustomBrowser: async () => {
browser = await puppeteer.launch({
args: [
'--no-sandbox ',
'--headless',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-lcd-text',
],
});
return browser;
},
});
testFn.afterAll = afterAll;
initStoryshots({
suite: 'Image storyshots',
storyKindRegex: /^((?!.*?skipImageSnapshot).)*$/,
test: testFn,
});

Related

Node puppeteer scraping YouTube and encountering redirected you too many times

I'm trying to scrape a YouTube playlists URL using Node / puppeteer. It was working, but now I'm getting ERR_TOO_MANY_REDIRECTS error. I can still access the page using chrome from my desktop.
I've tried using the chromium browser and chrome browsers. I've also tried using the puppeteer-extra stealth plugin and the random-useragent.
This is how my code stand at the moment:
const browser = await puppeteer.launch({
stealth: true,
headless: false // true,
executablePath: "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
args: [
'--disable-notifications', '--disable-features=site-per-process'
],
defaultViewport: null
});
const page = await browser.newPage()
await page.setUserAgent(random_useragent.getRandom());
await page.goto(<playlist-url, {
waitUntil: 'networkidle2',
timeout: 0
})
await page.waitForSelector('button[aria-label="Agree to the use of cookies and other data for the purposes described"')
It at the page.goto it bombs. And it happens even if I try going to https://www.youtube.com.
Any suggestions what I should try next. I tried a proxy server but couldn't get it to work. I suspect I need a proxy to actually route through.
If all you need is playlist IDs for a given channel, it's possible to query a feed at:
https://youtube.com/feeds/videos.xml?channel_id=<Channel ID>
To get IDs of videos you can query a feed at:
https://youtube.com/feeds/videos.xml?playlist_id=PLAYLIST_ID
You can get playlists (and Mixes) links from YouTube like in the code example below (also check full code the online IDE):
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
puppeteer.use(StealthPlugin());
const searchString = "java course";
const requestParams = {
baseURL: `https://www.youtube.com`,
encodedQuery: encodeURI(searchString), // what we want to search for in URI encoding
};
async function fillPlaylistsDataFromPage(page) {
const dataFromPage = await page.evaluate((requestParams) => {
const mixes = Array.from(document.querySelectorAll("#contents > ytd-radio-renderer")).map((el) => ({
title: el.querySelector("a > h3 > #video-title")?.textContent.trim(),
link: `${requestParams.baseURL}${el.querySelector("a#thumbnail")?.getAttribute("href")}`,
videos: Array.from(el.querySelectorAll("ytd-child-video-renderer a")).map((el) => ({
title: el.querySelector("#video-title")?.textContent.trim(),
link: `${requestParams.baseURL}${el.getAttribute("href")}`,
length: el.querySelector("#length")?.textContent.trim(),
})),
thumbnail: el.querySelector("a#thumbnail #img")?.getAttribute("src"),
}));
const playlists = Array.from(document.querySelectorAll("#contents > ytd-playlist-renderer")).map((el) => ({
title: el.querySelector("a > h3 > #video-title")?.textContent.trim(),
link: `${requestParams.baseURL}${el.querySelector("a#thumbnail")?.getAttribute("href")}`,
channel: {
name: el.querySelector("#channel-name a")?.textContent.trim(),
link: `${requestParams.baseURL}${el.querySelector("#channel-name a")?.getAttribute("href")}`,
},
videoCount: el.querySelector("yt-formatted-string.ytd-thumbnail-overlay-side-panel-renderer")?.textContent.trim(),
videos: Array.from(el.querySelectorAll("ytd-child-video-renderer a")).map((el) => ({
title: el.querySelector("#video-title")?.textContent.trim(),
link: `${requestParams.baseURL}${el.getAttribute("href")}`,
length: el.querySelector("#length")?.textContent.trim(),
})),
thumbnail: el.querySelector("a#thumbnail #img")?.getAttribute("src"),
}));
return [...mixes, ...playlists];
}, requestParams);
return dataFromPage;
}
async function getYoutubeSearchResults() {
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
const URL = `${requestParams.baseURL}/results?search_query=${requestParams.encodedQuery}`;
await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForSelector("#contents > ytd-video-renderer");
const playlists = await fillPlaylistsDataFromPage(page);
await browser.close();
return playlists;
}
getYoutubeSearchResults().then(console.log);
📌Note: to get thumbnail you need to scroll playlist into view (using .scrollIntoView() method).
Output:
[
{
"title":"Java Complete Course | Placement Series",
"link":"https://www.youtube.com/watch?v=yRpLlJmRo2w&list=PLfqMhTWNBTe3LtFWcvwpqTkUSlB32kJop",
"channel":{
"name":"Apna College",
"link":"https://www.youtube.com/c/ApnaCollegeOfficial"
},
"videoCount":"35",
"videos":[
{
"title":"Introduction to Java Language | Lecture 1 | Complete Placement Course",
"link":"https://www.youtube.com/watch?v=yRpLlJmRo2w&list=PLfqMhTWNBTe3LtFWcvwpqTkUSlB32kJop",
"length":"18:46"
},
{
"title":"Variables in Java | Input Output | Complete Placement Course | Lecture 2",
"link":"https://www.youtube.com/watch?v=LusTv0RlnSU&list=PLfqMhTWNBTe3LtFWcvwpqTkUSlB32kJop",
"length":"42:36"
}
],
"thumbnail":null
},
{
"title":"Java Tutorials For Beginners In Hindi",
"link":"https://www.youtube.com/watch?v=ntLJmHOJ0ME&list=PLu0W_9lII9agS67Uits0UnJyrYiXhDS6q",
"channel":{
"name":"CodeWithHarry",
"link":"https://www.youtube.com/c/CodeWithHarry"
},
"videoCount":"113",
"videos":[
{
"title":"Introduction to Java + Installing Java JDK and IntelliJ IDEA for Java",
"link":"https://www.youtube.com/watch?v=ntLJmHOJ0ME&list=PLu0W_9lII9agS67Uits0UnJyrYiXhDS6q",
"length":"19:00"
},
{
"title":"Basic Structure of a Java Program: Understanding our First Java Hello World Program",
"link":"https://www.youtube.com/watch?v=zIdg7hkqNE0&list=PLu0W_9lII9agS67Uits0UnJyrYiXhDS6q",
"length":"14:09"
}
],
"thumbnail":null
}
]
You can read more about scraping YouTube playlists from blog post Web scraping YouTube secondary search results with Nodejs.

Can you run multiple tests in one browser context Playwright Javascript?

Is it possible to run multiple tests in one browser window for playwright/test?
currently it will hit browser.close(); after every test even though they are testing on the same page which puts a lot of extra time on the tests.
test.beforeAll(async ({ browser }) => {
const context = await browser.newContext();
const page = await context.newPage();
await page.goto('https://example.com');
});
test('nav test', async ({ page }) => {
const name = await page.innerText('.navbar__title');
expect(name).toBe('Playwright');
});
test('header test', async ({ page }) => {
const name = await page.innerText('.navbar__header');
expect(name).toBe('Playwright');
});
When you create a tests like this test('header test', async ({page}) => { you're specifying page and telling it to create a new page context.
Remove the page from the test - and share the one you create from your .beforeAll
Try this:
test.describe('1 page multiple tests', () => {
let page;
test.beforeAll(async ({ browser }) => {
const context = await browser.newContext();
page = await context.newPage();
await page.goto('https://example.com');
});
test.afterAll(async ({ browser }) => {
browser.close;
});
test('nav test', async () => {
const name = await page.innerText('h1');
expect(name).toContain('Example');
});
test('header test', async () => {
const name = await page.innerText('h1');
expect(name).toContain('Domain');
});
});
Run it like this :
npx playwright test .\StackTests_SinglePage.spec.ts --headed
(you can see the name of my file in there)
You might need to toggle it down to 1 worker if it tries to parallel run your test.
For me, that code opens 1 borwser, 1 page, passes both tests the closes out.
Can you try wrapping the tests in a describe block? So they are treated as a group and not as an individual tests.
test.describe('two tests for same page', () => {
test('nav test', async ({ page }) => {
const name = await page.innerText('.navbar__title');
expect(name).toBe('Playwright');
});
test('header test', async ({ page }) => {
const name = await page.innerText('.navbar__header');
expect(name).toBe('Playwright');
});
});

Using the default chrome profile with puppeteer that my chrome app uses

I'm having issues getting puppeteer to use the default profile that my chrome browser uses. I've tried setting path to the user profile, but when I go to a site with puppeteer that I know is saved with chrome app's userDataDir, there's nothing saved there. What am I doing wrong? I appreciate any help!
const browser = await puppeteer.launch({
headless: false,
userDataDir: 'C:\\Users\\Bob\\AppData\\Local\\Google\\Chrome\\User Data',
}).then(async browser => {
I've also tried userDataDir: 'C:/Users/Phil/AppData/Local/Google/Chrome/User Data',, but still nothing.
UPDATED:
const username = os.userInfo().username;
(async () => {
try {
const browser = await puppeteer.launch({
headless: false, args: [
`--user-data-dir=C:/Users/${username}/AppData/Local/Google/Chrome/User Data`]
}).then(async browser => {
I had same exact issue before. However connecting my script to a real chrome instance helped to solve a lot of problems specially the profile one.
You can see the steps here:
https://medium.com/#jaredpotter1/connecting-puppeteer-to-existing-chrome-window-8a10828149e0
//MACOS
/*
Open this instance first:
/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir=$(mktemp -d -t 'chrome-remote_data_dir')
// Windows:
- Add this to Target of launching chrome --remote-debugging-port=9222
- Navigate to http://127.0.0.1:9222/json/version
- copy webSocketDebuggerUrl
More Info: https://medium.com/#jaredpotter1/connecting-puppeteer-to-existing-chrome-window-8a10828149e0
*/
// Puppeteer Part
// Always update this socket after running the instance in terminal (look up ^)
and this is abstracted controller written in Typescript, that I always use in any project:
import * as puppeteer from 'puppeteer';
import { Browser } from 'puppeteer/lib/cjs/puppeteer/common/Browser';
import { Page } from 'puppeteer/lib/cjs/puppeteer/common/Page';
import { PuppeteerNode } from 'puppeteer/lib/cjs/puppeteer/node/Puppeteer';
import { getPuppeteerWSUrl } from './config/config';
export default class Puppeteer {
public browser: Browser;
public page: Page;
getBrowser = () => {
return this.browser;
};
getPage = () => {
return this.page;
};
init = async () => {
const webSocketUrl = await getPuppeteerWSUrl();
try {
this.browser = await ((puppeteer as unknown) as PuppeteerNode).connect({
browserWSEndpoint: webSocketUrl,
defaultViewport: {
width: 1920,
height: 1080,
},
});
console.log('BROWSER CONNECTED OK');
} catch (e) {
console.error('BROWSER CONNECTION FAILED', e);
}
this.page = await this.browser.newPage();
this.page.on('console', (log: any) => console.log(log._text));
};
}
Abstracted webosocket fecther:
import axios from "axios";
import { exit } from "process";
export const getPuppeteerWSUrl = async () => {
try {
const response = await axios.get("http://127.0.0.1:9222/json/version");
return response.data.webSocketDebuggerUrl;
} catch (error) {
console.error("Can not get puppeteer ws url. error %j", error);
console.info(
"Make sure you run this command (/Applications/Google Chrome.app/Contents/MacOS/Google Chrome --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir=$(mktemp -d -t 'chrome-remote_data_dir')) first on a different shell"
);
exit(1);
}
};
Feel free to adjust the template to suit whatever you enviroment/tools currrently look like.

While using capture-website, puppeteer with webpack throwing error:browser not downloaded, at ChromeLauncher.launch (webpack-internal)

I am trying to take screenshot by providing html file on node js.
I have used capture-website package.
Here is the code:
try{
await captureWebsite.file('file.html', 'file.png', {overwrite: true}, function (error) {
if (error) {
console.log('error',error);
}
});
}catch(e){
console.log('error in capture image:', e)
}
Version:
node:12.16.1
capture-website: "0.8.1"
Angular : 7
You can do something like:
async function screenshotFromHtml ({ html, timeout = 2000 }: ScreenshotOptions) {
const browser = await puppeteer.launch({
headless: !process.env.DEBUG_HEADFULL,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
]
})
const page = await browser.newPage()
// Set viewport to something big
// Prevents Carbon from cutting off lines
await page.setViewport({
width: 2560,
height: 1080,
deviceScaleFactor: 2
})
page.setContent(html)
const base64 = await page.screenshot({ encoding: "base64" }) as string;
// Wait some more as `waitUntil: 'load'` or `waitUntil: 'networkidle0'
await page.waitFor(timeout)
// Close browser
await browser.close()
return base64
}
This code is in typescript, but you can use the function body in your JS project
In this github file you can see a html render code too

Mock 'element' in jest tests

I have a helper which I use for Detox tests which hold abstractions of most commonly occurring actions. Like this.
/**
* Looks for a search input and inputs the query
*/
export const inputSearchQuery = async ({ query = '', placeholderText = '' }) => {
if (placeholderText) {
// look for search input
await expect(element(by.id(TestID.SEARCH_INPUT).withDescendant(by.text(placeholderText)))).toBeVisible();
// tap search input
await element(by.id(TestID.SEARCH_INPUT).withDescendant(by.text(placeholderText))).tap();
// type in query
await element(by.id(TestID.SEARCH_INPUT).withDescendant(by.text(placeholderText))).typeText(query);
} else {
// look for search input
await expect(element(by.id(TestID.SEARCH_INPUT))).toBeVisible();
// tap search input
await element(by.id(TestID.SEARCH_INPUT)).tap();
// type in query
await element(by.id(TestID.SEARCH_INPUT)).typeText(query);
}
};
I want this layer of abstraction to be tested with Jest. So I did the following,
describe('e2e helper', () => {
describe('inputSearchQuery()', () => {
test('should check whether the search input is visible', async () => {
const mockToBeVisible = jest.fn();
await inputSearchQuery({});
expect(mockToBeVisible).toBeCalledTimes(1);
});
});
});
And obviously got the error,
ReferenceError: element is not defined
I understand that element comes from the environment. How do I configure my Jest setup...
"jest": {
"preset": "react-native",
"snapshotSerializers": [
"./node_modules/enzyme-to-json/serializer"
],
"transformIgnorePatterns": [
"node_modules/(?!(jest-)?react-native|react-navigation)"
],
"moduleNameMapper": {
"package.json": "<rootDir>/__mocks__/package.json"
},
"testPathIgnorePatterns": [
"/e2e/"
],
// I'm guessing something like this but doesn't work out of the box
"testEnvironment": "detox"
},
...to address this?

Resources