Puppeteer is calling in a loop in nextjs - node.js

I have a nextjs page which consists of a react video player which plays a YouTube video based on some id passed in the url. The YouTube video is fetched in getServerSideProps based on the id. Then on the client side I am using /api/some-route to take a screenshot of that video player div using Puppeteer. Problem is when in api side I am opening a browser with Puppeteer with that particular URL, getServerSideProps is called and again my api/some-routes is getting called. So It has made a loop and is not finishing. How do I stop this?
My page:
export default function Home() {
useEffect(() => {
if (typeof window === undefined) {
return;
}
const url = window.location.href;
setTimeout(() => {
fetch(`/api/scrapper?url=${url}`)
.then((res) => {
res.json();
})
.then((data) => {
console.log(data);
});
}, 10000);
}, [params.slug[0]);
return (
<>
<Layout>
<Frame id="capture" />
</Layout>
</>
);
}
export const getServerSideProps = async ({ params }) => {
return {
props: { params, serverData },
};
}
/api/scrapper.js
import puppeteer from "puppeteer";
export default async function My(req, res) {
const url = req.query.url;
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);
const img = await page.screenshot({ path: "output.png" });
console.log("img", img);
await page.close();
await browser.close();
return res.json("done");
}

Related

Can't fetch my data from my backend(NodeJS) to my react native app

I have tried to get my json data but it's not working. It works to get the data on postman and on chrome but when I try to fetch the data it comes back as null. I'm using apisauce.
any ideas? I'm new to react native :)
const [listings, setListings] = useState([])
useEffect(() => {
loadListings()
}, []);
const loadListings = async () => {
const response = await listingsApi.getListings()
console.log(response.data, "response.data")
setListings(response.data)
}
import { create } from "apisauce";
const apiClient = create({
baseURL: "http://127.0.0.1:9000/api",
});
export default apiClient;
import client from "./client";
const endpoint = "/listings";
const getListings = () => client.get(endpoint);
export default {
getListings,
};
You can call loadListing like that (because loadListing is asynchronous function)
useEffect(() => {
loadListings().then(() => {}).catch(() => {})
},[])
or can be called using IIFE
useEffect(() => {
(async() => {
await loadListings();
})()
},[])

puppeteer can not take screenshot of react player in nextjs

I have a simple nextjs page which has only one div with simple h1 Tag and React video player component. I want to take a screenshot of my view of h1 tag and video player whatever is playing in that moment using puppeteer. I have implemented puppeteer but it does not take the screenshot of video player instead it returns only h1 tag and blank afterwards.
Actual image I want:
Puppeteer screenshot:
I am using nextjs client side api folder to call puppeteer.
Node js code:
import puppeteer from "puppeteer";
export default async function My(req, res) {
const url = req.query.url;
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);
const img = await page.screenshot({ path: "output.png" });
console.log("img", img);
await page.close();
await browser.close();
return res.json("done");
}
My component:
import styles from "../styles/Home.module.css";
import React, { useEffect, useRef, useState } from "react";
import dynamic from "next/dynamic";
const Player = dynamic(() => import("../components/player"), {
ssr: false,
});
export default function Home() {
useEffect(() => {
if (typeof window === undefined) {
return;
}
const url = "http://localhost:3000";
setTimeout(() => {
fetch(`http://localhost:3000/api/scrapper?url=${url}`)
.then((res) => {
res.json();
})
.then((data) => {
console.log(data);
});
}, 10000);
}, []);
return (
<>
<div
crossOrigin="true"
id="capture"
style={{ display: "block" }}
>
<h3>Hello</h3>
<Player />
</div>
<div style={{ marginTop: "100px" }} id="placement"></div>
</>
);
}
I solved it. Just added waitUntil Parameter in goto method. Actually the reason was puppeteer was taking screenshot before the player could initialize itself in the dom. That's why picture was blank. networkidle0 waits until the component is functional.
await page.goto(url, { waitUntil: "networkidle0" });

How to download files with NodeJS backend and React frontend? (I get the files corrupted)

I would like to be able to send pdf files with nodejs to the frontend. But when I do this, I get an error and I can't open the file. This is the error (translation of error: an error occurred when loading the PDF document):
I think that all is well but still without working.
Here is the nodeJS code:
routerTrainer.get("/download-training", verifyJWT, async (req, res) => {
const { training_id } = req.headers;
let training = await Training.findOne({
where: { id: training_id },
});
if (training) {
res.download(`${path}${dirname}${training.file_id}`);
}
});
And here is the React frontend code:
const downloadTraining = async (id) => {
const JWT = new ClassJWT();
const axiosReq = axios.create();
await JWT.checkJWT();
axiosReq
.get(`${serverPath}/download-training`, {
headers: {
training_id: id,
token: JWT.getToken(),
responseType: "blob"
},
})
.then((response) => {
const url = window.URL.createObjectURL(new Blob([response.data]));
const link = document.createElement("a");
link.href = url;
link.setAttribute("download", "file.pdf");
document.body.appendChild(link);
link.click();
})
.catch((err) => console.log(err));
};
Don`t worry about all that have JWT like verifyJWT or ClassJWT, this are implementations of json web tokens and it works correctly.
If anyone know how to fix it, please let me know.
you have to convert the binary file to a blob (in this example is set the responseType of xhr as blob), then convert it to base64 encoded file, here is an example:
<html>
<body>
<h1><a>dl</a></h1>
<script>
const pdfSrc = "https://blahblah.com/e-book.pdf";
const linkTag = document.querySelector("a");
const xhr = new XMLHttpRequest();
const fileReader = new FileReader();
xhr.open("GET", pdfSrc);
xhr.responseType = "blob";
xhr.addEventListener("loadend", () => {
fileReader.readAsDataURL(xhr.response);
});
fileReader.addEventListener("loadend", (event) => {
const base64File = event.srcElement.result;
linkTag.href = base64File;
linkTag.setAttribute("download", "file.pdf");
});
xhr.send();
</script>
</body>
</html>
In my case, in back-end (ExpressJs) I have something like -
app.get('/api/v1/getPdf', function (req, res) {
let resolve = require('path').resolve;
res.download(resolve('./folder/file.pdf'));
});
and in ReactJS (without TypeScrypt), I'm using native fetch instead of axios:
const onButtonClick = async () => {
let file = null;
await (async () => {
const rawResponse = await fetch('http://<host:port>/api/v1/getPdf', {
method: 'get',
headers: {
'Content-Type': 'application/json',
},
});
file = await rawResponse.blob();
})();
const pdfWindow = window.open();
pdfWindow.location.href = window.URL.createObjectURL(file);
};
where in component
return (
<>
<center>
<h1>Welcome</h1>
<h3>Click on below button to download PDF file</h3>
<button onClick={onButtonClick}>Download PDF</button>
</center>
</>
);
and it opens in the new window pdf file like (see link)
enter image description here
if you want to download file, you must implemente onButtonClick a little bit different
const onButtonClick = async () => {
let file = null;
await (async () => {
const rawResponse = await fetch('http://<host:port>/api/v1/getPdf', {
method: 'get',
headers: {
'Content-Type': 'application/json',
},
});
file = await rawResponse.blob();
})();
// const pdfWindow = window.open();
// pdfWindow.location.href = window.URL.createObjectURL(file);
const a = document.createElement('a');
a.href = window.URL.createObjectURL(file);
a.download = 'file.pdf';
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
};
On my side it works like a charm...

How to send data from node/express to preview pdf in front without saving file?

My aim is to generate pdf contact with puppeteer with an html page that I built.
I succesfully generate this pdf in my back. But I have a problem to send data to my front. I tried many things but... One I got an arrayBuffer, once a blob, now a readableStream and I can read with my front none of theses...
Is there a way to easily send pdf and preview it in browser (in modal) ?
Here is my back :
const date = Date.now();
const pathPDF = `contract-${date}.pdf`;
(async () => {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(
`http://localhost:3000/admin/reservation/${req.params.reservation}/contract`,
{
waitUntil: 'networkidle2',
}
);
const pdf = await page.pdf({
path: pathPDF,
format: 'A4',
printBackground: true,
});
await browser.close();
// res.json(pdf) <-- I tried this first, don't work
// res.contentType('application/pdf');
// res.sendFile(pathPDF); <-- Then this, not working...
// const data = fs.readFileSync(`./${pathPDF}`);
// res.contentType('application/pdf');
// res.send(data); <-- I tryed this too, same...
Here action :
export const createContract = (reservation) => {
return fetch(`${API}/reservation/contract/${reservation}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
})
.then((response) => {
// return response.blob(); <-- Tried this
return response.json();
})
.catch((err) => console.log(err));
};
Here my call in page :
const generateContract = () => {
setLoading(true);
createContract(reservation._id).then((result) => {
if (result.error) {
setLoading(false);
snackbarShowMessage(`${result.error}`);
} else {
setPdf(URL.createObjectURL(result)); <-- Tried this
setPdf(result) <-- This too
setLoading(false);
snackbarShowMessage(`${result.message}`, 'success');
setOpen(true);
}
});
};
Do you have any idea where I doing wrong..?

How to trigger "Followers" modal on Instagram?

I can't get my puppet to click on the "Followers" link in Instagram.
I assume that Instagram has done some anti-bot magic, but maybe I'm just too conspiratory about it.
How could I get the "Followers" modal to show?
This is my code:
const puppeteer = require('puppeteer');
var instagram = {
username: 'username',
password: 'password'
}
var run = ()=>{
(async () => {
const browser = await puppeteer.launch({headless: false, args: ['--lang=en-EN,en']});
const page = await browser.newPage();
await page.setExtraHTTPHeaders({ 'Accept-Language': 'en' });
await page.goto('https://www.instagram.com/accounts/login');
await page.waitFor(1500);
await page.click("input[type=\"text\"]")
await page.keyboard.type(instagram.username);
await page.click("input[type=\"password\"]")
await page.keyboard.type(instagram.password);
await page.click("button[type=\"submit\"]");
await page.waitForNavigation();
await page.click(".HoLwm");
await page.click(".glyphsSpriteUser__outline__24__grey_9")
await page.waitForNavigation();
await page.waitFor(2500);
await page.waitForSelector('a.-nal3');
await page.evaluate(() => document.querySelector('a.-nal3')) //does not work
//await page.click(".-nal3") //does not work
await page.waitFor(3000);
await page.waitFor(1000);
//await browser.close();
})();
}
run()
While reviewing your script I noticed that not all of your selectors are the same in my Instagram so I fixed it trying not to use exact selectors since they may change tomorrow.
But this works today (see comments on what changed in the script):
var run = ()=>{
(async () => {
const browser = await puppeteer.launch({headless: false, args: ['--lang=en-EN,en']});
const rand = function(){ return Math.floor(1000 + Math.random() * 2000) }
const page = await browser.newPage();
await page.setExtraHTTPHeaders({ 'Accept-Language': 'en' });
await page.goto('https://www.instagram.com/accounts/login');
// When you can try not to `waitFor` set periods of time
// Wait for selectors, wait for random periods of time
await page.waitForSelector('button[type=\"submit\"]');
await page.click("input[type=\"text\"]")
await page.keyboard.type(instagram.username);
await page.waitFor(rand())
await page.click("input[type=\"password\"]")
await page.keyboard.type(instagram.password);
await page.waitFor(rand())
await page.click("button[type=\"submit\"]");
await page.waitForNavigation();
await page.waitFor(rand())
// After login we're back on the main page
// Wait till React starts and paints the interface
// We're waiting for "Profile" icon to be visible
await page.waitForSelector("span[aria-label='Profile']");
// Then we click on the link inside of which the icon is located
// That is link to the profile
await page.evaluate(() => document.querySelector("span[aria-label='Profile']").parentNode.click() );
await page.waitForNavigation();
await page.waitFor(rand())
// Do not rely on a selector
// Find a link that contains "followers" in its href
await page.waitForSelector("a[href*='followers']");
const followers = await page.evaluate(() => document.querySelector("a[href*='followers']").textContent)
console.log("Total followers: " + followers);
// Click on the followers link
await page.evaluate( () => document.querySelector("a[href*='followers']").click() )
// Wait for the followers modal and profiles
await page.waitFor("div[role='presentation'] div[role='dialog'] div:nth-child(2) ul li");
// Get followers that are in the list in the second div of that modal
const people = await page.evaluate(() => {
return [...document.querySelectorAll("div[role='presentation'] div[role='dialog'] div:nth-child(2) ul li")]
.map(user => {
const profLink = user.querySelector("a[title]")
return {
"name" : profLink.textContent,
"url" : profLink.href
};
})
})
console.log(people)
// await browser.close();
})();
}

Resources