How to catch all promises with axios.all? - node.js

I am scraping a web page using axios and cheerio:
This web page has many links, while more load while scrolling down(like facebook).
I want to scrape each link while scrolling down until I reach the end.
This is a sample of my code:
cheerio = require('cheerio')
axios = require('axios')
function getLink(id) {
return axios(options).then(function(response) {
// Do stuff...
})
}
function scrollDown() {
axios(scrollOptions).then(function(response) {
$ = cheerio.load(response['data'])
isScrollFinished = ($('.page_more').length == 0)
promises = []
newLinks = $('.link') // Get the new links that were loaded while scrolling
newLinks.each(function() {
promises.push(getLink($(this).attr('id')))
})
axios.all(promises).then(responseArr => {
if(isScrollFinished) {
// Exit script
}
})
if(!isScrollFinished) {
scrollDown()
}
})
}
scrollDown()
The problem with this code is that sometimes it doesn't scrape all the links before I exit.
This is because the last axios.all only waits until all the links of the last scrolled page were scraped.
How do I fix this?

I created the promises array as a static variable and only called axios.all on it when the scrolling reached the end:
cheerio = require('cheerio')
axios = require('axios')
function getLink(id) {
return axios(options).then(function(response) {
// Do stuff...
})
}
function scrollDown() {
if (typeof scrollDown.promises === 'undefined') {
scrollDown.promises = [] // Define static variable if undefined
}
axios(scrollOptions).then(function(response) {
$ = cheerio.load(response['data'])
isScrollFinished = ($('.page_more').length == 0)
newLinks = $('.link') // Get the new links that were loaded while scrolling
newLinks.each(function() {
scrollDown.promises.push(getLink($(this).attr('id')))
})
if(isScrollFinished) {
axios.all(scrollDown.promises).then(responseArr => {
// Exit script
})
}
else {
scrollDown()
}
})
}
scrollDown()
Better solutions will gladly be accepted.

Related

React 17.0.1 basic onChange is not updating values into state [duplicate]

I am trying to learn hooks and the useState method has made me confused. I am assigning an initial value to a state in the form of an array. The set method in useState is not working for me, both with and without the spread syntax.
I have made an API on another PC that I am calling and fetching the data which I want to set into the state.
Here is my code:
<div id="root"></div>
<script type="text/babel" defer>
// import React, { useState, useEffect } from "react";
// import ReactDOM from "react-dom";
const { useState, useEffect } = React; // web-browser variant
const StateSelector = () => {
const initialValue = [
{
category: "",
photo: "",
description: "",
id: 0,
name: "",
rating: 0
}
];
const [movies, setMovies] = useState(initialValue);
useEffect(() => {
(async function() {
try {
// const response = await fetch("http://192.168.1.164:5000/movies/display");
// const json = await response.json();
// const result = json.data.result;
const result = [
{
category: "cat1",
description: "desc1",
id: "1546514491119",
name: "randomname2",
photo: null,
rating: "3"
},
{
category: "cat2",
description: "desc1",
id: "1546837819818",
name: "randomname1",
rating: "5"
}
];
console.log("result =", result);
setMovies(result);
console.log("movies =", movies);
} catch (e) {
console.error(e);
}
})();
}, []);
return <p>hello</p>;
};
const rootElement = document.getElementById("root");
ReactDOM.render(<StateSelector />, rootElement);
</script>
<script src="https://unpkg.com/#babel/standalone#7/babel.min.js"></script>
<script src="https://unpkg.com/react#17/umd/react.production.min.js"></script>
<script src="https://unpkg.com/react-dom#17/umd/react-dom.production.min.js"></script>
Neither setMovies(result) nor setMovies(...result) works.
I expect the result variable to be pushed into the movies array.
Much like .setState() in class components created by extending React.Component or React.PureComponent, the state update using the updater provided by useState hook is also asynchronous, and will not be reflected immediately.
Also, the main issue here is not just the asynchronous nature but the fact that state values are used by functions based on their current closures, and state updates will reflect in the next re-render by which the existing closures are not affected, but new ones are created. Now in the current state, the values within hooks are obtained by existing closures, and when a re-render happens, the closures are updated based on whether the function is recreated again or not.
Even if you add a setTimeout the function, though the timeout will run after some time by which the re-render would have happened, the setTimeout will still use the value from its previous closure and not the updated one.
setMovies(result);
console.log(movies) // movies here will not be updated
If you want to perform an action on state update, you need to use the useEffect hook, much like using componentDidUpdate in class components since the setter returned by useState doesn't have a callback pattern
useEffect(() => {
// action on update of movies
}, [movies]);
As far as the syntax to update state is concerned, setMovies(result) will replace the previous movies value in the state with those available from the async request.
However, if you want to merge the response with the previously existing values, you must use the callback syntax of state updation along with the correct use of spread syntax like
setMovies(prevMovies => ([...prevMovies, ...result]));
Additional details to the previous answer:
While React's setState is asynchronous (both classes and hooks), and it's tempting to use that fact to explain the observed behavior, it is not the reason why it happens.
TLDR: The reason is a closure scope around an immutable const value.
Solutions:
read the value in render function (not inside nested functions):
useEffect(() => { setMovies(result) }, [])
console.log(movies)
add the variable into dependencies (and use the react-hooks/exhaustive-deps eslint rule):
useEffect(() => { setMovies(result) }, [])
useEffect(() => { console.log(movies) }, [movies])
use a temporary variable:
useEffect(() => {
const newMovies = result
console.log(newMovies)
setMovies(newMovies)
}, [])
use a mutable reference (if we don't need a state and only want to remember the value - updating a ref doesn't trigger re-render):
const moviesRef = useRef(initialValue)
useEffect(() => {
moviesRef.current = result
console.log(moviesRef.current)
}, [])
Explanation why it happens:
If async was the only reason, it would be possible to await setState().
However, both props and state are assumed to be unchanging during 1 render.
Treat this.state as if it were immutable.
With hooks, this assumption is enhanced by using constant values with the const keyword:
const [state, setState] = useState('initial')
The value might be different between 2 renders, but remains a constant inside the render itself and inside any closures (functions that live longer even after render is finished, e.g. useEffect, event handlers, inside any Promise or setTimeout).
Consider following fake, but synchronous, React-like implementation:
// sync implementation:
let internalState
let renderAgain
const setState = (updateFn) => {
internalState = updateFn(internalState)
renderAgain()
}
const useState = (defaultState) => {
if (!internalState) {
internalState = defaultState
}
return [internalState, setState]
}
const render = (component, node) => {
const {html, handleClick} = component()
node.innerHTML = html
renderAgain = () => render(component, node)
return handleClick
}
// test:
const MyComponent = () => {
const [x, setX] = useState(1)
console.log('in render:', x) // ✅
const handleClick = () => {
setX(current => current + 1)
console.log('in handler/effect/Promise/setTimeout:', x) // ❌ NOT updated
}
return {
html: `<button>${x}</button>`,
handleClick
}
}
const triggerClick = render(MyComponent, document.getElementById('root'))
triggerClick()
triggerClick()
triggerClick()
<div id="root"></div>
I know that there are already very good answers. But I want to give another idea how to solve the same issue, and access the latest 'movie' state, using my module react-useStateRef.
As you understand by using React state you can render the page every time the state change. But by using React ref, you can always get the latest values.
So the module react-useStateRef let you use state's and ref's together. It's backward compatible with React.useState, so you can just replace the import statement
const { useEffect } = React
import { useState } from 'react-usestateref'
const [movies, setMovies] = useState(initialValue);
useEffect(() => {
(async function() {
try {
const result = [
{
id: "1546514491119",
},
];
console.log("result =", result);
setMovies(result);
console.log("movies =", movies.current); // will give you the latest results
} catch (e) {
console.error(e);
}
})();
}, []);
More information:
react-usestsateref
I just finished a rewrite with useReducer, following #kentcdobs article (ref below) which really gave me a solid result that suffers not one bit from these closure problems.
See: https://kentcdodds.com/blog/how-to-use-react-context-effectively
I condensed his readable boilerplate to my preferred level of DRYness -- reading his sandbox implementation will show you how it actually works.
import React from 'react'
// ref: https://kentcdodds.com/blog/how-to-use-react-context-effectively
const ApplicationDispatch = React.createContext()
const ApplicationContext = React.createContext()
function stateReducer(state, action) {
if (state.hasOwnProperty(action.type)) {
return { ...state, [action.type]: state[action.type] = action.newValue };
}
throw new Error(`Unhandled action type: ${action.type}`);
}
const initialState = {
keyCode: '',
testCode: '',
testMode: false,
phoneNumber: '',
resultCode: null,
mobileInfo: '',
configName: '',
appConfig: {},
};
function DispatchProvider({ children }) {
const [state, dispatch] = React.useReducer(stateReducer, initialState);
return (
<ApplicationDispatch.Provider value={dispatch}>
<ApplicationContext.Provider value={state}>
{children}
</ApplicationContext.Provider>
</ApplicationDispatch.Provider>
)
}
function useDispatchable(stateName) {
const context = React.useContext(ApplicationContext);
const dispatch = React.useContext(ApplicationDispatch);
return [context[stateName], newValue => dispatch({ type: stateName, newValue })];
}
function useKeyCode() { return useDispatchable('keyCode'); }
function useTestCode() { return useDispatchable('testCode'); }
function useTestMode() { return useDispatchable('testMode'); }
function usePhoneNumber() { return useDispatchable('phoneNumber'); }
function useResultCode() { return useDispatchable('resultCode'); }
function useMobileInfo() { return useDispatchable('mobileInfo'); }
function useConfigName() { return useDispatchable('configName'); }
function useAppConfig() { return useDispatchable('appConfig'); }
export {
DispatchProvider,
useKeyCode,
useTestCode,
useTestMode,
usePhoneNumber,
useResultCode,
useMobileInfo,
useConfigName,
useAppConfig,
}
With a usage similar to this:
import { useHistory } from "react-router-dom";
// https://react-bootstrap.github.io/components/alerts
import { Container, Row } from 'react-bootstrap';
import { useAppConfig, useKeyCode, usePhoneNumber } from '../../ApplicationDispatchProvider';
import { ControlSet } from '../../components/control-set';
import { keypadClass } from '../../utils/style-utils';
import { MaskedEntry } from '../../components/masked-entry';
import { Messaging } from '../../components/messaging';
import { SimpleKeypad, HandleKeyPress, ALT_ID } from '../../components/simple-keypad';
export const AltIdPage = () => {
const history = useHistory();
const [keyCode, setKeyCode] = useKeyCode();
const [phoneNumber, setPhoneNumber] = usePhoneNumber();
const [appConfig, setAppConfig] = useAppConfig();
const keyPressed = btn => {
const maxLen = appConfig.phoneNumberEntry.entryLen;
const newValue = HandleKeyPress(btn, phoneNumber).slice(0, maxLen);
setPhoneNumber(newValue);
}
const doSubmit = () => {
history.push('s');
}
const disableBtns = phoneNumber.length < appConfig.phoneNumberEntry.entryLen;
return (
<Container fluid className="text-center">
<Row>
<Messaging {...{ msgColors: appConfig.pageColors, msgLines: appConfig.entryMsgs.altIdMsgs }} />
</Row>
<Row>
<MaskedEntry {...{ ...appConfig.phoneNumberEntry, entryColors: appConfig.pageColors, entryLine: phoneNumber }} />
</Row>
<Row>
<SimpleKeypad {...{ keyboardName: ALT_ID, themeName: appConfig.keyTheme, keyPressed, styleClass: keypadClass }} />
</Row>
<Row>
<ControlSet {...{ btnColors: appConfig.buttonColors, disabled: disableBtns, btns: [{ text: 'Submit', click: doSubmit }] }} />
</Row>
</Container>
);
};
AltIdPage.propTypes = {};
Now everything persists smoothly everywhere across all my pages
React's useEffect has its own state/lifecycle. It's related to mutation of state, and it will not update the state until the effect is destroyed.
Just pass a single argument in parameters state or leave it a black array and it will work perfectly.
React.useEffect(() => {
console.log("effect");
(async () => {
try {
let result = await fetch("/query/countries");
const res = await result.json();
let result1 = await fetch("/query/projects");
const res1 = await result1.json();
let result11 = await fetch("/query/regions");
const res11 = await result11.json();
setData({
countries: res,
projects: res1,
regions: res11
});
} catch {}
})(data)
}, [setData])
# or use this
useEffect(() => {
(async () => {
try {
await Promise.all([
fetch("/query/countries").then((response) => response.json()),
fetch("/query/projects").then((response) => response.json()),
fetch("/query/regions").then((response) => response.json())
]).then(([country, project, region]) => {
// console.log(country, project, region);
setData({
countries: country,
projects: project,
regions: region
});
})
} catch {
console.log("data fetch error")
}
})()
}, [setData]);
Alternatively, you can try React.useRef() for instant change in the React hook.
const movies = React.useRef(null);
useEffect(() => {
movies.current='values';
console.log(movies.current)
}, [])
The closure is not the only reason.
Based on the source code of useState (simplified below). Seems to me the value is never assigned right away.
What happens is that an update action is queued when you invoke setValue. And after the schedule kicks in and only when you get to the next render, these update action then is applied to that state.
Which means even we don't have closure issue, react version of useState is not going to give you the new value right away. The new value doesn't even exist until next render.
function useState(initialState) {
let hook;
...
let baseState = hook.memoizedState;
if (hook.queue.pending) {
let firstUpdate = hook.queue.pending.next;
do {
const action = firstUpdate.action;
baseState = action(baseState); // setValue HERE
firstUpdate = firstUpdate.next;
} while (firstUpdate !== hook.queue.pending);
hook.queue.pending = null;
}
hook.memoizedState = baseState;
return [baseState, dispatchAction.bind(null, hook.queue)];
}
function dispatchAction(queue, action) {
const update = {
action,
next: null
};
if (queue.pending === null) {
update.next = update;
} else {
update.next = queue.pending.next;
queue.pending.next = update;
}
queue.pending = update;
isMount = false;
workInProgressHook = fiber.memoizedState;
schedule();
}
There's also an article explaining the above in the similar way, https://dev.to/adamklein/we-don-t-know-how-react-state-hook-works-1lp8
I too was stuck with the same problem. As other answers above have clarified the error here, which is that useState is asynchronous and you are trying to use the value just after setState. It is not updating on the console.log() part because of the asynchronous nature of setState, it lets your further code to execute, while the value updating happens on the background. Thus you are getting the previous value. When the setState is completed on the background it will update the value and you will have access to that value on the next render.
If anyone is interested to understand this in detail. Here is a really good Conference talk on the topic.
https://www.youtube.com/watch?v=8aGhZQkoFbQ
I found this to be good. Instead of defining state (approach 1) as, example,
const initialValue = 1;
const [state,setState] = useState(initialValue)
Try this approach (approach 2),
const [state = initialValue,setState] = useState()
This resolved the rerender issue without using useEffect since we are not concerned with its internal closure approach with this case.
P.S.: If you are concerned with using old state for any use case then useState with useEffect needs to be used since it will need to have that state, so approach 1 shall be used in this situation.
If we have to update state only, then a better way can be if we use the push method to do so.
Here is my code. I want to store URLs from Firebase in state.
const [imageUrl, setImageUrl] = useState([]);
const [reload, setReload] = useState(0);
useEffect(() => {
if (reload === 4) {
downloadUrl1();
}
}, [reload]);
const downloadUrl = async () => {
setImages([]);
try {
for (let i = 0; i < images.length; i++) {
let url = await storage().ref(urls[i].path).getDownloadURL();
imageUrl.push(url);
setImageUrl([...imageUrl]);
console.log(url, 'check', urls.length, 'length', imageUrl.length);
}
}
catch (e) {
console.log(e);
}
};
const handleSubmit = async () => {
setReload(4);
await downloadUrl();
console.log(imageUrl);
console.log('post submitted');
};
This code works to put URLs in state as an array. This might also work for you.
With custom hooks from my library, you can wait for the state values to update:
useAsyncWatcher(...values):watcherFn(peekPrevValue: boolean)=>Promise - is a promise wrapper around useEffect that can wait for updates and return a new value and possibly a previous one if the optional peekPrevValue argument is set to true.
(Live Demo)
import React, { useState, useEffect, useCallback } from "react";
import { useAsyncWatcher } from "use-async-effect2";
function TestComponent(props) {
const [counter, setCounter] = useState(0);
const [text, setText] = useState("");
const textWatcher = useAsyncWatcher(text);
useEffect(() => {
setText(`Counter: ${counter}`);
}, [counter]);
const inc = useCallback(() => {
(async () => {
await new Promise((resolve) => setTimeout(resolve, 1000));
setCounter((counter) => counter + 1);
const updatedText = await textWatcher();
console.log(updatedText);
})();
}, []);
return (
<div className="component">
<div className="caption">useAsyncEffect demo</div>
<div>{counter}</div>
<button onClick={inc}>Inc counter</button>
</div>
);
}
export default TestComponent;
useAsyncDeepState is a deep state implementation (similar to this.setState (patchObject)) whose setter can return a promise synchronized with the internal effect. If the setter is called with no arguments, it does not change the state values, but simply subscribes to state updates. In this case, you can get the state value from anywhere inside your component, since function closures are no longer a hindrance.
(Live Demo)
import React, { useCallback, useEffect } from "react";
import { useAsyncDeepState } from "use-async-effect2";
function TestComponent(props) {
const [state, setState] = useAsyncDeepState({
counter: 0,
computedCounter: 0
});
useEffect(() => {
setState(({ counter }) => ({
computedCounter: counter * 2
}));
}, [state.counter]);
const inc = useCallback(() => {
(async () => {
await new Promise((resolve) => setTimeout(resolve, 1000));
await setState(({ counter }) => ({ counter: counter + 1 }));
console.log("computedCounter=", state.computedCounter);
})();
});
return (
<div className="component">
<div className="caption">useAsyncDeepState demo</div>
<div>state.counter : {state.counter}</div>
<div>state.computedCounter : {state.computedCounter}</div>
<button onClick={() => inc()}>Inc counter</button>
</div>
);
}
var [state,setState]=useState(defaultValue)
useEffect(()=>{
var updatedState
setState(currentState=>{ // Do not change the state by get the updated state
updateState=currentState
return currentState
})
alert(updateState) // the current state.
})
Without any addtional NPM package
//...
const BackendPageListing = () => {
const [ myData, setMyData] = useState( {
id: 1,
content: "abc"
})
const myFunction = ( x ) => {
setPagenateInfo({
...myData,
content: x
})
console.log(myData) // not reflecting change immediately
let myDataNew = {...myData, content: x };
console.log(myDataNew) // Reflecting change immediately
}
return (
<>
<button onClick={()=>{ myFunction("New Content")} }>Update MyData</button>
</>
)
Not saying to do this, but it isn't hard to do what the OP asked without useEffect.
Use a promise to resolve the new state in the body of the setter function:
const getState = <T>(
setState: React.Dispatch<React.SetStateAction<T>>
): Promise<T> => {
return new Promise((resolve) => {
setState((currentState: T) => {
resolve(currentState);
return currentState;
});
});
};
And this is how you use it (example shows the comparison between count and outOfSyncCount/syncCount in the UI rendering):
const App: React.FC = () => {
const [count, setCount] = useState(0);
const [outOfSyncCount, setOutOfSyncCount] = useState(0);
const [syncCount, setSyncCount] = useState(0);
const handleOnClick = async () => {
setCount(count + 1);
// Doesn't work
setOutOfSyncCount(count);
// Works
const newCount = await getState(setCount);
setSyncCount(newCount);
};
return (
<>
<h2>Count = {count}</h2>
<h2>Synced count = {syncCount}</h2>
<h2>Out of sync count = {outOfSyncCount}</h2>
<button onClick={handleOnClick}>Increment</button>
</>
);
};
Use the Background Timer library. It solved my problem.
const timeoutId = BackgroundTimer.setTimeout(() => {
// This will be executed once after 1 seconds
// even when the application is the background
console.log('tac');
}, 1000);
// replace
return <p>hello</p>;
// with
return <p>{JSON.stringify(movies)}</p>;
Now you should see, that your code actually does work. What does not work is the console.log(movies). This is because movies points to the old state. If you move your console.log(movies) outside of useEffect, right above the return, you will see the updated movies object.

Extensions not returned in GraphQL query results

I'm creating an Apollo Client like this:
var { ApolloClient } = require("apollo-boost");
var { InMemoryCache } = require('apollo-cache-inmemory');
var { createHttpLink } = require('apollo-link-http');
var { setContext } = require('apollo-link-context');
exports.createClient = (shop, accessToken) => {
const httpLink = createHttpLink({
uri: `https://${shop}/admin/api/2019-07/graphql.json`,
});
const authLink = setContext((_, { headers }) => {
return {
headers: {
"X-Shopify-Access-Token": accessToken,
"User-Agent": `shopify-app-node 1.0.0 | Shopify App CLI`,
}
}
});
return new ApolloClient({
cache: new InMemoryCache(),
link: authLink.concat(httpLink),
});
};
to hit the Shopify GraphQL API and then running a query like that:
return client.query({
query: gql` {
productVariants(first: 250) {
edges {
node {
price
product {
id
}
}
cursor
}
pageInfo {
hasNextPage
}
}
}
`})
but the returned object only contain data and no extensions which is a problem to figure out the real cost of the query.
Any idea why?
Many thanks for your help
There's a bit of a hacky way to do it that we wrote up before:
You'll need to create a custom apollo link (Apollo’s equivalent of middleware) to intercept the response data as it’s returned from the server, but before it’s inserted into the cache and the components re-rendered.
Here's an example were we pull metrics data from the extensions in our API:
import { ApolloClient, InMemoryCache, HttpLink, ApolloLink } from 'apollo-boost'
const link = new HttpLink({
uri: 'https://serve.onegraph.com/dynamic?show_metrics=true&app_id=<app_id>',
})
const metricsWatchers = {}
let id = 0
export function addMetricsWatcher(f) {
const watcherId = (id++).toString(36)
metricsWatchers[watcherId] = f
return () => {
delete metricsWatchers[watcherId]
}
}
function runWatchers(requestMetrics) {
for (const watcherId of Object.keys(metricsWatchers)) {
try {
metricsWatchers[watcherId](requestMetrics)
} catch (e) {
console.error('error running metrics watcher', e)
}
}
}
// We intercept the response, extract our extensions, mutatively store them,
// then forward the response to the next link
const trackMetrics = new ApolloLink((operation, forward) => {
return forward(operation).map(response => {
runWatchers(
response
? response.extensions
? response.extensions.metrics
: null
: null
)
return response
})
})
function create(initialState) {
return new ApolloClient({
link: trackMetrics.concat(link),
cache: new InMemoryCache().restore(initialState || {}),
})
}
const apolloClient = create(initialState);
Then to use the result in our React components:
import { addMetricsWatcher } from '../integration/apolloClient'
const Page = () => {
const [requestMetrics, updateRequestMetrics] = useState(null)
useEffect(() => {
return addMetricsWatcher(requestMetrics =>
updateRequestMetrics(requestMetrics)
)
})
// Metrics from extensions are available now
return null;
}
Then use a bit of mutable state to track each request and its result, and the use that state to render the metrics inside the app.
Depending on how you're looking to use the extensions data, this may or may not work for you. The implementation is non-deterministic, and can have some slight race conditions between the data that’s rendered and the data that you've extracted from the extensions.
In our case, we store performance metrics data in the extensions - very useful, but ancillary - so we felt the tradeoff was acceptable.
There's also an open issue on the Apollo client repo tracking this feature request
I dont have any idea of ApolloClient but i tried to run your query in shopify graphql app. It return results with extensions. Please find screenshot below. Also You can put questions in ApolloClient github.

Puppeteer mock page request object

import { Page } from 'puppeteer/lib/Page';
export class MonitorRequestHelper {
public static monitorRequests(page: Page, on = false) {
if(on) {
page.on('request', req => {
if (['image', 'font', 'stylesheet'].includes(req.resourceType())) {
// Abort requests for images, fonts & stylesheets to increase page load speed.
req.abort();
} else {
req.continue();
}
});
} else {
return true;
}
}
}
I am trying to mock and spy the function to check if it got called at least once.
Also, it would be helpful if some explain me how to mock and spy event-emitter object.
The source code is available on https://github.com/Mukesh23singh/puppeteer-unit-testing
If you want to test that your logic in monitorRequests works, you need to pass in a fake Page object with an event emitter interface that produces a fake request that you can test on.
Something like:
import {spy} from 'sinon';
// Arrange
const fakePage = { on(type, cb) { this[type] = cb; } }; // "event emitter"
const fakeRequest = {
abort: sinon.spy(),
resourceType() { return 'image'; }
};
monitorRequests( fakePage, true );
// Act
// trigger fake request
fakePage['request'](fakeRequest);
// Assert
assert(fakeRequest.abort.called);

"Extension context invalidated" error when calling chrome.runtime.sendMessage()

I have a content script in a Chrome Extension that's passing messages. Every so often, when the content script calls
chrome.runtime.sendMessage({
message: 'hello',
});
it throws an error:
Uncaught Error: Extension context invalidated.
What does this error mean? I couldn't find any documentation on it.
It doesn't happen consistently. In fact, it's hard to reproduce. Seems to happen if I just leave the page open for a while in the background.
Another clue: I've written many Chrome Extensions with content scripts that pass messages and I haven't seen this error before. The main difference is that this content script is injected by the background page using
chrome.tabs.executeScript({
file: 'contentScript.js',
});
Does using executeScript instead of the manifest file somehow change the lifecycle of the content script?
This is certainly related to the message listener being lost in the middle of the connection between content and background scripts.
I've been using this approach in my extensions, so that I have a single module that I can use in both background and content scripts.
messenger.js
const context = (typeof browser.runtime.getBackgroundPage !== 'function') ? 'content' : 'background'
chrome.runtime.onConnect.addListener(function (port) {
port.onMessage.addListener(function (request) {
try {
const object = window.myGlobalModule[request.class]
object[request.action].apply(module, request.data)
} catch () {
console.error(error)
}
})
})
export function postMessage (request) {
if (context === 'content') {
const port = chrome.runtime.connect()
port.postMessage(request)
}
if (context === 'background') {
if (request.allTabs) {
chrome.tabs.query({}, (tabs) => {
for (let i = 0; i < tabs.length; ++i) {
const port = chrome.tabs.connect(tabs[i].id)
port.postMessage(request)
}
})
} else if (request.tabId) {
const port = chrome.tabs.connect(request.tabId)
port.postMessage(request)
} else if (request.tabDomain) {
const url = `*://*.${request.tabDomain}/*`
chrome.tabs.query({ url }, (tabs) => {
tabs.forEach((tab) => {
const port = chrome.tabs.connect(tab.id)
port.postMessage(request)
})
})
} else {
query({ active: true, currentWindow: true }, (tabs) => {
const port = chrome.tabs.connect(tabs[0].id)
port.postMessage(request)
})
}
}
}
export default { postMessage }
Now you'll just need to import this module in both content and background script. If you want to send a message, just do:
messenger.postMessage({
class: 'someClassInMyGlobalModuçe',
action: 'someMethodOfThatClass',
data: [] // any data type you want to send
})
You can specify if you want to send to allTabs: true, a specific domain tabDomain: 'google.com' or a single tab tabId: 12.

Duplicate post request sent by Service Worker

I just found this issue because I was trying to solve the duplicate post request issue when I am using workbox-background-sync. There is a function of my web app to upload the photos. But every time I did uploaded twice to the database. Here is the code I have:
const bgSyncQueue = new workbox.backgroundSync.Queue(
'photoSubmissions',
{
maxRetentionTime: 48 * 60,//48 hours
callbacks: {
queueDidReplay: function (requests) {
if (requests.length === 0) {
removeAllPhotoSubmissions();
}
else {
for(let request of requests) {
if (request.error === undefined && (request.response && request.response.status === 200)) {
removePhotoSubmission();
}
}
}
}
}
});
workbox.routing.registerRoute(
new RegExp('.*\/Home\/Submit'),
args => {
const promiseChain = fetch(args.event.request.clone())
.catch(err => {
bgSyncQueue.addRequest(args.event.request);
addPhotoSubmission();
changePhoto();
});
event.waitUntil(promiseChain);
},
'POST'
);
It may because the fetch(args.event.request.clone()). If I remove it, then there is no duplication anymore. I am using workbox 3.6.1 .
Finally I found the solution. Below is my code:
const photoQueue = new workbox.backgroundSync.Plugin('photoSubmissions', {
maxRetentionTime: 48 * 60, // Retry for max of 48 Hours
callbacks: {
queueDidReplay: function (requests) {
if (requests.length === 0) {
removeAllPhotoSubmissions();
}
else {
for(let request of requests) {
if (request.error === undefined && (request.response && request.response.status === 200)) {
removePhotoSubmission();
}
}
}
}
}
});
const myPhotoPlugin = {
fetchDidFail: async ({originalRequest, request, error, event}) => {
addPhotoSubmission();
changePhoto();
}
};
workbox.routing.registerRoute(
new RegExp('.*\/Home\/Submit'),
workbox.strategies.networkOnly({
plugins: [
photoQueue,
myPhotoPlugin
]
}),
'POST'
);
I removed fetch. If we still want to controll by ourselves, we need to use respondWith(). I have tested it, it is working. But I would like to use more workbox way to solve the problem. I am using workbox 3.6.3 and I created my own plugin to include a callback function fetchDidFail to update my views. Here are the references I found:
one and two. There are no duplicate posts anymore.

Resources