Connection Refused from NextJS API (Incremental Static Regeneration calls) - node.js

I set up my NodeJS server to make a post request every time a user adds a product to perform incremental static regeneration. I have my NextJS running on port 3000 and NodeJS running on 3001
This is my /api/revalidate file in my NextJS application:
async function handleRevalidate(req, res) {
console.log("/api/revalidate received:", req.body);
const event = req.body;
if (event.model === "product") {
const id = event.entry.id;
await Promise.all([res.revalidate("/"), res.revalidate(`/products/${id}`)]);
console.log("revalidated product", id);
}
res.status(204).end();
}
export default handleRevalidate;
This is my NodeJS controller that's making the request to the NextJS's /api/revalidate:
const addProduct = async (req, res) => {
try {
const query = mysql2.format("INSERT INTO products VALUES ?", [
[Object.values(req.body)],
]);
const [resData, _] = await db.execute(query);
console.log(resData);
await axios.post("http://localhost:3000/api/revalidate", {
model: "product",
entry: {
id: req.body.id,
title: req.body.title,
description: req.body.description,
price: req.body.price,
},
});
res.status(201).send("Product Added");
console.log(query);
} catch (error) {
console.log(error);
}
};
module.exports = addProduct
The NodeJS and Next App interacts fine in Next Dev mode, but when I build the Next App and start it in production mode, this connect error occurs.
AxiosError: connect ECONNREFUSED ::1:3000
at AxiosError.from (/Users/aiden/Documents/theavocado_backend/node_modules/axios/dist/node/axios.cjs:789:14)
at RedirectableRequest.handleRequestError (/Users/aiden/Documents/theavocado_backend/node_modules/axios/dist/node/axios.cjs:2744:25)
at RedirectableRequest.emit (node:events:513:28)
at eventHandlers.<computed> (/Users/aiden/Documents/theavocado_backend/node_modules/follow-redirects/index.js:14:24)
at ClientRequest.emit (node:events:513:28)
at Socket.socketErrorListener (node:_http_client:490:9)
at Socket.emit (node:events:513:28)
at emitErrorNT (node:internal/streams/destroy:151:8)
at emitErrorCloseNT (node:internal/streams/destroy:116:3)
at process.processTicksAndRejections (node:internal/process/task_queues:82:21) {
port: 3000,
address: '::1',
syscall: 'connect',
code: 'ECONNREFUSED',
errno: -61,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [ 'xhr', 'http' ],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function], Blob: [class Blob] },
validateStatus: [Function: validateStatus],
headers: AxiosHeaders {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/1.2.2',
'Content-Length': '118',
'Accept-Encoding': 'gzip, compress, deflate, br'
},
method: 'post',
url: 'http://localhost:3000/api/revalidate',
data: '{"model":"product","entry":{"id":87663301,"title":"KL Light","description":"Very energy efficient light.","price":12}}'
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: Infinity,
protocol: 'http:',
path: '/api/revalidate',
method: 'POST',
headers: [Object: null prototype],
agents: [Object],
auth: undefined,
beforeRedirect: [Function: dispatchBeforeRedirect],
beforeRedirects: [Object],
hostname: 'localhost',
port: '3000',
agent: undefined,
nativeProtocols: [Object],
pathname: '/api/revalidate'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 118,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: false,
chunkedEncoding: false,
shouldKeepAlive: true,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
strictContentLength: false,
_contentLength: '118',
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [Socket],
_header: 'POST /api/revalidate HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/1.2.2\r\n' +
'Content-Length: 118\r\n' +
'Accept-Encoding: gzip, compress, deflate, br\r\n' +
'Host: localhost:3000\r\n' +
'Connection: keep-alive\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/api/revalidate',
_ended: false,
res: null,
aborted: false,
timeoutCb: [Function: emitRequestTimeout],
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kBytesWritten)]: 0,
[Symbol(kEndCalled)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype],
[Symbol(errored)]: null,
[Symbol(kUniqueHeaders)]: null
},
_currentUrl: 'http://localhost:3000/api/revalidate',
[Symbol(kCapture)]: false
},
cause: Error: connect ECONNREFUSED ::1:3000
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1471:16) {
errno: -61,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '::1',
port: 3000
}
}

There was no problem with the NextJS part, but with my NodeJS backend. It is not okay to use http://localhost:3000, but instead have to use http://127.0.0.1:3000. So by changing my NodeJS file to:
const addProduct = async (req, res) => {
try {
const query = mysql2.format("INSERT INTO products VALUES ?", [
[Object.values(req.body)],
]);
const [resData, _] = await db.execute(query);
console.log(resData);
await axios.post("http://127.0.0.1:3000/api/revalidate", {
model: "product",
entry: {
id: req.body.id,
title: req.body.title,
description: req.body.description,
price: req.body.price,
},
});
res.status(201).send("Product Added");
console.log(query);
} catch (error) {
console.log(error);
}
};
module.exports = addProduct
everything will work as expected.

Related

Connect ECONNREFUSED 127.0.0.1:8000 with Docker devcontainers and WSL2

I'm working with Docker and WSL2 containers in my enterprise. I have an application which is separated in two: the front (ReactJS) and a mock, temporary before having the real API.
And when I try to connect them together I'm getting this error:
Backend is running on port 8000
AxiosError: connect ECONNREFUSED 127.0.0.1:3030
at Function.AxiosError.from (/workspaces/backoffice/app/node_modules/axios/dist/node/axios.cjs:785:14)
at RedirectableRequest.handleRequestError (/workspaces/backoffice/app/node_modules/axios/dist/node/axios.cjs:2725:25)
at RedirectableRequest.emit (node:events:513:28)
at ClientRequest.eventHandlers.<computed> (/workspaces/backoffice/app/node_modules/follow-redirects/index.js:14:24)
at ClientRequest.emit (node:events:513:28)
at Socket.socketErrorListener (node:_http_client:494:9)
at Socket.emit (node:events:513:28)
at emitErrorNT (node:internal/streams/destroy:157:8)
at emitErrorCloseNT (node:internal/streams/destroy:122:3)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
port: 3030,
address: '127.0.0.1',
syscall: 'connect',
code: 'ECONNREFUSED',
errno: -111,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [ 'xhr', 'http' ],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function], Blob: null },
validateStatus: [Function: validateStatus],
headers: AxiosHeaders {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/1.2.0',
'Accept-Encoding': 'gzip, deflate, br'
},
method: 'get',
url: 'http://localhost:3030/catalogs/items',
data: undefined
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: Infinity,
protocol: 'http:',
path: '/catalogs/items',
method: 'GET',
headers: [Object: null prototype],
agents: [Object],
auth: undefined,
beforeRedirect: [Function: dispatchBeforeRedirect],
beforeRedirects: [Object],
hostname: 'localhost',
port: '3030',
agent: undefined,
nativeProtocols: [Object],
pathname: '/catalogs/items'
},
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
strictContentLength: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: [Socket],
_header: 'GET /catalogs/items HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/1.2.0\r\n' +
'Accept-Encoding: gzip, deflate, br\r\n' +
'Host: localhost:3030\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/catalogs/items',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kBytesWritten)]: 0,
[Symbol(kEndCalled)]: true,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype],
[Symbol(kUniqueHeaders)]: null
},
_currentUrl: 'http://localhost:3030/catalogs/items',
[Symbol(kCapture)]: false
},
cause: Error: connect ECONNREFUSED 127.0.0.1:3030
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1278:16) {
errno: -111,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 3030
}
}
My files are like that:
Server.js
const express = require('express')
const cors = require('cors')
const axios = require('axios')
require('dotenv').config()
const bodyParser = require("body-parser");
const PORT = process.env.REACT_APP_EXPRESS_PORT
const app = express()
app.use(bodyParser.json());
app.use(cors())
app.get('/items',(req, res) => {
axios.get(`${process.env.REACT_APP_EXPRESS_HOST}:3030/catalogs/items`)
.then((response) => {
res.json(response.data);
})
.catch((error) => {
console.log(error);
})
})
app.post('/items',(req, res) => {
axios.post(`${process.env.REACT_APP_EXPRESS_HOST}:3030/catalogs/items`, req.body)
.then((response) => {
res.json(response.data);
})
.catch((error) => {
console.log(error);
})
})
app.delete('/items/:id',(req, res,) => {
axios.delete(`${process.env.REACT_APP_EXPRESS_HOST}:3030/catalogs/items/:id`)
.then((response) => {
res.json(response.data);
})
.catch((error) => {
console.log(error);
})
})
app.put('/commandes/:id', (req, res) => {
axios.put(`${process.env.REACT_APP_EXPRESS_HOST}:3030/commandes/:id`, req.body)
.then(
(response) => {
res.json(response.data);
}
)
.catch(
(error) => {
console.log(error);
}
)
})
app.listen(PORT, () => console.log(`Backend is running on port ${PORT}`))
.env
REACT_APP_EXPRESS_HOST="http://localhost"
REACT_APP_EXPRESS_PORT="8000"
With Docker
My two applications are running with a devcontainer.json, and that's it, no docker-compose or Dockerfile.
I create a network bridge to make them run on the same network, and add in each devcontainer.json:
"runArgs": [
"--network=mercure-bridge"
],
Here is one of my devcontainer.json, they are the same, just the names are different:
devcontainer.json
{
"name": "MercureFront",
"image": "mcr.microsoft.com/devcontainers/javascript-node:16-bullseye",
"customizations": {
"vscode": {
"settings": {},
"extensions": [
"esbenp.prettier-vscode",
"bierner.color-info",
"jpoissonnier.vscode-styled-components",
"visualstudioexptteam.vscodeintellicode",
"dsznajder.es7-react-js-snippets",
"eg2.vscode-npm-script",
"christian-kohler.npm-intellisense",
"cssho.vscode-svgviewer",
"ms-azuretools.vscode-docker",
"davidanson.vscode-markdownlint",
"IBM.output-colorizer",
"Gruntfuggly.todo-tree",
"bierner.emojisense",
"stkb.rewrap",
"vscode-icons-team.vscode-icons",
"JakeWilson.vscode-cdnjs",
"quicktype.quicktype"
]
}
},
"portsAttributes": {
"3000": {
"label": "Hello Mercure",
"onAutoForward": "notify"
}
},
"runArgs": [
"--network=mercure-bridge"
],
"remoteUser": "node"
}
I checked if they runs on the same network with a Docker inspect:
$ docker network inspect mercure-bridge
[
{
"Name": "mercure-bridge",
"Id": "f98f63b6d9c2d79135ecb9ca564418c52e9936e3413f9dd25fcc0c09b69c7d3b",
"Created": "2022-12-06T15:26:09.919998982Z",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "172.20.0.0/16",
"Gateway": "172.20.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"40584f384551892d974061f621f2106ceaaced6983aafff184dc58a4573d7934": {
"Name": "frosty_taussig",
"EndpointID": "7e8a0454355cf75e6af62da70ac7f75889350dd3a9e8e028112a9aff8991baea",
"MacAddress": "02:42:ac:14:00:03",
"IPv4Address": "172.20.0.3/16",
"IPv6Address": ""
},
"8d78c0afadb2c8086f6747ef726141bdb6d7bae43436a78a6c9a942b0d0279e9": {
"Name": "charming_sanderson",
"EndpointID": "e11d13680f0b2d715ed430160412c7b9d078d61813212c4338bb7a597fe146a3",
"MacAddress": "02:42:ac:14:00:02",
"IPv4Address": "172.20.0.2/16",
"IPv6Address": ""
}
},
"Options": {},
"Labels": {}
}
]
So I tried to change the base URL with :
http://localhost
http://127.0.0.1
the ip of the network
the ip of the mock container
the name of the network
the name of the mock container
the endpoint ID of the mock
the WSL gateway and the WSL2 IP address
I don't have service name, because I don't have docker-compose... but I really tried a lot of things. And have no more ideas, I'm going mad

proxy setup with http-proxy-middleware, unexpected error occurs periodically

As a part of my React (front) / Express (backend) project, I have a web crawler that crawls buffer, decodes then obtain desired data using cheerio.
(Just to clarify, I am not using any data obtained for commercial use, but only for a personal project.)
server.js:
require("dotenv").config();
const express = require("express");
const cors = require("cors");
const http = require("http");
const axios = require("axios");
const cheerio = require("cheerio");
const app = express();
app.set("port", process.env.PORT || 3001);
app.use(cors());
app.use(
express.json({
verify: (request, _, buffer) => {
request.buffer = buffer;
},
})
);
app.get("/", (_, response) => {
response.send("index");
});
app.get("/:menu_id", async (request, response) => {
try {
const buffer = await axios.get("https://cafe.naver.com/ArticleList.nhn", {
params: {
"search.clubid": 10050146,
"search.menuid": request.params.menu_id,
"search.boardType": "L",
userDisplay: 10,
},
headers: {
"Content-Type": "application/json",
},
responseType: "arraybuffer",
});
const decoder = new TextDecoder("EUC-KR");
const content = decoder.decode(buffer.data);
const $ = cheerio.load(content);
const list = $("div.article-board:nth-of-type(4)>table>tbody>tr");
let articles = [];
list.each((_, tag) => {
const url = `https://cafe.naver.com${$(tag).find("a.article").attr("href")}`;
const number = $(tag).find("div.inner_number").text();
const title = $(tag).find("a.article").contents().last().text().trim();
const author = $(tag).find(".p-nick>a").text();
const date = $(tag)
.find("td.td_date")
.text()
.replace(/(\d{4}).(\d{2}).(\d{2})./g, "$2-$3");
articles.push({ number, title, author, date, url });
});
response.send(articles);
} catch (error) {
response.send(error);
}
});
const server = http.createServer(app);
server.listen(app.get("port"), () => {
console.log(`App is listening to port: ${app.get("port")}`);
});
Then I have setup a proxy using http-proxy-middleware on front-end.
setupProxy.js:
const { createProxyMiddleware } = require("http-proxy-middleware");
module.exports = (app) => {
app.use(
createProxyMiddleware("/api", {
target: "http://localhost:3001",
changeOrigin: true,
pathRewrite: {
"^/api": "",
},
})
);
};
Articles.js:
import React, { useEffect, useState } from "react";
import axios from "axios";
import Article from "./Article";
import Loading from "./Loading";
const Articles = (props) => {
const [articles, setArticles] = useState();
useEffect(() => {
const getArticles = () => {
axios
.get(`/api/naver/cafe/${props.menu.id}`)
.then((response) => {
console.log(response);
setArticles(response.data);
})
.catch((error) => {
console.log(error);
});
};
getArticles();
}, []);
return (
<div className="articles">
<div className="menuTitle">{props.menu.title}</div>
{articles ? (
<ul className="menuContent">
{Object.entries(articles).map(([key, article]) => (
<Article article={article} key={key} />
))}
</ul>
) : (
<Loading />
)}
</div>
);
};
export default Articles;
So, this actually works and fetches data as I expected, but then when the server is left running idle, doing nothing, the client fails to fetch data from the server on page refresh (or maybe on a new session) and spits the long, long error below:
/project/node_modules/axios/dist/node/axios.cjs:725
AxiosError.call(axiosError, error.message, code, config, request, response);
^
AxiosError: read ECONNRESET
at AxiosError.from (/project/node_modules/axios/dist/node/axios.cjs:725:14)
at RedirectableRequest.handleRequestError (/project/node_modules/axios/dist/node/axios.cjs:2467:25)
at RedirectableRequest.emit (node:events:513:28)
at eventHandlers.<computed> (/project/node_modules/follow-redirects/index.js:14:24)
at ClientRequest.emit (node:events:513:28)
at TLSSocket.socketErrorListener (node:_http_client:494:9)
at TLSSocket.emit (node:events:513:28)
at emitErrorNT (node:internal/streams/destroy:151:8)
at emitErrorCloseNT (node:internal/streams/destroy:116:3)
at process.processTicksAndRejections (node:internal/process/task_queues:82:21) {
syscall: 'read',
code: 'ECONNRESET',
errno: -104,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: {
FormData: [Function: FormData] {
LINE_BREAK: '\r\n',
DEFAULT_CONTENT_TYPE: 'application/octet-stream'
},
Blob: [class Blob]
},
validateStatus: [Function: validateStatus],
headers: AxiosHeaders {
'User-Agent': 'axios/1.1.3',
'Accept-Encoding': 'gzip, deflate, br',
[Symbol(defaults)]: { Accept: 'application/json, text/plain, */*' }
},
params: {
'search.clubid': 10050146,
'search.menuid': '385',
'search.boardType': 'L',
userDisplay: 10
},
responseType: 'arraybuffer',
method: 'get',
url: 'https://cafe.naver.com/ArticleList.nhn',
data: undefined
},
request: <ref *3> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: Infinity,
protocol: 'https:',
path: '/ArticleList.nhn?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10',
method: 'GET',
headers: [Object: null prototype] {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/1.1.3',
'Accept-Encoding': 'gzip, deflate, br'
},
agents: { http: undefined, https: undefined },
auth: undefined,
beforeRedirect: [Function: dispatchBeforeRedirect],
beforeRedirects: { proxy: [Function: beforeRedirect] },
hostname: 'cafe.naver.com',
port: '',
agent: undefined,
nativeProtocols: {
'http:': {
_connectionListener: [Function: connectionListener],
METHODS: [
'ACL', 'BIND', 'CHECKOUT',
'CONNECT', 'COPY', 'DELETE',
'GET', 'HEAD', 'LINK',
'LOCK', 'M-SEARCH', 'MERGE',
'MKACTIVITY', 'MKCALENDAR', 'MKCOL',
'MOVE', 'NOTIFY', 'OPTIONS',
'PATCH', 'POST', 'PROPFIND',
'PROPPATCH', 'PURGE', 'PUT',
'REBIND', 'REPORT', 'SEARCH',
'SOURCE', 'SUBSCRIBE', 'TRACE',
'UNBIND', 'UNLINK', 'UNLOCK',
'UNSUBSCRIBE'
],
STATUS_CODES: {
'100': 'Continue',
'101': 'Switching Protocols',
'102': 'Processing',
'103': 'Early Hints',
'200': 'OK',
'201': 'Created',
'202': 'Accepted',
'203': 'Non-Authoritative Information',
'204': 'No Content',
'205': 'Reset Content',
'206': 'Partial Content',
'207': 'Multi-Status',
'208': 'Already Reported',
'226': 'IM Used',
'300': 'Multiple Choices',
'301': 'Moved Permanently',
'302': 'Found',
'303': 'See Other',
'304': 'Not Modified',
'305': 'Use Proxy',
'307': 'Temporary Redirect',
'308': 'Permanent Redirect',
'400': 'Bad Request',
'401': 'Unauthorized',
'402': 'Payment Required',
'403': 'Forbidden',
'404': 'Not Found',
'405': 'Method Not Allowed',
'406': 'Not Acceptable',
'407': 'Proxy Authentication Required',
'408': 'Request Timeout',
'409': 'Conflict',
'410': 'Gone',
'411': 'Length Required',
'412': 'Precondition Failed',
'413': 'Payload Too Large',
'414': 'URI Too Long',
'415': 'Unsupported Media Type',
'416': 'Range Not Satisfiable',
'417': 'Expectation Failed',
'418': "I'm a Teapot",
'421': 'Misdirected Request',
'422': 'Unprocessable Entity',
'423': 'Locked',
'424': 'Failed Dependency',
'425': 'Too Early',
'426': 'Upgrade Required',
'428': 'Precondition Required',
'429': 'Too Many Requests',
'431': 'Request Header Fields Too Large',
'451': 'Unavailable For Legal Reasons',
'500': 'Internal Server Error',
'501': 'Not Implemented',
'502': 'Bad Gateway',
'503': 'Service Unavailable',
'504': 'Gateway Timeout',
'505': 'HTTP Version Not Supported',
'506': 'Variant Also Negotiates',
'507': 'Insufficient Storage',
'508': 'Loop Detected',
'509': 'Bandwidth Limit Exceeded',
'510': 'Not Extended',
'511': 'Network Authentication Required'
},
Agent: [Function: Agent] { defaultMaxSockets: Infinity },
ClientRequest: [Function: ClientRequest],
IncomingMessage: [Function: IncomingMessage],
OutgoingMessage: [Function: OutgoingMessage],
Server: [Function: Server],
ServerResponse: [Function: ServerResponse],
createServer: [Function: createServer],
validateHeaderName: [Function: __node_internal_],
validateHeaderValue: [Function: __node_internal_],
get: [Function: get],
request: [Function: request],
setMaxIdleHTTPParsers: [Function: setMaxIdleHTTPParsers],
maxHeaderSize: [Getter],
globalAgent: [Getter/Setter]
},
'https:': {
Agent: [Function: Agent],
globalAgent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object: null prototype],
requests: [Object: null prototype] {},
sockets: [Object: null prototype],
freeSockets: [Object: null prototype] {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: [Object],
[Symbol(kCapture)]: false
},
Server: [Function: Server],
createServer: [Function: createServer],
get: [Function: get],
request: [Function: request]
}
},
pathname: '/ArticleList.nhn',
search: '?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10'
},
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: <ref *1> ClientRequest {
_events: [Object: null prototype] {
response: [Function: bound onceWrapper] {
listener: [Function (anonymous)]
},
abort: [Function (anonymous)],
aborted: [Function (anonymous)],
connect: [Function (anonymous)],
error: [Function (anonymous)],
socket: [Function (anonymous)],
timeout: [Function (anonymous)]
},
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
strictContentLength: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: <ref *2> TLSSocket {
_tlsOptions: {
allowHalfOpen: undefined,
pipe: false,
secureContext: SecureContext { context: SecureContext {} },
isServer: false,
requestCert: true,
rejectUnauthorized: true,
session: undefined,
ALPNProtocols: undefined,
requestOCSP: undefined,
enableTrace: undefined,
pskCallback: undefined,
highWaterMark: undefined,
onread: undefined,
signal: undefined
},
_secureEstablished: false,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
secureConnecting: true,
_SNICallback: null,
servername: null,
alpnProtocol: null,
authorized: false,
authorizationError: null,
encrypted: true,
_events: [Object: null prototype] {
close: [
[Function: onSocketCloseDestroySSL],
[Function],
[Function: onClose],
[Function: socketCloseListener]
],
end: [ [Function: onConnectEnd], [Function: onReadableStreamEnd] ],
newListener: [Function: keylogNewListener],
secure: [Function: onConnectSecure],
session: [Function (anonymous)],
free: [Function: onFree],
timeout: [Function: onTimeout],
agentRemove: [Function: onRemove],
error: [Function: socketErrorListener],
drain: [Function: ondrain]
},
_eventsCount: 10,
connecting: false,
_hadError: true,
_parent: null,
_host: 'cafe.naver.com',
_closeAfterHandlingError: false,
_readableState: ReadableState {
objectMode: false,
highWaterMark: 16384,
buffer: BufferList { head: null, tail: null, length: 0 },
length: 0,
pipes: [],
flowing: true,
ended: false,
endEmitted: false,
reading: true,
constructed: true,
sync: false,
needReadable: true,
emittedReadable: false,
readableListening: false,
resumeScheduled: false,
errorEmitted: true,
emitClose: false,
autoDestroy: true,
destroyed: true,
errored: Error: read ECONNRESET
at TLSWrap.onStreamRead (node:internal/stream_base_commons:217:20) {
errno: -104,
code: 'ECONNRESET',
syscall: 'read'
},
closed: true,
closeEmitted: true,
defaultEncoding: 'utf8',
awaitDrainWriters: null,
multiAwaitDrain: false,
readingMore: false,
dataEmitted: false,
decoder: null,
encoding: null,
[Symbol(kPaused)]: false
},
_maxListeners: undefined,
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: true,
decodeStrings: false,
defaultEncoding: 'utf8',
length: 253,
writing: true,
corked: 0,
sync: false,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: [Function: bound onFinish],
writelen: 253,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 1,
constructed: true,
prefinished: false,
errorEmitted: true,
emitClose: false,
autoDestroy: true,
errored: Error: read ECONNRESET
at TLSWrap.onStreamRead (node:internal/stream_base_commons:217:20) {
errno: -104,
code: 'ECONNRESET',
syscall: 'read'
},
closed: true,
closeEmitted: true,
[Symbol(kOnFinished)]: []
},
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: null,
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular *1],
[Symbol(res)]: TLSWrap {
_parent: TCP {
reading: [Getter/Setter],
onconnection: null,
[Symbol(owner_symbol)]: [Circular *2],
[Symbol(handle_onclose)]: [Function: done]
},
_parentWrap: undefined,
_secureContext: SecureContext { context: SecureContext {} },
reading: true,
onkeylog: [Function: onkeylog],
onhandshakestart: {},
onhandshakedone: [Function (anonymous)],
onocspresponse: [Function: onocspresponse],
onnewsession: [Function: onnewsessionclient],
onerror: [Function: onerror],
[Symbol(owner_symbol)]: [Circular *2]
},
[Symbol(verified)]: false,
[Symbol(pendingSession)]: null,
[Symbol(async_id_symbol)]: 18,
[Symbol(kHandle)]: null,
[Symbol(lastWriteQueueSize)]: 253,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kSetNoDelay)]: false,
[Symbol(kSetKeepAlive)]: true,
[Symbol(kSetKeepAliveInitialDelay)]: 60,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 253,
[Symbol(connect-options)]: {
rejectUnauthorized: true,
ciphers: 'TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA',
checkServerIdentity: [Function: checkServerIdentity],
minDHSize: 1024,
maxRedirects: 21,
maxBodyLength: Infinity,
protocol: 'https:',
path: null,
method: 'GET',
headers: [Object: null prototype] {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/1.1.3',
'Accept-Encoding': 'gzip, deflate, br'
},
agents: { http: undefined, https: undefined },
auth: undefined,
beforeRedirect: [Function: dispatchBeforeRedirect],
beforeRedirects: { proxy: [Function: beforeRedirect] },
hostname: 'cafe.naver.com',
port: 443,
agent: undefined,
nativeProtocols: { 'http:': [Object], 'https:': [Object] },
pathname: '/ArticleList.nhn',
search: '?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10',
_defaultAgent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object: null prototype],
requests: [Object: null prototype] {},
sockets: [Object: null prototype],
freeSockets: [Object: null prototype] {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: [Object],
[Symbol(kCapture)]: false
},
host: 'cafe.naver.com',
noDelay: true,
servername: 'cafe.naver.com',
_agentKey: 'cafe.naver.com:443:::::::::::::::::::::',
encoding: null,
singleUse: true
}
},
_header: 'GET /ArticleList.nhn?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10 HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/1.1.3\r\n' +
'Accept-Encoding: gzip, deflate, br\r\n' +
'Host: cafe.naver.com\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: Agent {
_events: [Object: null prototype] {
free: [Function (anonymous)],
newListener: [Function: maybeEnableKeylog]
},
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object: null prototype] { noDelay: true, path: null },
requests: [Object: null prototype] {},
sockets: [Object: null prototype] {
'cafe.naver.com:443:::::::::::::::::::::': [ [TLSSocket] ]
},
freeSockets: [Object: null prototype] {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: { map: {}, list: [] },
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/ArticleList.nhn?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'cafe.naver.com',
protocol: 'https:',
_redirectable: [Circular *3],
[Symbol(kCapture)]: false,
[Symbol(kBytesWritten)]: 0,
[Symbol(kEndCalled)]: true,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [ 'Accept', 'application/json, text/plain, */*' ],
'user-agent': [ 'User-Agent', 'axios/1.1.3' ],
'accept-encoding': [ 'Accept-Encoding', 'gzip, deflate, br' ],
host: [ 'Host', 'cafe.naver.com' ]
},
[Symbol(kUniqueHeaders)]: null
},
_currentUrl: 'https://cafe.naver.com/ArticleList.nhn?search.clubid=10050146&search.menuid=385&search.boardType=L&userDisplay=10',
[Symbol(kCapture)]: false
},
cause: Error: read ECONNRESET
at TLSWrap.onStreamRead (node:internal/stream_base_commons:217:20) {
errno: -104,
code: 'ECONNRESET',
syscall: 'read'
}
}
It just feels like the client loses the connection to the server via proxy.
Here's the things I've tried:
adding connection: "keep-alive" option to createProxyMiddleware header
switching http request client to built in fetch from axios
trying full length(?) url instead of shortified version (/api/naver/cafe/${props.menu.id} → http://localhost:3000/api/naver/cafe/${props.menu.id})
and nothing really resolved the issue.
The only way to fix this error is either by refreshing the page until it works or restarting the server.
I do have some Twitch's helix related apis as well, but this error only occurs with the crawling function above.
Literally spent hours and hours to troubleshoot this... and failed.
Any ideas, please?
I am using NodeJS v18.10.0 btw.

connect EADDRINUSE 18.62.228.13:443 when parsing data to the db

i'm scraping some data from the web and want to push all the data that comes to my db.
I do it with node.js with help of thread_workers.
each thread scrap data from another url and when the scraping is done and I got all the objects that I need I post it.
in each thread when it's end to scrap the data i get an array with minimum 40000 objects ,
that i need to post when i start to post it to the db in some point it's throw me this error connect EADDRINUSE 18.62.228.13:443 full exception :
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 168
},
method: 'post'
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: '/m/insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalalalal'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 168,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST lalalala HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 168\r\n' +
'Host: alalalala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'search.findmanual.guru',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: ,
[Symbol(kCapture)]: false
}
}
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 167
},
method: 'post',
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: 'insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalallalaa'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 167,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 167\r\n' +
'Host: lallala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'sllalallaau',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'https://lalalalal',
[Symbol(kCapture)]: false
}
}
and here is my thread function :
async function parseData(url) {
const {data} = await axios.get(url ).catch(console.log)
const $ = cheerio.load(data);
const result = [];
const obj = {};
const header = $('h1.Uheader');
obj.brand = $(header[0]).text();
const category = $('div.cathead');
const categoryArray = [];
for (let i = 0; i < category.length; i++) {
categoryArray.push({"href": $(category[i]).children("a").attr('href'), "text":
$(category[i]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim()})
}
for (let i = 0; i < categoryArray.length; i++) {
try {
obj.category = categoryArray[i].text;
const dataTwo = await axios.get("https://www.lalalala.com" +
categoryArray[i].href).catch()
const cher = cheerio.load(dataTwo.data);
const aTag = cher('div.col-sm-2.mname')
for (let j = 0 ; j < aTag.length; j++) {
obj.url = "https://www.lalalala.com" + $(aTag[j]).children("a").attr('href');
obj.title = obj.brand + " " + categoryArray[i].text + " " + $(aTag[j]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim();
//result.push(obj)
axios.post("https://lalalalala", obj)
.then(data => console.log("ok " + j))
.catch(e => {
console.log(e)
});
}
} catch (e) {
}
}
parentPort.postMessage({message : "done"});
}
please can someone please explain me why this is happning and how can I fix it ?
Because (based on your comment here) your application is performing a lot of POST requests, it's important to limit the number of concurrent POST requests that can be performed. Otherwise, there may be (10's of) thousands of POST requests being executed at about the same time, which can cause problems (not only locally, but also on the server the requests are sent to).
The error connect EADDRINUSE means that there are no more free TCP ports available on the local machine (a TCP connection to a remote server also requires a local TCP port, called an "ephemeral port" because it's only in use for the duration of the request) because there are too many requests pending.
An possible fix, and one that you said is working, is to wait for each POST request to finish before continuing the rest of the code:
await axios.post(…)
If that also doesn't help you'd have to look into job queues that can be configured to limit the number of concurrent requests/promises/jobs/...).

Axios Typescript, fetching from openweathermap api causes bad request or ECONNREFUSED 127.0.0.1:80

Im having issues connecting to openweathermaps API in my express app,
app.get("/", cors(), async (req, res) => {
const lat: number = + req.query.lat;
const lon: number = + req.query.lon;
if (lat && lon) {
const entry = cach.filter(e => e.lat === lat && e.lon === lon);
if (entry.some(e => (Date.now() - e.last_updated < 36000000))) {
res.json(entry[0]);
} else { // no entry found
res.json(await getWeather(lat, lon));
}
}
res.send("no location specified")
});
app.listen(PORT, function () {
console.log(`App is listening on port ${PORT}`); //5000 by default
});
to fetch the data im using axios:
async function getWeather(lat: number, lon: number) {
try {
const res = await axios({
url:`http:api.openweathermap.org/data/2.5/weather?lat=${lat}&lon=${lon}&appid=${process.env.WAPI}`,
method:"GET"
});
const data = res.data;
...
catch (error){
console.log(error)
}
but the response i get is:
App is listening on port 5000
at Wed Dec 09 2020 15:22:21 GMT+0100 (Central European Standard Time): http://localhost:5000/?lat=65.58415&lon=22.15465: GET request
Error: connect ECONNREFUSED 127.0.0.1:80
>at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1133:16)
{
errno: -111,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 80,
config: {
url: 'http:api.openweathermap.org/data/2.5/weather?lat=65.58415&lon=22.15465&appid=--apiket--',
method: 'get',
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.21.0'
},
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
validateStatus: [Function: validateStatus],
data: undefined
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError]
},
_eventsCount: 2,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'http:',
path: 'api.openweathermap.org/data/2.5/weather?lat=65.58415&lon=22.15465&appid=--apikey--',
method: 'GET',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: null,
port: null,
nativeProtocols: [Object],
pathname: 'api.openweathermap.org/data/2.5/weather',
search: '?lat=65.58415&lon=22.15465&appid=--apikey--'
},
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: [Socket],
_header: 'GET api.openweathermap.org/data/2.5/weather?lat=65.58415&lon=22.15465&appid=--apikey-- HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.21.0\r\n' +
'Host: localhost\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: 'api.openweathermap.org/data/2.5/weather?lat=65.58415&lon=22.15465&appid=--apikey--',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'http:api.openweathermap.org/data/2.5/weather?lat=65.58415&lon=22.15465&appid=--apikey--',
[Symbol(kCapture)]: false
},
response: undefined,
isAxiosError: true,
toJSON: [Function: toJSON]
}
node:internal/process/promises:225
triggerUncaughtException(err, true /* fromPromise */);
^
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
at new NodeError (node:internal/errors:278:15)
at ServerResponse.setHeader (node:_http_outgoing:563:11)
at ServerResponse.header (/home/johanrh/Documents/Git/M7011E_AJ/node_modules/express/lib/response.js:771:10)
at ServerResponse.send (/home/johanrh/Documents/Git/M7011E_AJ/node_modules/express/lib/response.js:170:12)
at /home/johanrh/Documents/Git/M7011E_AJ/build/src/Weather-Module/index.js:32:9
at processTicksAndRejections (node:internal/process/task_queues:93:5) {
code: 'ERR_HTTP_HEADERS_SENT'
}
the provided url path when put in the browser causes no issus, Anyone have any ideas?

Fix Axios Error ECONNREFUSED on local domain with no proxy

I'm bumping into this stupid issue with Axios.
I'm trying to send a local POST request to my basic local Symfony app with the following piece of code that is a nightmare JS crawler:
Yuzu.fun is my local dev domain with an entry in /etc/hosts to resolve it, the app is running on port 80.
await axios.request({
url: 'http://yuzu.fun/app_dev.php/api/path/to/endpoint',
headers: {
'key': 'xxxxxx'
},
method: 'post',
proxy: false,
data: {
image_data: data
}
})
.then(function (response) {
map_hash = response.message;
console.log('buildMap', map_hash);
})
.catch(function (error) {
console.error('upload failed:', error);
});
When i run my script, here is the error i get:
upload failed: { Error: connect ECONNREFUSED 127.0.0.1:80
at Object._errnoException (util.js:1022:11)
at _exceptionWithHostPort (util.js:1044:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)
code: 'ECONNREFUSED',
errno: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 80,
config:
{ adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers:
{ Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json;charset=utf-8',
key: 'xxxxx',
'User-Agent': 'axios/0.18.0',
'Content-Length': 4173 },
method: 'post',
url: 'http://yuzu.fun/app_dev.php/api/path/to/endpoint',
proxy: false,
data: '{"image_data":"imagedatainbase64"}' },
request:
Writable {
_writableState:
WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
bufferedRequest: null,
lastBufferedRequest: null,
pendingcb: 0,
prefinished: false,
errorEmitted: false,
bufferedRequestCount: 0,
corkedRequestsFree: [Object] },
writable: true,
domain: null,
_events:
{ response: [Function: handleResponse],
error: [Function: handleRequestError] },
_eventsCount: 2,
_maxListeners: undefined,
_options:
{ maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'http:',
path: '/app_dev.php/api/path/to/endpoint',
method: 'post',
headers: [Object],
agent: undefined,
auth: undefined,
hostname: 'yuzu.fun',
port: null,
nativeProtocols: [Object],
pathname: '/app_dev.php/api/path/to/endpoint' },
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 4173,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function],
_currentRequest:
ClientRequest {
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: true,
upgrading: false,
chunkedEncoding: false,
shouldKeepAlive: false,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
socket: [Object],
connection: [Object],
_header: 'POST /app_dev.php/api/path/to/endpoint HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: application/json;charset=utf-8\r\nkey: xxxxx\r\nUser-Agent: axios/0.18.0\r\nContent-Length: 4173\r\nHost: yuzu.fun\r\nConnection: close\r\n\r\n',
_onPendingData: [Function: noopPendingOutput],
agent: [Object],
socketPath: undefined,
timeout: undefined,
method: 'POST',
path: '/app_dev.php/api/path/to/endpoint',
_ended: false,
res: null,
aborted: undefined,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
_redirectable: [Circular],
[Symbol(outHeadersKey)]: [Object] },
_currentUrl: 'http://yuzu.fun//app_dev.php/api/path/to/endpoint' },
response: undefined }
I tried to post to a random endpoint to check if the connection was working when reaching a live domain and it's OK but it doesn't work locally.
When i check stackoverflow or forums i see people mentioning proxies issues so i forced it to false but i don't think that something i'm concerned with.
This code was first written with request.post() but i need to have Axios Promises benefits. When the code was written with request, the local domain was reachable correctly. So i exclude the fact that nightmare could interfere with the domain resolution.
Does anyone have any idea of was could go wrong here?
Thanks in advance!

Resources