Related
I am trying to use Marea Tide Api to build a tide application returning results based on location (latitude, longitude).
I was originally struggling to make a successful fetch from the front end due to a CORS error. I am now trying to build an express backend to both hide my api keys and navigate the CORS issue with a proxy.
I keep getting an unauthorized error when trying to navigate to localhost:8000/tides.
Express:
const PORT = 8000;
const express = require('express');
const cors = require('cors');
const axios = require('axios');
const { application, json } = require('express');
require('dotenv').config
const backend = express()
backend.listen(PORT, () => console.log(`Tide backend is working on PORT:${PORT}`))
backend.get('/tides', (req, res, next) => {
const url = 'https://api.marea.ooo/v2/tides';
const options = {
params: {
duration: 1440,
interval: 60,
latitude: 36.888,
longitude: -76.100,
}
};
const config = {
headers: {
'x-marea-api-token': process.env.MareaToken,
}
};
axios.get(url, options, config)
.then(response => {console.log(response)})
.catch(err => console.log(err))
});
Terminal Error:
[nodemon] starting `node backend.js`
Tide backend is working on PORT:8000
[AxiosError: Request failed with status code 401] {
code: 'ERR_BAD_REQUEST',
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.27.2'
},
params: {
duration: 1440,
interval: 60,
latitude: 36.888,
longitude: -76.1
},
method: 'get',
url: 'https://api.marea.ooo/v2/tides',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype] {
abort: [Function (anonymous)],
aborted: [Function (anonymous)],
connect: [Function (anonymous)],
error: [Function (anonymous)],
socket: [Function (anonymous)],
timeout: [Function (anonymous)],
prefinish: [Function: requestOnPrefinish]
},
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: TLSSocket {
_tlsOptions: [Object],
_secureEstablished: true,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
secureConnecting: false,
_SNICallback: null,
servername: 'api.marea.ooo',
alpnProtocol: false,
authorized: true,
authorizationError: null,
encrypted: true,
_events: [Object: null prototype],
_eventsCount: 10,
connecting: false,
_hadError: false,
_parent: null,
_host: 'api.marea.ooo',
_readableState: [ReadableState],
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: [TLSWrap],
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular *1],
[Symbol(res)]: [TLSWrap],
[Symbol(verified)]: true,
[Symbol(pendingSession)]: null,
[Symbol(async_id_symbol)]: 17,
[Symbol(kHandle)]: [TLSWrap],
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kSetNoDelay)]: false,
[Symbol(kSetKeepAlive)]: true,
[Symbol(kSetKeepAliveInitialDelay)]: 60,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(connect-options)]: [Object],
[Symbol(RequestTimeout)]: undefined
},
_header: 'GET /v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1 HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Host: api.marea.ooo\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object: null prototype],
requests: [Object: null prototype] {},
sockets: [Object: null prototype],
freeSockets: [Object: null prototype] {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
maxCachedSessions: 100,
_sessionCache: [Object],
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1',
_ended: true,
res: IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 4,
_maxListeners: undefined,
socket: [TLSSocket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
rawHeaders: [Array],
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 401,
statusMessage: 'Unauthorized',
client: [TLSSocket],
_consuming: false,
_dumped: false,
req: [Circular *1],
responseUrl: 'https://api.marea.ooo/v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1',
redirects: [],
[Symbol(kCapture)]: false,
[Symbol(kHeaders)]: [Object],
[Symbol(kHeadersCount)]: 16,
[Symbol(kTrailers)]: null,
[Symbol(kTrailersCount)]: 0,
[Symbol(RequestTimeout)]: undefined
},
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'api.marea.ooo',
protocol: 'https:',
_redirectable: Writable {
_writableState: [WritableState],
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
_options: [Object],
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: [Circular *1],
_currentUrl: 'https://api.marea.ooo/v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1',
[Symbol(kCapture)]: false
},
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [Array],
'user-agent': [Array],
host: [Array]
}
},
response: {
status: 401,
statusText: 'Unauthorized',
headers: {
'cache-control': 'max-age=0, private, must-revalidate',
'content-length': '40',
'content-type': 'application/json; charset=utf-8',
date: 'Tue, 19 Jul 2022 18:46:46 GMT',
server: 'Caddy',
'x-request-id': 'FwNPaorYW0wySfUApCrB',
connection: 'close'
},
config: {
transitional: [Object],
adapter: [Function: httpAdapter],
transformRequest: [Array],
transformResponse: [Array],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: [Object],
validateStatus: [Function: validateStatus],
headers: [Object],
params: [Object],
method: 'get',
url: 'https://api.marea.ooo/v2/tides',
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'GET /v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1 HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Host: api.marea.ooo\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/v2/tides?duration=1440&interval=60&latitude=36.888&longitude=-76.1',
_ended: true,
res: [IncomingMessage],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'api.marea.ooo',
protocol: 'https:',
_redirectable: [Writable],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
data: { status: 401, error: 'Unauthorized' }
}
}
axios.get() accepts only two parameters...
axios.get(url[, config])
Combine your options and config objects and don't forget to send an actual response.
axios.get(url, { ...options, ...config })
.then(({ data }) => res.json(data))
.catch(next);
i'm scraping some data from the web and want to push all the data that comes to my db.
I do it with node.js with help of thread_workers.
each thread scrap data from another url and when the scraping is done and I got all the objects that I need I post it.
in each thread when it's end to scrap the data i get an array with minimum 40000 objects ,
that i need to post when i start to post it to the db in some point it's throw me this error connect EADDRINUSE 18.62.228.13:443 full exception :
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 168
},
method: 'post'
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: '/m/insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalalalal'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 168,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST lalalala HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 168\r\n' +
'Host: alalalala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'search.findmanual.guru',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: ,
[Symbol(kCapture)]: false
}
}
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 167
},
method: 'post',
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: 'insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalallalaa'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 167,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 167\r\n' +
'Host: lallala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'sllalallaau',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'https://lalalalal',
[Symbol(kCapture)]: false
}
}
and here is my thread function :
async function parseData(url) {
const {data} = await axios.get(url ).catch(console.log)
const $ = cheerio.load(data);
const result = [];
const obj = {};
const header = $('h1.Uheader');
obj.brand = $(header[0]).text();
const category = $('div.cathead');
const categoryArray = [];
for (let i = 0; i < category.length; i++) {
categoryArray.push({"href": $(category[i]).children("a").attr('href'), "text":
$(category[i]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim()})
}
for (let i = 0; i < categoryArray.length; i++) {
try {
obj.category = categoryArray[i].text;
const dataTwo = await axios.get("https://www.lalalala.com" +
categoryArray[i].href).catch()
const cher = cheerio.load(dataTwo.data);
const aTag = cher('div.col-sm-2.mname')
for (let j = 0 ; j < aTag.length; j++) {
obj.url = "https://www.lalalala.com" + $(aTag[j]).children("a").attr('href');
obj.title = obj.brand + " " + categoryArray[i].text + " " + $(aTag[j]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim();
//result.push(obj)
axios.post("https://lalalalala", obj)
.then(data => console.log("ok " + j))
.catch(e => {
console.log(e)
});
}
} catch (e) {
}
}
parentPort.postMessage({message : "done"});
}
please can someone please explain me why this is happning and how can I fix it ?
Because (based on your comment here) your application is performing a lot of POST requests, it's important to limit the number of concurrent POST requests that can be performed. Otherwise, there may be (10's of) thousands of POST requests being executed at about the same time, which can cause problems (not only locally, but also on the server the requests are sent to).
The error connect EADDRINUSE means that there are no more free TCP ports available on the local machine (a TCP connection to a remote server also requires a local TCP port, called an "ephemeral port" because it's only in use for the duration of the request) because there are too many requests pending.
An possible fix, and one that you said is working, is to wait for each POST request to finish before continuing the rest of the code:
await axios.post(…)
If that also doesn't help you'd have to look into job queues that can be configured to limit the number of concurrent requests/promises/jobs/...).
I'm trying to bundle together a node app and a Open Policy Agent server using docker compose, i can access the OPA server from my browser but when i tried to to a request using axios from the node app i get this:
Error: connect ECONNREFUSED 127.0.0.1:8181
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1159:16) {
errno: -111,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 8181,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/0.26.0'
},
method: 'get',
url: 'http://localhost:8181',
data: undefined
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'http:',
path: '/',
method: 'GET',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'localhost',
port: '8181',
nativeProtocols: [Object],
pathname: '/'
},
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [Socket],
_header: 'GET / HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/0.26.0\r\n' +
'Host: localhost:8181\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'http://localhost:8181/',
[Symbol(kCapture)]: false
},
response: undefined,
isAxiosError: true,
toJSON: [Function: toJSON]
}
I also tried to run 2 node app with the same results so the opa servers is no the problem
I created an network and make sure both containers are part of it:
$ docker network inspect -f '{{range .Containers}}{{.Name}} {{end}}' opa_network
result: opa-nodejs_client_1 opa-nodejs_server_1
$ docker exec opa-nodejs_client_1 ping opa-nodejs_server_1 -c2
Result: PING opa-nodejs_server_1 (172.31.0.3) 56(84) bytes of data.
64 bytes from opa-nodejs_server_1.opa_network (172.31.0.3): icmp_seq=1 ttl=64 time=0.065 ms
64 bytes from opa-nodejs_server_1.opa_network (172.31.0.3): icmp_seq=2 ttl=64 time=0.128 ms
--- opa-nodejs_server_1 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1023ms
rtt min/avg/max/mdev = 0.065/0.096/0.128/0.033 ms
this is the docker compose file:
version: '3.9'
services:
server:
build:
dockerfile: Dockerfile
context: ./OPA
ports:
- 8181:8181
networks:
- opa_network
client:
build:
dockerfile: Dockerfile
context: ./nodejs_google_auth
ports:
- 3000:3000
networks:
- opa_network
networks:
opa_network:
external: true
thanks, i found the solution, each docker container has its own ip address, so i had to use the ip address of the server container instead of "localhost"
I have node running from a docker container and I tried to write some websocket example. But I an getting the next problem:
root#7cd5a41c8eea:/application/webSocketEx# node
> var connection = new WebSocket('ws://html5rocks.websocket.org/echo', ['soap', 'xmpp']);
ReferenceError: WebSocket is not defined
> const WebSocket = require('ws');
undefined
> var ws = new WebSocket("ws://www.websocket.org");
undefined
> { Error: Unexpected server response: 404
at ClientRequest.req.on (/application/webSocketEx/node_modules/ws/lib/websocket.js:579:7)
at ClientRequest.emit (events.js:182:13)
at ClientRequest.EventEmitter.emit (domain.js:460:23)
at HTTPParser.parserOnIncomingClient [as onIncoming] (_http_client.js:555:21)
at HTTPParser.parserOnHeadersComplete (_http_common.js:109:17)
at Socket.socketOnData (_http_client.js:441:20)
at Socket.emit (events.js:182:13)
at Socket.EventEmitter.emit (domain.js:460:23)
at addChunk (_stream_readable.js:283:12)
at readableAddChunk (_stream_readable.js:264:11)
domainEmitter:
WebSocket {
domain:
Domain {
domain: null,
_events: [Object],
_eventsCount: 3,
_maxListeners: undefined,
members: [] },
_events: {},
_eventsCount: 0,
_maxListeners: undefined,
readyState: 2,
protocol: '',
_binaryType: 'nodebuffer',
_closeFrameReceived: false,
_closeFrameSent: false,
_closeMessage: '',
_closeTimer: null,
_closeCode: 1006,
_extensions: {},
_receiver: null,
_sender: null,
_socket: null,
_bufferedAmount: 0,
_isServer: false,
_redirects: 0,
url: 'ws://www.websocket.org',
_req:
ClientRequest {
domain: [Domain],
_events: [Object],
_eventsCount: 5,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [Socket],
connection: [Socket],
_header:
'GET / HTTP/1.1\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: 2c4HivAKv2OJ6rO7qYyaNA==\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Extensions: permessage-deflate; client_max_window_bits\r\nHost: www.websocket.org\r\n\r\n',
_onPendingData: [Function: noopPendingOutput],
agent: undefined,
socketPath: undefined,
timeout: undefined,
method: 'GET',
path: '/',
_ended: false,
res: [IncomingMessage],
aborted: 1563389938612,
timeoutCb: null,
upgradeOrConnect: false,
parser: [HTTPParser],
maxHeadersCount: null,
[Symbol(isCorked)]: false,
[Symbol(outHeadersKey)]: [Object] } },
domain:
Domain {
domain: null,
_events:
{ removeListener: [Function: updateExceptionCapture],
newListener: [Function: updateExceptionCapture],
error: [Function: debugDomainError] },
_eventsCount: 3,
_maxListeners: undefined,
members: [] },
domainThrown: false }
>
Is there a configuration parameter I am missing to use websockets from a docker container and node? I installed ws using npm install ws already. I have the feeling I need to open a port for the docker container but I do not understand which one.
Thanks
Docker container runs on their own subnet and therefore they are not allowed to access any other hosts from it. You need to explicitly change your docker subnet to your host subnet (bridging network) and once your docker container is on the same network as host your container will be able to access internet.
You need to implement something like this.
docker network create -d bridge --subnet 192.168.0.0/24 --gateway 192.168.0.1 dockernet
version: '2'
services:
web:
image: some/image
networks:
- dockernet
networks:
dockernet:
external: true
For more information you can read the document here for your specific use case:
https://docs.docker.com/network/network-tutorial-standalone/
https://technology.amis.nl/2018/08/18/docker-host-and-bridged-networking-running-library-httpd-on-different-ports/
I'm bumping into this stupid issue with Axios.
I'm trying to send a local POST request to my basic local Symfony app with the following piece of code that is a nightmare JS crawler:
Yuzu.fun is my local dev domain with an entry in /etc/hosts to resolve it, the app is running on port 80.
await axios.request({
url: 'http://yuzu.fun/app_dev.php/api/path/to/endpoint',
headers: {
'key': 'xxxxxx'
},
method: 'post',
proxy: false,
data: {
image_data: data
}
})
.then(function (response) {
map_hash = response.message;
console.log('buildMap', map_hash);
})
.catch(function (error) {
console.error('upload failed:', error);
});
When i run my script, here is the error i get:
upload failed: { Error: connect ECONNREFUSED 127.0.0.1:80
at Object._errnoException (util.js:1022:11)
at _exceptionWithHostPort (util.js:1044:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)
code: 'ECONNREFUSED',
errno: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 80,
config:
{ adapter: [Function: httpAdapter],
transformRequest: { '0': [Function: transformRequest] },
transformResponse: { '0': [Function: transformResponse] },
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus],
headers:
{ Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json;charset=utf-8',
key: 'xxxxx',
'User-Agent': 'axios/0.18.0',
'Content-Length': 4173 },
method: 'post',
url: 'http://yuzu.fun/app_dev.php/api/path/to/endpoint',
proxy: false,
data: '{"image_data":"imagedatainbase64"}' },
request:
Writable {
_writableState:
WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
bufferedRequest: null,
lastBufferedRequest: null,
pendingcb: 0,
prefinished: false,
errorEmitted: false,
bufferedRequestCount: 0,
corkedRequestsFree: [Object] },
writable: true,
domain: null,
_events:
{ response: [Function: handleResponse],
error: [Function: handleRequestError] },
_eventsCount: 2,
_maxListeners: undefined,
_options:
{ maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'http:',
path: '/app_dev.php/api/path/to/endpoint',
method: 'post',
headers: [Object],
agent: undefined,
auth: undefined,
hostname: 'yuzu.fun',
port: null,
nativeProtocols: [Object],
pathname: '/app_dev.php/api/path/to/endpoint' },
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 4173,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function],
_currentRequest:
ClientRequest {
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: true,
upgrading: false,
chunkedEncoding: false,
shouldKeepAlive: false,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
socket: [Object],
connection: [Object],
_header: 'POST /app_dev.php/api/path/to/endpoint HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: application/json;charset=utf-8\r\nkey: xxxxx\r\nUser-Agent: axios/0.18.0\r\nContent-Length: 4173\r\nHost: yuzu.fun\r\nConnection: close\r\n\r\n',
_onPendingData: [Function: noopPendingOutput],
agent: [Object],
socketPath: undefined,
timeout: undefined,
method: 'POST',
path: '/app_dev.php/api/path/to/endpoint',
_ended: false,
res: null,
aborted: undefined,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
_redirectable: [Circular],
[Symbol(outHeadersKey)]: [Object] },
_currentUrl: 'http://yuzu.fun//app_dev.php/api/path/to/endpoint' },
response: undefined }
I tried to post to a random endpoint to check if the connection was working when reaching a live domain and it's OK but it doesn't work locally.
When i check stackoverflow or forums i see people mentioning proxies issues so i forced it to false but i don't think that something i'm concerned with.
This code was first written with request.post() but i need to have Axios Promises benefits. When the code was written with request, the local domain was reachable correctly. So i exclude the fact that nightmare could interfere with the domain resolution.
Does anyone have any idea of was could go wrong here?
Thanks in advance!