Related
I have been trying to connect and retrieve a list of data from Business Central.
The Web Service has been exposed from BC and the OData link works and displays the JSON for the data in "Articles".
I'm trying to consumer that Service with NodeJS using NTLM (i also tried basic authentication and the same issue persists.
Please find below the file responsible for consuming the BC Web Service followed by the 401 error.
I have already stored all the information needed for NTLM config in my .env file, I did try to hardcode them and the same issue
const express = require("express");
const axios = require("axios");
const router = express.Router();
const ntlm = require("express-ntlm");
//const Article = require("../models/Articles.js");
const domain = process.env.DOMAIN;
const username = process.env.USERNAME;
const password = process.env.PASSWORD;
// Define auth function
const auth = ntlm({
debug: console.log,
domain,
username,
password,
ntlm_version: 2,
reconnect: true,
send_401: function (res) {
res.sendStatus(401);
},
badrequest: function (res) {
res.sendStatus(400);
},
});
// Get customer data from Business Central
async function getArticlesFromBC() {
try {
const options = {
auth: {
username,
password,
workstation: process.env.WORKSTATION,
domain,
},
};
const companyId = "CRONUS France S.A.";
const encodedCompanyId = encodeURIComponent(companyId);
const url = `http://${process.env.SERVER}:7048/BC210/ODataV4/Company('${encodedCompanyId}')/ItemListec`;
const response = await axios.get(url, options);
return response.data;
} catch (error) {
console.error(error);
throw new Error("Error retrieving articles data from Business Central");
}
}
// Route to get article data
router.get("/", auth, async (req, res) => {
try {
const user = req.ntlm;
console.log(user);
let articles = await getArticlesFromBC();
res.json(articles);
} catch (error) {
console.error(error);
res
.status(500)
.send("Error retrieving articles data from Business Central");
}
});
module.exports = router;
Explanation
This is the auth function that will use the variables set in .env to authenticate the user in order to access the BC endpoint
// Define auth function
const auth = ntlm({
debug: console.log,
domain,
username,
password,
ntlm_version: 2,
reconnect: true,
send_401: function (res) {
res.sendStatus(401);
},
badrequest: function (res) {
res.sendStatus(400);
},
});
This is the function that connects to the endpoint URL and returns the JSON file, this is where the error triggers
// Get article data from Business Central
async function getArticlesFromBC() {
try {
const options = {
auth: {
username,
password,
workstation: process.env.WORKSTATION,
domain,
},
};
const companyId = "CRONUS France S.A.";
const encodedCompanyId = encodeURIComponent(companyId);
const url = `http://${process.env.SERVER}:7048/BC210/ODataV4/Company('${encodedCompanyId}')/ItemListec`;
const response = await axios.get(url, options);
return response.data;
} catch (error) {
console.error(error);
throw new Error("Error retrieving articles data from Business Central");
}
}
This is the route to access the nodejs api
// Route to get article data
router.get("/", auth, async (req, res) => {
try {
const user = req.ntlm;
console.log(user);
let articles = await getArticlesFromBC();
res.json(articles);
} catch (error) {
console.error(error);
res
.status(500)
.send("Error retrieving articles data from Business Central");
}
});
This is the 401 error json, you can see at the beginning that NTLM authenticates successfully but then it throws the catch block in the get function
[express-ntlm] No Authorization header present
[express-ntlm] No domaincontroller was specified, all Authentication messages are valid.
{
DomainName: 'DESKTOP-1EF91E4',
UserName: 'ahmed',
Workstation: 'DESKTOP-1EF91E4',
Authenticated: true
}
AxiosError: Request failed with status code 401
at settle (D:\ESPRIT\5eme\PFE\B2B-ERP-MERN-App\node_modules\axios\dist\node\axios.cjs:1900:12)
at IncomingMessage.handleStreamEnd (D:\ESPRIT\5eme\PFE\B2B-ERP-MERN-App\node_modules\axios\dist\node\axios.cjs:2944:11)
at IncomingMessage.emit (events.js:412:35)
at endReadableNT (internal/streams/readable.js:1317:12)
at processTicksAndRejections (internal/process/task_queues.js:82:21) {
code: 'ERR_BAD_REQUEST',
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [ 'xhr', 'http' ],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function], Blob: null },
validateStatus: [Function: validateStatus],
headers: AxiosHeaders {
Accept: 'application/json, text/plain, */*',
'User-Agent': 'axios/1.3.3',
'Accept-Encoding': 'gzip, compress, deflate, br'
},
auth: {
username: 'ahmed',
password: undefined,
workstation: 'DESKTOP-1EF91E4',
domain: undefined
},
method: 'get',
url: "http://desktop-1ef91e4:7048/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype] {
abort: [Function (anonymous)],
aborted: [Function (anonymous)],
connect: [Function (anonymous)],
error: [Function (anonymous)],
socket: [Function (anonymous)],
timeout: [Function (anonymous)],
prefinish: [Function: requestOnPrefinish]
},
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: Socket {
connecting: false,
_hadError: false,
_parent: null,
_host: 'desktop-1ef91e4',
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: null,
_server: null,
parser: null,
_httpMessage: [Circular *1],
[Symbol(async_id_symbol)]: 465,
[Symbol(kHandle)]: [TCP],
[Symbol(kSetNoDelay)]: false,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0,
[Symbol(RequestTimeout)]: undefined
},
_header: "GET /BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec HTTP/1.1\r\n" +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/1.3.3\r\n' +
'Accept-Encoding: gzip, compress, deflate, br\r\n' +
'Host: desktop-1ef91e4:7048\r\n' +
'Authorization: Basic YWhtZWQ6\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: [Object],
requests: {},
sockets: [Object],
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: "/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
_ended: true,
res: IncomingMessage {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 4,
_maxListeners: undefined,
socket: [Socket],
httpVersionMajor: 1,
httpVersionMinor: 1,
httpVersion: '1.1',
complete: true,
headers: [Object],
rawHeaders: [Array],
trailers: {},
rawTrailers: [],
aborted: false,
upgrade: false,
url: '',
method: null,
statusCode: 401,
statusMessage: 'Unauthorized',
client: [Socket],
_consuming: false,
_dumped: false,
req: [Circular *1],
responseUrl: "http://ahmed:#desktop-1ef91e4:7048/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
redirects: [],
[Symbol(kCapture)]: false,
[Symbol(RequestTimeout)]: undefined
},
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'desktop-1ef91e4',
protocol: 'http:',
_redirectable: Writable {
_writableState: [WritableState],
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
_options: [Object],
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 0,
_requestBodyBuffers: [],
_onNativeResponse: [Function (anonymous)],
_currentRequest: [Circular *1],
_currentUrl: "http://ahmed:#desktop-1ef91e4:7048/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
[Symbol(kCapture)]: false
},
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [Array],
'user-agent': [Array],
'accept-encoding': [Array],
host: [Array],
authorization: [Array]
}
},
response: {
status: 401,
statusText: 'Unauthorized',
headers: AxiosHeaders {
'content-length': '0',
server: 'Microsoft-HTTPAPI/2.0',
'www-authenticate': 'Negotiate',
date: 'Thu, 16 Feb 2023 11:49:51 GMT',
connection: 'close'
},
config: {
transitional: [Object],
adapter: [Array],
transformRequest: [Array],
transformResponse: [Array],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: [Object],
validateStatus: [Function: validateStatus],
headers: [AxiosHeaders],
auth: [Object],
method: 'get',
url: "http://desktop-1ef91e4:7048/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
data: undefined
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: false,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 0,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [Socket],
_header: "GET /BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec HTTP/1.1\r\n" +
'Accept: application/json, text/plain, */*\r\n' +
'User-Agent: axios/1.3.3\r\n' +
'Accept-Encoding: gzip, compress, deflate, br\r\n' +
'Host: desktop-1ef91e4:7048\r\n' +
'Authorization: Basic YWhtZWQ6\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
method: 'GET',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: "/BC210/ODataV4/Company('CRONUS%20France%20S.A.')/ItemListec",
_ended: true,
res: [IncomingMessage],
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'desktop-1ef91e4',
protocol: 'http:',
_redirectable: [Writable],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
data: ''
}
}
Error: Error retrieving articles data from Business Central
at getArticlesFromBC (D:\ESPRIT\5eme\PFE\B2B-ERP-MERN-App\routes\ArticleRoutes.js:44:11)
at processTicksAndRejections (internal/process/task_queues.js:95:5)
at async D:\ESPRIT\5eme\PFE\B2B-ERP-MERN-App\routes\ArticleRoutes.js:53:20
Even though it works perfectly in postman (NTLM authentication on Server Instance is enabled for that to happen)
express-ntlm seemed to authenticate the user but dosent follow up for some reason.
I switched to httpntlm and it worked perfectly, here is the new code in the node API file :
const express = require("express");
const router = express.Router();
const httpntlm = require("httpntlm");
// Get article data from Business Central
async function getArticlesFromBC() {
const username = process.env.USERNAME;
const password = process.env.PASSWORD;
const domain = process.env.DOMAIN;
const workstation = process.env.WORKSTATION;
const encodedCompanyId = encodeURIComponent("CRONUS France S.A.");
const url = `http://${process.env.SERVER}:7048/BC210/ODataV4/Company('${encodedCompanyId}')/ItemListec`;
const options = {
url: url,
username: username,
password: password,
workstation: workstation,
domain: domain
};
return new Promise((resolve, reject) => {
httpntlm.get(options, (err, res) => {
if (err) {
console.error(err);
reject(err);
} else {
resolve(res.body);
}
});
});
}
// Route to get article data
router.get("/", async (req, res) => {
try {
const articles = await getArticlesFromBC();
res.json(articles);
} catch (error) {
console.error("Error retrieving articles data from Business Central:", error.message);
res.status(500).send("Error retrieving articles data from Business Central");
}
});
module.exports = router;
So I've been trying to implement image upload to Backblaze B2 bucket for about three days now. I struggled with getting the data, but I'm getting an error about connection refusing I guess. It returns error code 504 and I've tried fixing it by sending just the image, without converting it to base64 data. I don't understand a lot about base64 data, so I think the error might be caused by base64Data. I also don't have a lot of experience with backblaze-b2 and found the npm package "backblaze-b2" not too long ago.
Here's the code:
const B2 = require('backblaze-b2');
const fs = require('fs');
export const uploadImage = async (req, res) => {
try {
const b2 = new B2({
accountId: process.env.BACKBLAZE_ACCOUNT_ID,
applicationKey: process.env.BACKBLAZE_APPLICATION_MASTER_KEY,
});
await b2.authorize();
// console.log(req.body);
const { image } = req.body;
if (!image) return res.status(400).send('No image found!');
// prepare the image
const base64Data = new Buffer.from(
image.replace(/^data:image\/\w+;base64,/,""),
'base64'
);
const handleImage = async () => {
try {
let uploadUrl = await b2.getUploadUrl({
bucketId: process.env.BACKBLAZE_BUCKET_ID,
});
// console.log('Is this not working?', uploadUrl);
const data = b2.uploadFile({
uploadUrl: uploadUrl.data.bucketId,
uploadAuthToken: uploadUrl.data.authorizationToken,
fileName: 'Pepe', //<-- TODO: Fix later
data: base64Data, // <-- Figure out what to pass in
onUploadProgress: (e) => null,
});
console.log(data);
res.send(data);
} catch (err) {
console.log('Bucket error or something: ', err);
}
};
handleImage();
} catch (err) {
console.log(err);
}
};
Here's the console:
Server is running on port 8000
DB CONNECTED
GET /api/csrf-token 200 2.732 ms - 52
GET /api/current-instructor 304 176.213 ms - -
Bucket error or something: Error: connect ECONNREFUSED 127.0.0.1:80
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1132:16) {
errno: -4078,
code: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 80,
config: {
url: 'deleted, for safety reasons',
method: 'post',
data: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 00 00 01 00 01 00 00 ff e2 02 28 49 43 43 5f 50 52 4f 46
49 4c 45 00 01 01 00 00 02 18 00 00 00 00 02 10 00 00 ... 71126 more bytes>,
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'b2/x-auto',
Authorization: 'deleted, for safety reasons',
'Content-Length': 71176,
'X-Bz-File-Name': 'Pepe',
'X-Bz-Content-Sha1': 'deleted, for safety reasons(possibly)',
'User-Agent': 'axios/0.21.4'
},
transformRequest: [ [Function (anonymous)] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
onUploadProgress: [Function: onUploadProgress],
maxContentLength: -1,
maxBodyLength: -1,
maxRedirects: 0,
validateStatus: [Function: validateStatus],
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
'axios-retry': { retryCount: 3, lastRequestTime: 1635347528124 }
},
request: <ref *1> ClientRequest {
_events: [Object: null prototype] {
response: [Function],
error: [Function: handleRequestError]
},
_eventsCount: 2,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: 71176,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
_closed: false,
socket: Socket {
connecting: false,
_hadError: true,
_parent: null,
_host: 'localhost',
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 8,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: false,
_sockname: null,
_pendingData: [Array],
_pendingEncoding: '',
server: null,
_server: null,
parser: null,
_httpMessage: [Circular *1],
[Symbol(async_id_symbol)]: 325,
[Symbol(kHandle)]: null,
[Symbol(kSetNoDelay)]: false,
[Symbol(lastWriteQueueSize)]: 0,
[Symbol(timeout)]: null,
[Symbol(kBuffer)]: null,
[Symbol(kBufferCb)]: null,
[Symbol(kBufferGen)]: null,
[Symbol(kCapture)]: false,
[Symbol(kBytesRead)]: 0,
[Symbol(kBytesWritten)]: 0
},
_header: 'POST deleted, for safety reasons HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: b2/x-auto\r\n' +
'Authorization: deleted, for safety reasons=\r\n' +
'Content-Length: 71176\r\n' +
'X-Bz-File-Name: Pepe\r\n' +
'X-Bz-Content-Sha1: deleted, for safety reasons\r\n' +
'User-Agent: axios/0.21.4\r\n' +
'Host: localhost\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: Agent {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: [Object: null prototype],
requests: [Object: null prototype] {},
sockets: [Object: null prototype],
freeSockets: [Object: null prototype] {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256,
scheduling: 'lifo',
maxTotalSockets: Infinity,
totalSocketCount: 1,
[Symbol(kCapture)]: false
},
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: 'deleted, for safety reasons',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'localhost',
protocol: 'http:',
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype] {
accept: [Array],
'content-type': [Array],
authorization: [Array],
'content-length': [Array],
'x-bz-file-name': [Array],
'x-bz-content-sha1': [Array],
'user-agent': [Array],
host: [Array]
}
},
response: undefined,
isAxiosError: true,
toJSON: [Function: toJSON]
}
Any help would be appreciated.
Thanks.
There are a few changes I made to get this working:
Use bodyParser.raw() to parse the body into the Buffer format that the backblaze-b2 library is expecting. It doesn't want base64.
Correct uploadUrl.data.bucketId in the call to b2.uploadFile to uploadUrl.data.uploadUrl.data.uploadUrl. This is what was causing the 'connection refused' error. Since the uploadUrl wasn't a URL, I'm guessing that b2.uploadFile assumed it was a path and you wanted to connect to localhost.
Await the response from b2.uploadFile.
Use response.data rather than just response to see the API response.
I built out your code into a runnable sample:
// I like to put my env vars in a .env file
const dotenv = require('dotenv');
dotenv.config();
const B2 = require('backblaze-b2');
const fs = require('fs');
const express = require('express')
const bodyParser = require('body-parser')
const app = express()
const port = process.env.PORT || 3000
const uploadImage = async (req, res) => {
try {
const b2 = new B2({
accountId: process.env.BACKBLAZE_ACCOUNT_ID,
applicationKey: process.env.BACKBLAZE_APPLICATION_MASTER_KEY,
});
await b2.authorize();
// console.log("req.body:", req.body);
if (!req.body) return res.status(400).send('No image found!');
const handleImage = async () => {
try {
let uploadUrl = await b2.getUploadUrl({
bucketId: process.env.BACKBLAZE_BUCKET_ID,
});
// Make the JSON more readable
console.log('getUploadUrl:', JSON.stringify(uploadUrl.data, undefined, 2));
// uploadFile returns a promise, so we need to await the response
const response = await b2.uploadFile({
uploadUrl: uploadUrl.data.uploadUrl,
uploadAuthToken: uploadUrl.data.authorizationToken,
fileName: 'Pepe', //<-- TODO: Fix later
data: req.body, // <-- This is the raw data as a buffer
onUploadProgress: (e) => null,
});
const prettyResponse = JSON.stringify(response.data, undefined, 2);
console.log('uploadFile: ', prettyResponse);
res.send(prettyResponse);
} catch (err) {
console.log('Bucket error or something: ', err);
}
};
handleImage();
} catch (err) {
console.log(err);
}
};
app.use(bodyParser.raw({ // Raw mode returns the posted body as a Buffer
type: '*/*' // Parse any mime type
}))
app.post('/', function (req, res) {
uploadImage(req, res)
})
app.listen(port, () => {
console.log(`Listening at http://localhost:${port}`)
})
Send a file with curl:
curl http://localhost:3000/ --data-binary #image.png
Console output (somewhat redacted!):
Listening at http://localhost:3000
getUploadUrl: {
"authorizationToken": "********",
"bucketId": "********",
"uploadUrl": "https://********.backblaze.com/b2api/v2/b2_upload_file/********"
}
uploadFile: {
"accountId": "********",
"action": "upload",
"bucketId": "********",
"contentLength": 3802,
"contentMd5": "d9b8b28f7fda3acfe7838ead41d8df38",
"contentSha1": "f8040f1068715160ef98ab98fde80f9214cb2845",
"contentType": "application/octet-stream",
"fileId": "********",
"fileInfo": {},
"fileName": "Pepe",
"fileRetention": {
"isClientAuthorizedToRead": true,
"value": {
"mode": null,
"retainUntilTimestamp": null
}
},
"legalHold": {
"isClientAuthorizedToRead": true,
"value": null
},
"serverSideEncryption": {
"algorithm": null,
"mode": null
},
"uploadTimestamp": 1641496698000
}
I'm using firebase functions to schedule a puppeteer task from a firebase pubsub namespace.
I also want to post the response back to an API endpoint outside google using axios.
I tried Axios directly from the worker but I get a 500 error.
My goal:
Use firebase schedule and pubsub namespace to generate puppeteer
jobs - complete
Post results back to external endpoint using axios - need your help :)
Code:
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const functions = require("firebase-functions");
const admin = require("firebase-admin");
const puppeteer = require('puppeteer');
const url = require('url');
// Request Data From A URL
var axios = require('axios');
var https = require('https');
// var cors = require("cors");
// Initalise App
admin.initializeApp();
const db = admin.firestore();
const workers = {
extract: async ({ uri, post_id, store_id, domain }) => {
let theTitle = null;
let thePrice = null;
let host = url.parse(uri).host;
let postId = post_id;
let storeId = store_id;
let theDomain = domain;
const SELECTORS = {
amazonPrice: '#priceblock_ourprice',
ebayPrice: '#prcIsum',
const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] });
console.log('spawning chrome headless');
console.log(postId);
console.log(storeId);
console.log(domain);
const page = await browser.newPage();
// Goto page and then do stuff
console.log('going to ', uri);
await page.goto(uri, {
waitUntil: ["domcontentloaded", "networkidle0"]
});
console.log('waiting for page to load ');
console.log(host);
// theTitle = await page.title();
try {
theTitle = await page.title();
// find amazon price
if (host === 'www.amazon.co.uk' || 'amazon.co.uk') {
const priceInput = await page.$(SELECTORS.amazonPrice)
thePrice = await page.evaluate(element => element.textContent, priceInput)
}
// find ebay price
if (host === 'www.ebay.co.uk' || 'ebay.co.uk') {
const priceInput = await page.$(SELECTORS.ebayPrice)
thePrice = await page.evaluate(element => element.value, priceInput)
}
else {
console.log('failed scrape at', host);
}
}
catch (error) {
console.error('There was an error processing the page:', error);
}
finally {
// close browser
if (browser !== null) {
await browser.close();
}
}
console.log(theTitle);
console.log(thePrice);
const response = {
title: theTitle,
price: thePrice,
};
console.log('post' + postId + storeId + thePrice + theDomain);
axios.post('endpoint', {
price: thePrice,
store_id: storeId,
post_id: postId,
domain: theDomain,
},
{
headers: {
'Content-Type': 'multipart/form-data',
}
})
.then(function (response) {
console.log(response);
})
.catch(function (error) {
console.log(error);
});
// axios post end
return response;
}
};
exports.taskRunner = functions.runWith({ memory: '2GB' }).pubsub
// export const taskRunner = functions.region('europe-west2').runWith( { memory: '2GB' }).pubsub
.schedule('*/15 * * * *').onRun(async (context) => {
// Consistent timestamp
const now = admin.firestore.Timestamp.now();
// Query all documents ready to perform
const query = db.collection('tasks').where('performAt', '<=', now).where('status', '==', 'scheduled');
const tasks = await query.get();
// Jobs to execute concurrently.
const jobs = [];
// Loop over documents and push job.
tasks.forEach(snapshot => {
const { worker, options } = snapshot.data();
const job = workers[worker](options)
// Update doc with status on success or error
// .then(() => snapshot.ref.update({ status: 'complete' }))
.catch((err) => snapshot.ref.update({ status: 'error' }));
jobs.push(job);
});
// Execute all jobs concurrently
return await Promise.all(jobs);
});
Error
Error: Request failed with status code 500
at createError (/srv/node_modules/axios/lib/core/createError.js:16:15)
at settle (/srv/node_modules/axios/lib/core/settle.js:17:12)
at IncomingMessage.handleStreamEnd (/srv/node_modules/axios/lib/adapters/http.js:236:11)
at emitNone (events.js:111:20)
at IncomingMessage.emit (events.js:208:7)
at endReadableNT (_stream_readable.js:1064:12)
at _combinedTickCallback (internal/process/next_tick.js:139:11)
at process._tickDomainCallback (internal/process/next_tick.js:219:9)
config:
{ url: 'endpoint',
method: 'post',
data: '{"£59.99":null,"store_id":32,"post_id":25,"domain":"amazon.co.uk"}',
headers:
{ Accept: 'application/json, text/plain, */*',
'Content-Type': 'multipart/form-data',
'User-Agent': 'axios/0.19.2',
'Content-Length': 65 },
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus] },
request:
ClientRequest {
domain:
Domain {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
members: [Array] },
_events:
{ socket: [Function],
abort: [Function],
aborted: [Function],
error: [Function],
timeout: [Function],
prefinish: [Function: requestOnPrefinish] },
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: true,
upgrading: false,
chunkedEncoding: false,
shouldKeepAlive: false,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket:
TLSSocket {
_tlsOptions: [Object],
_secureEstablished: true,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
_SNICallback: null,
servername: 'SERVERNAME',
npnProtocol: false,
alpnProtocol: false,
authorized: true,
authorizationError: null,
encrypted: true,
_events: [Object],
_eventsCount: 9,
connecting: false,
_hadError: false,
_handle: [Object],
_parent: null,
_host: 'HOST',
_readableState: [Object],
readable: true,
domain: [Object],
_maxListeners: undefined,
_writableState: [Object],
writable: false,
allowHalfOpen: false,
_bytesDispatched: 259,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: [Object],
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular],
[Symbol(asyncId)]: 538,
[Symbol(bytesRead)]: 0 },
connection:
TLSSocket {
_tlsOptions: [Object],
_secureEstablished: true,
_securePending: false,
_newSessionPending: false,
_controlReleased: true,
_SNICallback: null,
servername: 'HOST',
npnProtocol: false,
alpnProtocol: false,
authorized: true,
authorizationError: null,
encrypted: true,
_events: [Object],
_eventsCount: 9,
connecting: false,
_hadError: false,
_handle: [Object],
_parent: null,
_host: 'HOST',
_readableState: [Object],
readable: true,
domain: [Object],
_maxListeners: undefined,
_writableState: [Object],
writable: false,
allowHalfOpen: false,
_bytesDispatched: 259,
_sockname: null,
_pendingData: null,
_pendingEncoding: '',
server: undefined,
_server: null,
ssl: [Object],
_requestCert: true,
_rejectUnauthorized: true,
parser: null,
_httpMessage: [Circular],
[Symbol(asyncId)]: 538,
[Symbol(bytesRead)]: 0 },
_header: 'POST ENDPOINT HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: multipart/form-data\r\nUser-Agent: axios/0.19.2\r\nContent-Length: 65\r\nHost: HOST\r\nConnection: close\r\n\r\n',
_onPendingData: [Function: noopPendingOutput],
agent:
Agent {
domain: null,
_events: [Object],
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 443,
protocol: 'https:',
options: [Object],
requests: {},
sockets:
more_vert
Logs are subject to Cloud Logging's
After some work and trial & error. I settled on using the Request package to solve for the issue - https://www.npmjs.com/package/request
Not had time to investigate why Axios was not working but will review and if enlightened I will post here.
I tried to log axios errors, so I defined a custom error and logged using Winston.error.
However, response detail is not displaying. Weird thing is that console.log prints all details.
With console.log, I got this.
{ MyError:
at new MyError (src/axios-test.ts:16:15)
at src/axios-test.ts:29:15
at Generator.throw (<anonymous>)
at rejected (/src/axios-test.ts:6:65)
at process._tickCallback (internal/process/next_tick.js:68:7)
name: 'MyError',
myerror:
{ Error: Request failed with status code 500
at createError (/Users/woonggeunjang/dev/node_modules/axios/lib/core/createError.js:16:15)
at settle (/Users/dev/node_modules/axios/lib/core/settle.js:17:12)
at IncomingMessage.handleStreamEnd (/Users/dev/node_modules/axios/lib/adapters/http.js:236:11)
at IncomingMessage.emit (events.js:203:15)
at IncomingMessage.EventEmitter.emit (domain.js:448:20)
at endReadableNT (_stream_readable.js:1145:12)
at process._tickCallback (internal/process/next_tick.js:63:19)
config:
{ url: 'https://ptsv2.com/t/gcnad-1585724700/post',
method: 'post',
data: 'hmm',
headers: [Object],
transformRequest: [Array],
transformResponse: [Array],
timeout: 0,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
validateStatus: [Function: validateStatus] },
request:
ClientRequest {
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
output: [],
outputEncodings: [],
outputCallbacks: [],
outputSize: 0,
writable: true,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: true,
_headerSent: true,
socket: [TLSSocket],
connection: [TLSSocket],
_header:
'POST /t/gcnad-1585724700/post HTTP/1.1\r\nAccept: application/json, text/plain, */*\r\nContent-Type: application/x-www-form-urlencoded\r\nUser-Agent: axios/0.19.2\r\nContent-Length: 3\r\nHost: ptsv2.com\r\nConnection: close\r\n\r\n',
_onPendingData: [Function: noopPendingOutput],
agent: [Agent],
socketPath: undefined,
timeout: undefined,
method: 'POST',
path: '/t/gcnad-1585724700/post',
_ended: true,
res: [IncomingMessage],
aborted: undefined,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
_redirectable: [Writable],
[Symbol(isCorked)]: false,
[Symbol(outHeadersKey)]: [Object] },
response:
{ status: 500,
statusText: 'Internal Server Error',
headers: [Object],
config: [Object],
request: [ClientRequest],
data: 'Thank you for this dump. I hope you have a lovely day!' },
isAxiosError: true,
toJSON: [Function] } }
As for Winston.error(), I got this
{"name":"MyError","myerror":{"message":"Request failed with status code 500","name":"Error","stack":"Error: Request failed with status code 500\n at createError (/Users/woonggeunjang/dev/onda-cms/node_modules/axios/lib/core/createError.js:16:15)\n at settle (/Users/woonggeunjang/dev/onda-cms/node_modules/axios/lib/core/settle.js:17:12)\n at IncomingMessage.handleStreamEnd (/Users/woonggeunjang/dev/onda-cms/node_modules/axios/lib/adapters/http.js:236:11)\n at IncomingMessage.emit (events.js:203:15)\n at IncomingMessage.EventEmitter.emit (domain.js:448:20)\n at endReadableNT (_stream_readable.js:1145:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)","config":{"url":"https://ptsv2.com/t/gcnad-1585724700/post","method":"post","data":"hmm","headers":{"Accept":"application/json, text/plain, */*","Content-Type":"application/x-www-form-urlencoded","User-Agent":"axios/0.19.2","Content-Length":3},"transformRequest":[null],"transformResponse":[null],"timeout":0,"xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN","maxContentLength":-1}},"level":"error","message":"mm","stack":"MyError: \n at new MyError (/Users/woonggeunjang/dev/onda-cms/src/axios-test.ts:16:15)\n at /Users/woonggeunjang/dev/onda-cms/src/axios-test.ts:29:15\n at Generator.throw (<anonymous>)\n at rejected (/Users/woonggeunjang/dev/onda-cms/src/axios-test.ts:6:65)\n at process._tickCallback (internal/process/next_tick.js:68:7)","timestamp":"2020-04-01T12:43:45.530Z"}
This is my codes
export default class MyError extends Error {
myerror: string;
constructor(error: string) {
super();
Object.setPrototypeOf(this, new.target.prototype);
this.name = 'MyError';
this.myerror = error;
Error.captureStackTrace(this);
}
}
async function testPost () {
let res = null;
try {
res = await axios.post('https://ptsv2.com/t/gcnad-1585724700/post', 'hmm');
} catch (error) {
// throw new Error(error);
// console.log(error);
// Logger.error('hmm', error.stack);
throw(new MyError(error));
}
}
testPost().catch(error => {
// console.log(error);
Logger.error('mm', error );
});
Seems like some formatting is applied when using winston.
If I set prettyPrint format then error detail is displaying.
Updated
This is my Winston config
const consoleTransport = new transports.Console();
const Logger = createLogger({
transports: [consoleTransport],
format: format.combine(
format.timestamp(),
format.json(),
),
});
Thanks in advance.
I am trying to achieve the following, however as a bit of a newbie, I can't see what I'm doing wrong:
File upload triggers a cloud function (https.onRequest())
Resize the image 3 times (100x100 thumb, 500x500 thumb, resize original to limit it's size)
Save all 3 images to the firebase storage bucket (Google Storage)
Get the respective signed URLs of the 3 images
Save signed URLs to Firestore
The following code works, but not consistently:
In my index.js file I create the HTTP trigger by:
export const image = functions.https.onRequest(uploadImage);
That calls:
import * as admin from "firebase-admin";
import * as Busboy from "busboy";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
const cors = require("cors")({ origin: true });
const spawn = require("child-process-promise").spawn;
const gcconfig = {
projectId: "PROJECT_ID",
keyFilename: "key.json"
};
const gcs = require("#google-cloud/storage")(gcconfig);
export const uploadImage = (req, res) => {
cors(req, res, () => {
if (req.method === "POST") {
const busboy = new Busboy({ headers: req.headers });
const docRef = req.query.docRef;
const fieldRef = req.query.fieldRef;
const storageRef = req.query.storageRef;
let fileData = null;
// Max height and width of the thumbnail in pixels.
const THUMB_MAX_HEIGHT = 100;
const THUMB_MAX_WIDTH = 100;
const THUMB_PREFIX = "thumb_";
const RESIZED_MAX_HEIGHT = 500;
const RESIZED_MAX_WIDTH = 500;
const RESIZED_PREFIX = "resized_";
const ORIGINAL_MAX_HEIGHT = 1000;
const ORIGINAL_MAX_WIDTH = 1400;
// Listen for event when Busboy finds a file to stream.
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const tempLocalFile = path.join(os.tmpdir(), filename);
fileData = { file: filename, type: mimetype };
// Download file local tmp.
file.pipe(fs.createWriteStream(tempLocalFile));
});
busboy.on("finish", () => {
// File and directory paths.
const tempLocalFile = path.join(os.tmpdir(), fileData.file);
const tempLocalThumbFile = path.join(os.tmpdir(),`${THUMB_PREFIX}${fileData.file}`);
const tempLocalResizedFile = path.join(os.tmpdir(),`${RESIZED_PREFIX}${fileData.file}`);
const origFilePath = path.normalize(path.join(storageRef, fileData.file));
const thumbFilePath = path.normalize(path.join(storageRef, `${THUMB_PREFIX}${fileData.file}`));
const resizedFilePath = path.normalize(path.join(storageRef, `${RESIZED_PREFIX}${fileData.file}`));
// Cloud Storage files.
const bucket = gcs.bucket("PROJECT_ID.appspot.com");
const metadata = {
contentType: fileData.type,
"Cache-Control": "public,max-age=3600"
};
// Generate a thumbnail called tempLocalThumbFile
const promise = spawn(
"convert",
[
tempLocalFile,
"-thumbnail",
`${THUMB_MAX_WIDTH}x${THUMB_MAX_HEIGHT}>`,
tempLocalThumbFile
],
{ capture: ["stdout", "stderr"] }
);
return promise
.then(() => {
// Generate a resized_ called tempLocalResizedFile
return spawn(
"convert",
[
tempLocalFile,
"-thumbnail",
`${RESIZED_MAX_WIDTH}x${RESIZED_MAX_HEIGHT}>`,
tempLocalResizedFile
],
{ capture: ["stdout", "stderr"] }
);
})
.then(() => {
// Resize original to ensure it's not massive
return spawn(
"convert",
[
tempLocalFile,
"-thumbnail",
`${ORIGINAL_MAX_WIDTH}x${ORIGINAL_MAX_HEIGHT}>`,
tempLocalFile
],
{ capture: ["stdout", "stderr"] }
);
})
.then(() => {
// upload images to storage
const thumbUp = bucket.upload(tempLocalThumbFile, {
destination: thumbFilePath,
metadata: metadata
});
const resizeUp = bucket.upload(tempLocalResizedFile, {
destination: resizedFilePath,
metadata: metadata
});
const origUp = bucket.upload(tempLocalFile, {
destination: origFilePath,
metadata: metadata
});
return Promise.all([thumbUp, resizeUp, origUp]);
})
.then(() => {
// Once the image has been uploaded delete the local files to free up disk space.
fs.unlinkSync(tempLocalFile);
fs.unlinkSync(tempLocalThumbFile);
fs.unlinkSync(tempLocalResizedFile);
const config = {
action: "read",
expires: "03-01-2500"
};
// Get the Signed URLs for the thumbnail and original image.
return Promise.all([
bucket.file(thumbFilePath).getSignedUrl(config),
bucket.file(resizedFilePath).getSignedUrl(config),
bucket.file(origFilePath).getSignedUrl(config)
]);
})
.then(results => {
console.log("Got Signed URLs.", results);
const thumbFileUrl = results[0][0];
const resizeFileUrl = results[1][0];
const origFileUrl = results[2][0];
// Add the URLs to the Database
return admin
.firestore()
.doc(docRef)
.update({
[fieldRef + ".original"]: origFileUrl,
[fieldRef + ".resized"]: resizeFileUrl,
[fieldRef + ".thumb"]: thumbFileUrl
});
})
.then(() => {
console.log("Image URLs saved to Firestore.");
res.status(200).send({
message: "File Saved."
});
})
.catch(err => {
console.log("Error:", err);
res.status(500).send({
error: err
});
});
});
busboy.end(req.rawBody);
} else {
return res.status(500).json({
message: "Not Allowed"
});
}
});
};
The error message I receive when it fails is:
Error: { ChildProcessError: convert /tmp/ff658860-cc0f-11e8-bd7d-178b6a853dfe.png -thumbnail 100x100> /tmp/thumb_ff658860-cc0f-11e8-bd7d-178b6a853dfe.png failed with code 1 at ChildProcess. (/user_code/node_modules/child-process-promise/lib/index.js:132:23) at emitTwo (events.js:106:13) at ChildProcess.emit (events.js:191:7) at maybeClose (internal/child_process.js:920:16) at Socket. (internal/child_process.js:351:11) at emitOne (events.js:96:13) at Socket.emit (events.js:188:7) at Pipe._handle.close [as _onclose] (net.js:509:12) name: 'ChildProcessError', code: 1, childProcess: ChildProcess { domain: Domain { domain: null, _events: [Object], _eventsCount: 1, _maxListeners: undefined, members: [] }, _events: { error: [Function: t], close: [Function] }, _eventsCount: 2, _maxListeners: undefined, _closesNeeded: 3, _closesGot: 3, connected: false, signalCode: null, exitCode: 1, killed: false, spawnfile: 'convert', _handle: null, spawnargs: [ 'convert', '/tmp/ff658860-cc0f-11e8-bd7d-178b6a853dfe.png', '-thumbnail', '100x100>', '/tmp/thumb_ff658860-cc0f-11e8-bd7d-178b6a853dfe.png' ], pid: 12, stdin: Socket { connecting: false, _hadError: false, _handle: null, _parent: null, _host: null, _readableState: [Object], readable: false, domain: [Object], _events: [Object], _eventsCount: 2, _maxListeners: undefined, _writableState: [Object], writable: false, allowHalfOpen: false, destroyed: true, _bytesDispatched: 0, _sockname: null, _pendingData: null, _pendingEncoding: '', server: null, _server: null, write: [Function: writeAfterFIN], _idleNext: null, _idlePrev: null, _idleTimeout: -1 }, stdout: Socket { connecting: false, _hadError: false, _handle: null, _parent: null, _host: null, _readableState: [Object], readable: false, domain: [Object], _events: [Object], _eventsCount: 4, _maxListeners: undefined, _writableState: [Object], writable: false, allowHalfOpen: false, destroyed: true, _bytesDispatched: 0, _sockname: null, _pendingData: null, _pendingEncoding: '', server: null, _server: null, read: [Function], _consuming: true, _idleNext: null, _idlePrev: null, _idleTimeout: -1, write: [Function: writeAfterFIN] }, stderr: Socket { connecting: false, _hadError: false, _handle: null, _parent: null, _host: null, _readableState: [Object], readable: false, domain: [Object], _events: [Object], _eventsCount: 4, _maxListeners: undefined, _writableState: [Object], writable: false, allowHalfOpen: false, destroyed: true, _bytesDispatched: 0, _sockname: null, _pendingData: null, _pendingEncoding: '', server: null, _server: null, read: [Function], _consuming: true, _idleNext: null, _idlePrev: null, _idleTimeout: -1, write: [Function: writeAfterFIN] }, stdio: [ [Object], [Object], [Object] ] }, stdout: '', stderr: 'convert: improper image header /tmp/ff658860-cc0f-11e8-bd7d-178b6a853dfe.png\' # error/png.c/ReadPNGImage/3927.\nconvert: no images defined/tmp/thumb_ff658860-cc0f-11e8-bd7d-178b6a853dfe.png\' # error/convert.c/ConvertImageCommand/3210.\n' }
I'm not familiar with busboy but believe I'm ending the process correctly, but as this typically only works for the first upload after saving the cloud function, I suspect the code may not be ending cleanly?
Any help is greatly appreciated. Thank you.