cloudant couchdb multipart insert throwing function_clause error - node.js

I am trying to insert a media document to cloudant couchdb using below below code.
var uuid = require('uuid').v4;
fs.readFile('./test.jpeg', function(err, data) {
if (!err) {
var newAttachmentObj = {
type: 'media',
media_mime: 'image/jpeg',
media_type: 'Photo',
media_filename: 'rabbit1'
}
var filename = 'rabbit1';
var media_mime = 'image/jpeg';
var attachment_id = uuid();
var media_data = data;
console.log(data);
console.log(newAttachmentObj);
console.log(attachment_id);
db.multipart.insert(newAttachmentObj,
[{ name: filename, data: media_data, content_type: media_mime }], attachment_id, function (err, body) {
console.log(body);
if (err) {
console.log('Error: Creating a media doc in cloudant.');
console.log(err);
// console.log(JSON.stringify(err));
} else {
console.log(body);
}
})
}
});
Document is getting created in the cloudant couchdb and also we can view the uploaded attachment, but the callback function returns an error as shown below.
{ Error: function_clause
at Request._callback (/home/boatman/anoop/forwarding-module/node_modules/cloudant-nano/lib/nano.js:248:15)
at Request.self.callback (/home/boatman/anoop/forwarding-module/node_modules/request/request.js:188:22)
at emitTwo (events.js:125:13)
at Request.emit (events.js:213:7)
at Request.<anonymous> (/home/boatman/anoop/forwarding-module/node_modules/request/request.js:1171:10)
at emitOne (events.js:115:13)
at Request.emit (events.js:210:7)
at IncomingMessage.<anonymous> (/home/boatman/anoop/forwarding-module/node_modules/request/request.js:1091:12)
at Object.onceWrapper (events.js:314:30)
at emitNone (events.js:110:20)
at IncomingMessage.emit (events.js:207:7)
at endReadableNT (_stream_readable.js:1045:12)
at _combinedTickCallback (internal/process/next_tick.js:138:11)
at process._tickDomainCallback (internal/process/next_tick.js:218:9)
name: 'Error',
error: 'unknown_error',
reason: 'function_clause',
ref: 944644368,
scope: 'couch',
statusCode: 500,
request:
{ method: 'PUT',
headers: { 'content-type': 'multipart/related' },
uri: 'https://XXXXXX:XXXXXX#account_id-bluemix.cloudant.com/db_media/a73d3788-d661-4944-964b-bcffce0286bd',
multipart: [ [Object], [Object] ] },
headers:
{ 'cache-control': 'must-revalidate',
'content-type': 'application/json',
date: 'Tue, 17 Apr 2018 08:37:28 GMT',
'x-couch-request-id': '5097b3e876',
'x-couch-stack-hash': '944644368',
'x-frame-options': 'DENY',
'strict-transport-security': 'max-age=31536000',
'x-content-type-options': 'nosniff',
'x-cloudant-request-class': 'write',
'x-cloudant-backend': 'bm-cc-uk-04',
via: '1.1 lb1.bm-cc-uk-04 (Glum/1.50.4)',
statusCode: 500,
uri: 'https://XXXXXX:XXXXXX#account_id-bluemix.cloudant.com/db_media/a73d3788-d661-4944-964b-bcffce0286bd' },
errid: 'non_200',
description: 'couch returned 500' }
Please find the image below.

I tried to reproduce your conditions with CouchDB 2.1.1, not Cloudant, therefore it may not be an exact reproduction, but I thought I'd share my results.
I created the following server.js file based on your code with a little modification. I used package nano for CouchDB not cloudant-nano since I'm not using Cloudant:
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; // Ignore rejection, becasue CouchDB SSL certificate is self-signed
var uuid = require('uuid').v4;
const fs = require('fs')
const nano = require('nano')('https://admin:****#192.168.1.106:6984');
// Database 'reproduce' is already created in CouchDB 2.1.1
const db = nano.db.use('reproduce');
fs.readFile('./test.jpeg', function(err, data) {
if(err){
console.log('err -> ', err)
}else if (!err) {
var newAttachmentObj = {
type: 'media',
media_mime: 'image/jpeg',
media_type: 'Photo',
media_filename: 'rabbit1'
}
var filename = 'rabbit1';
var media_mime = 'image/jpeg';
var attachment_id = uuid();
var media_data = data;
console.log(data);
console.log(newAttachmentObj);
console.log(attachment_id);
db.multipart.insert(newAttachmentObj,
[{ name: filename, data: media_data, content_type: media_mime }], attachment_id, function (err, body) {
console.log(body);
if (err) {
console.log('Error: Creating a media doc in cloudant.');
console.log(err);
// console.log(JSON.stringify(err));
} else {
console.log(body);
}
})
}
});
I placed a sample test.jpeg in the same directory as my server.js file. Then I ran the code with command $ node server.js and I got the following results:
$ node server.js
<Buffer ff d8 ff e1 00 32 45 78 69 66 00 00 49 49 2a 00 08 00 00 00 01 00 98 82 02 00 0e 00 00 00 1a 00 00 00 00 00 00 00 52 61 77 70 69 78 65 6c 20 4c 74 64 ... >
{ type: 'media',
media_mime: 'image/jpeg',
media_type: 'Photo',
media_filename: 'rabbit1' }
ec6a36d1-952e-4d86-9865-3587c6079fb5
{ ok: true,
id: 'ec6a36d1-952e-4d86-9865-3587c6079fb5',
rev: '1-896eca9e9980509aeaa8539b281c3257' }
{ ok: true,
id: 'ec6a36d1-952e-4d86-9865-3587c6079fb5',
rev: '1-896eca9e9980509aeaa8539b281c3257' }
Obviously, I don't receive the errors your getting.

Related

Uploading from AWS Lambda (Nodejs Express with multer) to S3 Bucket, returning white small box

I'm trying to upload from my lambda (nodejs express) to s3 bucket. But whenever I upload, and look for my uploaded file in S3, it only shows a white small box. 1
I already try converting my file buffer to base64, but it still not working.
My uploaded file only show if I upload it using my local api(localhost).
Here's my code:
// multer middleware
const multer = require("multer");
const helpers = require("../helpers/image-upload-helper");
const storage =multer.memoryStorage();
let upload = multer({
storage: storage,
fileFilter: helpers.imageFilter,
}).any();
//controller
try {
if(req.files){
for (const file of req.files) {
const ImageName = randomImageNameGenerator.randomImageName()
const params = {
Bucket: process.env.BUCKET_NAME,
Key: ImageName,
Body: file.buffer,
ContentType : file.mimetype,
}
const command = new PutObjectCommand(params)
const myData = await s3.send(command)
}
}
//log of my command
PutObjectCommand {
middlewareStack: {
add: [Function: add],
addRelativeTo: [Function: addRelativeTo],
clone: [Function: clone],
use: [Function: use],
remove: [Function: remove],
removeByTag: [Function: removeByTag],
concat: [Function: concat],
applyToStack: [Function: cloneTo],
identify: [Function: identify],
resolve: [Function: resolve]
},
input: {
Bucket: 'orex-product-images',
Key: 'b465138efab90aba02e5376ef247f536cfb1e7e32e34877bf21ab1bd655b3749',
Body: <Buffer 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 01 f8 00 00 00 f5 08 06 00 00 00 bc c1 e7 15 00 00 00 01 73 52 47 42 00 ae ce 1c e9 00 00 20 00 ... 10640 more bytes>,
ContentType: 'image/png'
}
}
// log of myData
{
'$metadata': {
httpStatusCode: 200,
requestId: '6C1EM009PP420NRK',
extendedRequestId: 'ZfGR4AR4mElYOSGes68YqEegptyO5PY5iPCvplP89wr1nqT4DZHwo0D0bl5qyZ7aAB0HaDaTAKU=',
cfId: undefined,
attempts: 1,
totalRetryDelay: 0
},
ETag: '"96425366df243451e35a941524b2a019a6ad2b"',
ServerSideEncryption: 'ABDS256',
VersionId: 'rpgj.L5AwGNCcKVzatIY5zHf_SYVNWt0'
}
Note: I didn't see any error in my cloud watch
1 Example of what the white box looks like
For those that arrived here as I did with the same issue, this answer solved it for me:
Using Lambda to get image from S3 returns a white box in Python
And for those using serverless, this is also relevant:
Serverless I image upload to S3 broken after deploy, local worked only

Javascript on client side: how to upload a zip file to AWS S3 bucket through pre-signed URL?

My web allows user to drag and drop a zip file and upload it to AWS S3 bucket. The steps are:
User drag-and-drops a zip file to the drop-zone of the UI;
User clicks send;
A request is made to AWS Lambda function and the function will generate a pre-signed URL that allows the user to upload any file.
An axios PUT request is made to the pre-signed S3 URL to upload the file.
I used local node.js code to test the pre-signed S3 URL:
const fileToUpload = fs.readFileSync(test_file_path);
console.log("fileToUpload: type: ", typeof fileToUpload, ", content: ", fileToUpload);
try {
const uploadResponse = await axios({
method: 'PUT',
url: presignedUrl,
data: fileToUpload,
headers: {
'Content-Type': '',
},
maxContentLength: Infinity,
maxBodyLength: Infinity
});
return uploadResponse.data;
} catch (error) {
console.error('Error while uploading object to S3:', error.message);
}
And it works well, which proves that the generated pre-signed URL is valid.
However, on client side Reactjs:
console.log(`formState.file: type: ${formState.file}, content: ${formState.file}`);
const uploadResponse = await axios({
method: 'PUT',
url: presignedS3Url,
data: formState.file,
headers: {
'Content-Type': ''
},
maxContentLength: Infinity,
maxBodyLength: Infinity
});
It fails and the request ends in a 403 forbidden error.
The difference is that in nodejs code, the fileToUpload is:
type: object, content: <Buffer 50 4b 03 04 14 00 08 00 08 00 78 84 cb 50 00 00 00 00 00 00 00 00 24 ae 12 01 3e 00 20 00 31 2e 32 2e 38 34 30 2e 31 31 33 35 36 34 2e 31 30 2e 31 2e ... 10573784 more bytes>
Whereas in client side, the formState.file is initialized by react-dropzone lib and has the type: formState.file: type: [object File] and its content is:
path: "1.2.840.113564.10.1.312260962047571814316520322884140128208155.zip"
lastModified: 1625164188712
lastModifiedDate: Fri Jul 02 2021 03:29:48 GMT+0900 (Japan Standard Time) {}
name: "1.2.840.113564.10.1.312260962047571814316520322884140128208155.zip"
size: 10573834
type: "application/zip"
webkitRelativePath: ""
[[Prototype]]: File
length: 1
I am not entirely sure that this is the cause. A few thoughts:
fs.readFileSync() is nodejs only, and it is not available in client side Reactjs.
On client side, should I get the zip file in the form of <Buffer ....> and how should I do it?
Or is it ok with the current [object File] type on client side? maybe there is another way to upload it to S3 bucket?

Nodejs: validation failed: img: Cast to Buffer failed for value

I have tried to save the image in mongoose. I have followed this tutorial. My dataSchema is look like below,
const dataSchema = new mongoose.Schema({
name: {
type: String,
required: false,
},
img: {
type: Buffer,
contentType: String,
required: false,
}
});
I have created a separate folder for routers. Here is my routers.js file,
//Define storage for multer
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "uploads");
},
filename: (req, file, cb) => {
const uniqueSuffix = Date.now() + "-" + Math.round(Math.random() * 1e9);
cb(null, file.fieldname + "-" + uniqueSuffix);
},
});
const upload = multer({ storage: storage });
// Post request
router.post("/", upload.single("img"), async (req, res) => {
const data = new Data({
name: req.body.name,
img: {
data: fs.readFileSync(
path.join(__dirname, "../uploads/" + req.file.filename)
),
contentType: "image/png",
},
});
await Data.create(data);
res.send(req.body);
});
I have tried without img field, It was working fine. But with the image field, it is given the following error. I am testing this API in Postman.
(node:476) UnhandledPromiseRejectionWarning: ValidationError: Data validation failed: img: Cast to Buffer failed for value " {
data: <Buffer 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 01 f0 00 00 01 0a 08 02 00 00 00 62 c7 90 ce 00 00 00 01 73 52 47 42 00 ae ce 1c e9 00 00 00 04 ... 22098 more bytes>,
contentType: 'image/png'
}" (type Object) at path "img"
at model.Document.invalidate (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\document.js:2879:32)
at model.$set (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\document.js:1426:12) at model.$set (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\document.js:1128:16) at model.Document (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\document.js:148:12)
at model.Model (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\model.js:106:12)
at new model (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\mongoose\lib\model.js:4752:15)
at C:\Users\gic\Downloads\FloodAPI\MEN API\routers\flood.js:51:17
at Layer.handle [as handle_request] (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\express\lib\router\layer.js:95:5)
at next (C:\Users\gic\Downloads\FloodAPI\MEN API\node_modules\express\lib\router\route.js:137:13)
at Immediate.<anonymous> (C:\Users\gic\Downloads\FloodAPI\node_modules\multer\lib\make-middleware.js:53:37)
(node:476) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection
id: 1)
(node:476) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
I used Django before. It was very easy to handle the files. Nodejs seems very complicated for me. Any detailed explanation would be highly appreciated. Please help me to solve this issue.

Firebase file upload with "express-fileupload" issue

trying to upload files using "firebase-admin" and "express-fileupload" , all firebase and express-fileupload configurations are setup as well :
my requires libs:
const firebaseAdmin = require("firebase-admin");
const fileUpload = require('express-fileupload');
my express-fileupload configs:
app.use(fileUpload({ createParentPath: true }))
app.use('/resources', express.static(path.resolve(__dirname, '../uploads')));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
i tested express-fileuploa and it works fine:
debug for req.files.file:
{
name: '66032925_2210721569219339_1152747532661555200_n.jpg',
data: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 02 00 00 01 00 01 00 00 ff ed 00 9c 50 68 6f 74 6f 73 68 6f 70 20 33 2e 30 00 38 42 49 4d 04 04 00 00 00 00 00 80 ... 23648 more bytes>,
size: 23698,
encoding: '7bit',
tempFilePath: '',
truncated: false,
mimetype: 'image/jpeg',
md5: '41145e65bb895e594e4657c9f6ce90a3',
mv: [Function: mv]
}
ok , now lets see my firebase configs:
var serviceAccount = require("../serviceAccountKey.json");
firebaseAdmin.initializeApp({
credential: firebaseAdmin.credential.cert(serviceAccount),
storageBucket: "*****-*****.appspot.com"
});
the code bellow is the post callback upload endpoint , there i am working only to upload a myfile .
app.post("/upload", (req, res) => {
try {
var file = req.files.file;
console.debug(file)
var bucket = firebaseAdmin.storage().bucket();
bucket.upload(file.data, { destination: "clients/avatars" }).then(data => {
console.debug("data", data);
return res.json({ message: "data", data: data });
}, error => {
console.debug("really ???", error)
return res.json({ message: "error", data: error + "" });
})
} catch (err) {
return res.json({ message: "error", data: err + "" });
}
});
now , when i test it , i get this error :
"TypeError [ERR_INVALID_ARG_VALUE]: The argument 'path' must be a
string or Uint8Array without null bytes. Received <Buffer ff d8 ff e0
00 10 4a 46 49 46 00 01 02 00 00 01 00 01 00 00 ff ed 00 9c 50 68 6f
74 6f 73 68 6f 70 20 33 2e 30 00 38 42 ..."
i debug console i see the error is throwing from ***
bucket.upload(file.data, { destination: "clients/avatars" })
*** method call.any help with ?
the reason for this is that you are passing the file data (as buffer) to a function that expects the file path.
bucket.upload(filePath, {
destination: remoteFile,
uploadType: "media",
metadata: {
contentType: fileMime,
metadata: {
firebaseStorageDownloadTokens: uuid
}
}
})
refer complete code here: Upload files to Firebase Storage using Node.js

Get all assets from RefTab API (Format RefTab API Response)

I'm trying to use RefTab's API to pull custom reports, but I'm not sure how to pull the data I want from the response.
I think it might have something to do with unzipping a gunzip?
I've tried accessing res.headers and res.body, but it's not exactly what I'm looking for.
I can't seem to find anything anywhere on RefTab specifically.
I'm not experienced in using REST APIs so I don't know if this is specific to RefTab
Any help would be greatly appreciated!
https://reftab.com
https://reftab.com/api-docs
My index.js file:
const btoa = require('btoa');
const fetch = require('node-fetch');
const CryptoJS = require('crypto-js');
//CryptoJS is needed for the md5 and HmacSHA256 methods
function signRequest(request) {
const publicKey = 'PUBLIC_KEY_HERE';
const secretKey = 'SECRET_KEY_HERE';
const body = request.body;
const method = request.method;
const url = request.url;
const now = new Date().toUTCString();
let contentMD5 = '';
let contentType = '';
if (body !== undefined) {
contentMD5 = CryptoJS.md5(body).toString();
contentType = 'application/json';
}
let signatureToSign = method + '\n' +
contentMD5 + '\n' +
contentType + '\n' +
now + '\n' +
url;
signatureToSign = unescape(encodeURIComponent(signatureToSign));
const token = btoa(CryptoJS.HmacSHA256(signatureToSign, secretKey));
const signature = 'RT ' + publicKey + ':' + token;
request.headers = {};
request.headers.Authorization = signature;
request.headers['x-rt-date'] = now;
return request;
}
//expected input object
const options = {
method: 'GET',
url: 'https://www.reftab.com/api/assets'
};
fetch('https://www.reftab.com/api/assets', signRequest(options))
.then((res) => {
console.log(res);
});
The response:
Response {
size: 0,
timeout: 0,
[Symbol(Body internals)]: {
body: Gunzip {
_writeState: [Uint32Array],
_readableState: [ReadableState],
readable: true,
_events: [Object: null prototype],
_eventsCount: 6,
_maxListeners: undefined,
_writableState: [WritableState],
writable: true,
allowHalfOpen: true,
_transformState: [Object],
_hadError: false,
bytesWritten: 0,
_handle: [Zlib],
_outBuffer: <Buffer a0 ac 36 be 49 02 00 00 e0 46 33 be 49 02 00 00 6e 5b b1 6b 01 09 00 00 8c 21 00 00 36 09 00 00 30 87 34 be 49 02 00 00 f0 7d 34 be 49 02 00 00 62 69 ... 16334 more bytes>,
_outOffset: 0,
_chunkSize: 16384,
_defaultFlushFlag: 2,
_finishFlushFlag: 2,
_defaultFullFlushFlag: 3,
_info: undefined,
_level: -1,
_strategy: 0,
[Symbol(kCapture)]: false
},
disturbed: false,
error: null
},
[Symbol(Response internals)]: {
url: 'https://www.reftab.com/api/assets',
status: 200,
statusText: 'OK',
headers: Headers { [Symbol(map)]: [Object: null prototype] },
counter: 0
}
}
I changed:
fetch('https://www.reftab.com/api/assets', signRequest(options))
.then((res) => {
console.log(res);
});
To:
fetch('https://www.reftab.com/api/assets', signRequest(options))
.then((res) => res.json())
.then(json => console.log(json))
And it worked!

Resources