I want to upload an image to S3 and save to user record with NodeJS, just like the Rails Paperclip gem.
I believe this should be the process, but again I'm quite confused about how this package should work:
receive an image and resize by paperclip
save or update to S3
save file to user in DB
I have a Rails Postgres database, and users can upload an image, stored in S3, and reformatted with Paperclip gem. Here's how it is stored:
irb(main):003:0> user.avatar
=> #<Paperclip::Attachment:0x000055b3e043aa50 #name=:avatar,
#name_string="avatar", #instance=#<User id: 1, email:
"example#gmail.com", created_at: "2016-06-11 22:52:36",
updated_at: "2019-06-16 17:17:16", first_name: "Clarissa",
last_name: "Jones", avatar_file_name: "two_people_talking.gif",
avatar_content_type: "image/gif", avatar_file_size: 373197,
avatar_updated_at: "2019-06-16 17:17:12", #options={:convert_options=>{},
:default_style=>:original, :default_url=>":style/missing.png",
:escape_url=>true, :restricted_characters=>/[&$+,\/:;=?#<>\[\]\
{\}\|\\\^~%# ]/, :filename_cleaner=>nil,
:hash_data=>":class/:attachment/:id/:style/:updated_at",
:hash_digest=>"SHA1", :interpolator=>Paperclip::Interpolations,
:only_process=>[],
:path=>"/:class/:attachment/:id_partition/:style/:filename",
:preserve_files=>false, :processors=>[:thumbnail],
:source_file_options=>{:all=>"-auto-orient"}, :storage=>:s3,
:styles=>{:large=>"500x500#", :medium=>"200x200#",
:thumb=>"100x100#"}, :url=>":s3_path_url",
:url_generator=>Paperclip::UrlGenerator,
:use_default_time_zone=>true, :use_timestamp=>true, :whiny=>true,
:validate_media_type=>true, :adapter_options=>
{:hash_digest=>Digest::MD5},
:check_validity_before_processing=>true, :s3_host_name=>"s3-us-
west-2.amazonaws.com", :s3_protocol=>"https", :s3_credentials=>
{:bucket=>"example", :access_key_id=>"REDACTED",
:secret_access_key=>"REDACTED",
:s3_region=>"us-west-2"}}, #post_processing=true,
#queued_for_delete=[], #queued_for_write={}, #errors={},
#dirty=false, #interpolator=Paperclip::Interpolations,
#url_generator=#<Paperclip::UrlGenerator:0x000055b3e043a8e8
#attachment=#<Paperclip::Attachment:0x000055b3e043aa50 ...>>,
#source_file_options={:all=>"-auto-orient"}, #whiny=true,
#s3_options={}, #s3_permissions={:default=>:"public-read"},
#s3_protocol="https", #s3_metadata={}, #s3_headers={},
#s3_storage_class={:default=>nil},
#s3_server_side_encryption=false, #http_proxy=nil,
#use_accelerate_endpoint=nil>
user.avatar(:thumb) returns:
https://s3-us-west-2.amazonaws.com/example/users/avatars/000/000/001/thumb/two_people_talking.gif?1560705432
Now, I'm trying to allow the user to upload a new/change image through a react-native app, and the backend is Nodejs, which is relatively new to me.
I'm so confused about how to implement this, especially because the examples are all referencing Mongoose, which I'm not using.
Just to show how I'd successfully update the user, here is how to update first_name of the user:
users.updateUserPhoto = (req, res) => {
let id = req.decoded.id
let first_name = req.body.first_name
models.Users.update(
first_name: first_name,
{
where: {
id: req.decoded.id
}
},
).then(response => {
res.status(200).json({ status: 200, data: { response } });
})
.catch(error => {
res.status(500).json({ status: 500, err: error });
})
}
Here is the package I found node-paperclip-s3, and here's what I'm trying to do:
'use strict'
let users = {};
const { Users } = require('../models');
let models = require("../models/index");
let Sequelize = require('sequelize');
let Paperclip = require('node-paperclip');
let Op = Sequelize.Op;
let sequelizeDB = require('../modules/Sequelize');
users.updateUserPhoto = (req, res) => {
let id = req.decoded.id
let avatar = req.body.avatar <- this is a file path
models.Users.plugin(Paperclip.plugins, {
avatar: {
styles: [
{ original: true },
{ large: { width: 500, height: 500 } },
{ medium: { width: 200, height: 200 } },
{ thumb: { width: 100, height: 100 } }
],
prefix: '/users/{{attachment}}/{{id}}/{{filename}}',
name_format: '{{style}}.{{extension}}',
storage: 's3',
s3: {
bucket: process.env.S3_BUCKET_NAME,
region: 'us-west-2',
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
}
}
})
models.Users.update(
avatar,
{
where: {
id: req.decoded.id
}
},
).then(response => {
res.status(200).json({ status: 200, data: { response } });
})
.catch(error => {
res.status(500).json({ status: 500, err: error });
})
}
I've also tried something like this:
models.Users.update(Paperclip.plugins, {
avatar: {
styles: [
{ original: true },
{ large: { width: 500, height: 500 } },
{ medium: { width: 200, height: 200 } },
{ thumb: { width: 100, height: 100 } }
],
prefix: '/users/{{attachment}}/{{id}}/{{filename}}',
name_format: '{{style}}.{{extension}}',
storage: 's3',
s3: {
bucket: process.env.S3_BUCKET_NAME,
region: 'us-west-2',
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
}
},
{
where: {
id: req.decoded.id
}
},
).then(response => {
res.status(200).json({ status: 200, data: { response } });
})
.catch(error => {
res.status(500).json({ status: 500, err: error });
})
})
I've tried:
let new_avatar = (Paperclip.plugins, {
avatar: {
styles: [
{ original: true },
{ large: { width: 500, height: 500 } },
{ medium: { width: 200, height: 200 } },
{ thumb: { width: 100, height: 100 } }
],
prefix: `/users/avatars/{{attachment}}/{{id}}/{{filename}}`,
name_format: '{{style}}.{{extension}}',
storage: 's3',
s3: {
bucket: process.env.S3_BUCKET_NAME,
region: 'us-west-2',
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
}
},
})
let data = {
avatar: new_avatar
}
models.Users.update(
data,
{
where: {
id: req.decoded.id
}
},
).then(response => {
res.status(200).json({ status: 200, data: { response } });
})
.catch(error => {
res.status(500).json({ status: 500, err: error });
})
From the example in the link above, I don't understand how it is saving to S3, or how it's updating the database in the same way the Rails gem is creating that record.
Question : how to save resized images + original in the exact same way that the Rails paperclip gem is saving to S3 AND the user record in the database.
I originally had this open for a 400 point bounty, and am more than happy to still offer 400 points to anyone who can help me solve this. Thanks!!
The below code is for nodeJs.
I have added an api to save an image from frontend to AWS S3.
I have added comments within code for better understanding.
var express = require("express");
var router = express.Router();
var aws = require('aws-sdk');
aws.config.update({
secretAccessKey: config.AwsS3SecretAccessKey,
accessKeyId: config.AwsS3AccessKeyId,
region: config.AwsS3Region
});
router
.route("/uploadImage")
.post(function (req, res) {
//req.files.imageFile contains the file from client, modify it as per you requirement
var file = getDesiredFileFromPaperclip(req.files.imageFile);
const fileName = new Date().getTime() + file.name;
//before uploading, we need to create an instance of client file
file.mv(fileName, (movErr, movedFile) => {
if (movErr) {
console.log(movErr);
res.send(400);
return;
}
//read file data
fs.readFile(fileName, (err, data) => {
if (err) {
console.error(err)
res.send(400);
}
else {
//as we have byte data of file, delete the file instance
try {
fs.unlink(fileName);
} catch (error) {
console.error(error);
}
//now, configure aws
var s3 = new aws.S3();
const params = {
Bucket: config.AwsS3BucketName, // pass your bucket name
Key: fileName, // file will be saved as bucket_name/file.ext
Body: data
}
//upload file
s3.upload(params, function (s3Err, awsFileData) {
if (s3Err) {
console.error(s3Err)
res.send(400);
} else {
console.log(`File uploaded successfully at ${awsFileData.Location}`)
//update uploaded file data in database using 'models.Users.update'
//send response to client/frontend
var obj = {};
obj.status = { "code": "200", "message": "Yipee!! Its Done" };
obj.result = { url: awsFileData.Location };
res.status(200).send(obj);
}
});
}
});
});
});
This is old school, non - fancy solution.Please try it out and let me know.
Im trying to get progress status values while uploading files to google Drive using nodeJs.
controller.js
exports.post = (req, res) => {
//file content is stored in req as a stream
// 1qP5tGUFibPNaOxPpMbCQNbVzrDdAgBD is the folder ID (in google drive)
googleDrive.makeFile("file.txt","1qP5tGUFibPNaOxPpMbCQNbVzrDdAgBD",req);
};
googleDrive.js
...
makeFile: function (fileName, root,req) {
var fileMetadata = {
'name': fileName,
'mimeType': 'text/plain',
'parents': [root]
};
var media = {
mimeType: 'text/plain',
body: req
};
var r = drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
// r => undefined
console.log("Uploaded: " + r);
}
});
},
...
i followed this link but got always an undefined value
How about this modification?
Modification point:
It used onUploadProgress.
Modified script:
makeFile: function (fileName, root,req) {
var fileMetadata = {
'name': fileName,
'mimeType': 'text/plain',
'parents': [root]
};
var media = {
mimeType: 'text/plain',
body: req
};
var r = drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, {
onUploadProgress: function(e) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(e.bytesRead.toString());
},
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
console.log("Uploaded: " + file.data.id);
}
});
},
Note:
If you want to show the progression as "%", please use the file size.
It was confirmed that this script worked at googleapis#33.0.0.
References:
axios
test of google/google-api-nodejs-client
In my environment, I'm using the script like above. But if this didn't work in your environment and if I misunderstand your question, I'm sorry.
I am developing a chatbot on the Facebook Messenger Platform using Node.js. This is my functioning code for setting up a text response:
const fbReq = request.defaults({
uri: 'https://graph.facebook.com/me/messages',
method: 'POST',
json: true,
qs: {
access_token: Config.FB_PAGE_TOKEN
},
headers: {
'Content-Type': 'application/json'
},
});
const fbMessage = (recipientId, msg, cb) => {
const opts = {
form: {
recipient: {
id: recipientId,
},
message: {
text: msg,
},
},
};
fbReq(opts, (err, resp, data) => {
if (cb) {
cb(err || data.error && data.error.message, data);
}
});
};
I am also able to set up an image response this way. However, when I try to make the response a button template (https://developers.facebook.com/docs/messenger-platform/send-api-reference/button-template), no response is received. No error is thrown either.
const fbInfo = (recipientId, cb) => {
const opts = {
form: {
recipient: {
id: recipientId,
},
message: {
attachment:{
type:"template",
text:"Check out our website",
payload:{
template_type:"button",
buttons:[
{
type:"web_url",
url:"https://some.website.com",
title:"Website"
}
]
}
}
}
}
};
fbReq(opts, (err, resp, data) => {
if (cb) {
cb(err || data.error && data.error.message, data);
}
});
};
Instead of form you should use json.
take a look at the code which I have written on glitch
should be something like:
request({
uri: 'https://graph.facebook.com/v2.6/me/messages',
qs: { access_token: <TOKEN> },
method: 'POST',
json: messageData}, ...)
I'm trying to export multiple Google Docs from a folder with the Drive REST Api. In the following code, 'fileId' is set to a specific fileId (and it works perfectly!):
var service = google.drive('v3');
fileId = <file_id>
dest = <local_file>
service.files.export({
fileId: fileId,
mimeType: 'text/plain',
auth: <auth>
})
.pipe(dest);
However, when I nest this in a loop in order to export multiple files like so:
var service = google.drive('v3');
fileId = <file_id>
dest = <local_file>
service.files.list({
auth: <auth>,
pageSize: 1,
fields: "nextPageToken, files(id)"
}, function(err, resp) {
if (err) { return err; }
var files = resp.files
for (var file of files) {
var fileId = file.id;
service.files.export({
fileId: fileId,
mimeType: 'text/plain',
auth: auth
})
.pipe(dest);
}
});
I'm given the following error:
{ [Error: Bad Request]
code: 400,
errors:
[ { domain: 'global',
reason: 'badRequest',
message: 'Bad Request' } ] }
And for the life of me I can't figure out why. Any help is much appreciated, Thanks!
I am trying to upload a file that is being uploaded to my hapijs server directly to slack using the web api files.upload.
I get in return an error message {"ok":false,"error":"no_file_data"}.
The same code works if i upload a file that already in the storage using fs.createReadStream
Here is my code:
const Promise = require('bluebird');
const Joi = require('joi');
const Path = require('path');
const Boom = require('boom');
const Utils = require('../utils');
const Request = require('request');
exports.routes = Promise.coroutine(function*(server, options) {
server.route({
method: 'POST',
path: options.prefix,
config: {
tags: ['api', 'v1'],
payload: {
maxBytes: 2097152,
output: 'stream',
parse: true
},
plugins: {
'hapi-swagger': {
payloadType: 'form'
}
},
validate: {
payload: Joi.object({
cv: Joi.object({ pipe: Joi.func().required() }).required().unknown().description('CV file').meta({ swaggerType: 'file' })
})
},
response: {
emptyStatusCode: 204,
schema: false
},
handler: Utils.coHandler(function*(request, reply) {
const supportedFiles = ['docx', 'doc', 'pdf', 'pages'];
const ext = Path.extname(request.payload.cv.hapi.filename).substr(1);
if (!supportedFiles.filter((supExt) => supExt === ext).length) {
return reply(Boom.badRequest('Unsupported file type'));
}
Request.post({
url: 'https://slack.com/api/files.upload',
formData: {
token: 'XXXX',
filename: `${request.payload.name} - CV.${ext}`,
file: request.payload.cv,
channels: 'XXXX'
}
}, (err, res, body) => {
console.log(body);
});
reply();
})
}
});
});