I have an api to file upload on Bluemix Object Storage by using request module. All are good but there is some unwanted character which append automatically.
example:
--38oi85df-b5d1-4d42-81ce-c547c860b512 //this is unwanted character
Email
abc#gmail.com
hsl#gmsl.com
pjeyjle#cse.com
--38oi85df-b5d1-4d42-81ce-c547c860b512-- // this is unwanted character
Here is my code:-
import request from 'request';
exports.putObjectStorageFile = function(authToken, file, csv, cb) {
var s = new stream.Readable();
s._read = function noop() {};
s.push(csv); //csv is string
s.push(null);
var options = {
url: 'https://xxxx.objectstorage.open.xxxx.com/v1/AUTH_' + config.objectStorage.projectId + '/xxxx/' + file,
method: 'PUT',
preambleCRLF: true,
postambleCRLF: true,
encoding: 'utf-8',
headers: {
'Content-Type': 'text/html; charset=UTF-8',
'Content-Length': 1,
'X-Auth-Token': authToken
},
multipart: {
chunked: false,
data: [
{ body: s }
]
} };
function callback(error, response) {
if (error) cb(error);
if (!error && response.statusCode == 201) {
cb(null);
}
}
request(options, callback);
You are sending a multipart message with a preambleCRLF and postambleCRLF in your request which is causing those lines.
You should use the pkgcloud library for uploading data to Object Storage:
https://github.com/pkgcloud/pkgcloud
Below is a sample of using pkgcloud with the Object Storage service on Bluemix (credentials from VCAP).
(function (module) {
var pkgcloud = require('pkgcloud'),
fs = require('fs');
function ObjectStorage(container, credentials) {
this.container = container;
this.config = {
provider: 'openstack',
useServiceCatalog: true,
useInternal: false,
keystoneAuthVersion: 'v3',
authUrl: credentials.auth_url,
tenantId: credentials.projectId,
domainId: credentials.domainId,
username: credentials.username,
password: credentials.password,
region: credentials.region
};
this.client = pkgcloud.storage.createClient(this.config);
}
ObjectStorage.prototype.validate = function () {
return new Promise(function (resolve, reject) {
this.client.auth(function (error) {
if (error) {
return reject(error);
}
resolve();
});
}.bind(this));
};
ObjectStorage.prototype.makeContainer = function () {
return new Promise(function (resolve, reject) {
this.client.createContainer({name: this.container}, function (error) {
if (error) {
return reject(error);
}
return resolve();
});
}.bind(this));
};
ObjectStorage.prototype.uploadFile = function (path, name) {
return new Promise(function (resolve, reject) {
var myPicture = fs.createReadStream(path);
var upload = this.client.upload({
container: this.container,
remote: name
});
upload.on('error', function (error) {
reject(error);
});
upload.on('success', function (file) {
resolve(file);
});
myPicture.pipe(upload);
}.bind(this));
};
module.exports = ObjectStorage;
})(module);
I got that lines due to sending data with multipart. I found a solution by adding just content-type,content-lenght and send data in body i.e,
var options = {
url: 'https://dal.objectstorage.open.softlayer.com/v1/AUTH_' + config.objectStorage.projectId + '/nrich-storage/' + file,
method: 'PUT',
headers: {
'Content-Type': 'text/csv',
'Content-Length': csv.length,
'X-Auth-Token': authToken
},
body:csv
};
function callback(error, response) {
if (error) cb(error);
if (!error && response.statusCode == 201) {
cb(null);
}
}
request(options, callback);
Related
I've been trying to figure this out for two days now.
I have the below function working, it basically takes URL and upload the file to S3. However the problem is I'm trying to return the file location on S3 (data.key) out of the main function but I'm either getting undefined or promise(pending).
var express = require("express"),
axios = require('axios').default,
stream = require("stream"),
aws = require('aws-sdk')
async function downloadImage(url) {
let contentType = 'application/octet-stream'
const s3 = new aws.S3();
// This returns undefined
var imageUrl
// This returns Promise { <pending>}
// var imageUrl = await uploadS3()
var imageRequest = axios({
method: 'get',
url: url,
responseType: 'stream'
}).then(function (response) {
if (response.status === 200) {
contentType = response.headers['content-type'];
response.data.pipe(uploadS3());
}
}).catch(function (error) {
console.log(error.message);
})
// This is where I can't get return
return imageUrl
function uploadS3() {
var pass = new stream.PassThrough();
var params = {
Bucket: "test_bucket",
Key: "test/" + Date.now().toString(),
Body: pass,
ContentType: contentType,
}
s3.upload(params, function (err, data) {
if (err) {
console.log(err)
} else {
// I have access to data.key only here
imageUrl = data.key
console.log(imageUrl)
}
})
return pass
}
}
I was trying something like this. This gets the value up to the Then of the first function, but still no luck getting it out of the main funciton
async function uploadImage(url) {
let contentType = 'application/octet-stream'
const s3 = new aws.S3();
// var imageUrl = await uploadStream()
// var imageUrl = await Promise.all
var imageUrl
var imageRequest = axios({
method: 'get',
url: url,
responseType: 'stream'
}).then(function (response) {
if (response.status === 200) {
const { writeStream, promise } = uploadStream();
contentType = response.headers['content-type'];
upload = response.data.pipe(writeStream);
promise.then((data) => {
// I have access to it here, but not
console.log(data.key)
imageUrl = data.key
}).catch((err) => {
console.log('upload failed.', err.message);
});
}
}).catch(function (error) {
console.log(error.message);
})
return imageUrl
function uploadStream() {
var pass = new stream.PassThrough();
var params = {
Bucket: "test_bucket",
Key: "test/" + Date.now().toString(),
Body: pass,
ContentType: contentType,
}
return {
writeStream: pass,
promise: s3.upload(params).promise(),
};
}
}
I was able to get it to work the following way if anybody is interested. I have a Promise created which returns a result (which is file location on S3) once upload is completed.
I then use this inside of my main functions to basically do download and then return the location.
var express = require("express"),
axios = require('axios').default,
stream = require("stream"),
aws = require('aws-sdk')
// This is now can be run from any other function while getting access to file location (result)
uploadImageAsync("imageURLGoesHere").then(function (result) {
// result contains the location on S3 data.key (this can be changed in the code to return anything else from data)
console.log(result)
}, function (err) {
console.log(err);
})
async function uploadImageAsync(url) {
let contentType = 'application/octet-stream'
const s3 = new aws.S3();
return new Promise(function (resolve, reject) {
var imageRequest = axios({
method: 'get',
url: url,
responseType: 'stream'
}).then(function (response) {
if (response.status === 200) {
const { writeStream, promise } = uploadStream();
contentType = response.headers['content-type'];
response.data.pipe(writeStream);
return new Promise(function (resolve, reject) {
promise.then((data) => {
resolve(data.key);
}).catch((err) => {
console.log('upload failed.', err.message);
});
});
}
}).catch(function (error) {
console.log(error.message);
})
imageRequest.then(function (result) {
resolve(result);
})
function uploadStream() {
var pass = new stream.PassThrough();
var params = {
Bucket: "test_bucket",
Key: "test/" + Date.now().toString(),
Body: pass,
ContentType: contentType,
}
return {
writeStream: pass,
promise: s3.upload(params).promise(),
};
}
})
}
I'm uploading a PDF file using AJAX and express and sending the data to external REST API for some file processing.
Whenever I upload more than 3MB file I will get an error in AJAX before the REST API response.
I get correct results from the external REST API, but processing takes more time so AJAX is not receiving the success msg.
How do I handle this?
Tried with AXIOS and other promise handler but same result
module.exports = (req, res) =>
{
var pdf = [],
form = new formidable.IncomingForm(),
pdfFile,
dirname = process.cwd();
form.multiples = false;
// Upload directory for the images
form.uploadDir = path.join(dirname, 'tmp_uploads');
form.on('file', function(name, file) {
let token;
console.log('FORM ON')
if (req.cookies.userData.token) {
token = req.cookies.userData.token;
let buffer = null,
type = null,
filename = '',
renameFile = '';
// Read a chunk of the file.
buffer = readChunk.sync(file.path, 0, 262);
// filename = Date.now() + '-' + file.name;
filename = file.name;
renameFile = path.join(dirname, 'uploads/' + filename);
fs.rename(file.path, renameFile, function(err) {
if (err) throw err;
console.log('renamed complete');
pdfFile = path.join(dirname, 'uploads/' + filename);
let readFileStream = fs.createReadStream(pdfFile),
data = '',
formData = {};
formData = {
name: filename,
my_file: readFileStream,
Width: 1024,
Height: 768
};
function postData() {
// Setting URL and headers for request
var options = {
url: EXTERNALURL,
headers: {
"Authorization": 'bearer ' + token,
"content-type": "multipart/form-data"
},
formData: formData
};
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.post(options, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
function getData(url) {
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.get({ url: url, encoding: null }, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
var errHandler = function(err) {
console.log(err);
}
var filePath;
function main() {
var dataPromise = postData();
// Get user details after that get followers from URL
dataPromise.then(JSON.parse, errHandler)
.then(function(result) {
fData = result;
var fileName = fData.Id,
file_url = fData.PresentationZipUrl;
filePath = path.join(dirname, fData.HtmlPath);
// Do one more async operation here
var downloadPromise = getData(file_url).then();
return downloadPromise;
}, errHandler)
.then(function(data) {
//console.log(data);
var zip = new AdmZip(data);
if (!fs.existsSync(filePath)) {
fs.mkdirSync(filePath);
zip.extractAllTo(filePath, true);
}
}, errHandler)
.then(function(data) {
console.log('Done');
res.status(200).json({ 'status': "success" });
}, errHandler);
}
// console.log('before');
main();
}); // END RENAME FILE
} else { //
console.log('ERROR')
res.redirect('/');
}
});
form.on('error', function(err) {
console.log('Error occurred during processing - ' + err);
});
// Invoked when all the fields have been processed.
form.on('end', function() {
console.log('All the request fields have been processed.');
});
// Parse the incoming form fields.
form.parse(req, function(err, fields, files) {})}
AJAX:
$.ajax({
enter code here
url: '/presentation/uploadFiles',
method: 'post',
data: formData,
processData: false,
contentType: false,
timeout: 0}).done(function(d) {
console.log(d);
if (d.status == 'success') {
location.reload()
}}).fail(function(e, t) {
console.log(e);
console.log(t)})
.always(function() {
});
I need to send a lot of messages. I use external api and request module:
email.js
var request = require('request');
module.exports = function(subject, email){
var message = ... ;
var options = {
method: 'POST',
url: 'https://api.emails.net.com/api/sendmail',
form: {
smtp_account: smtp,
to: {},
subject: subject,
html: message,
from: 'myapp#email.com',
from_name: 'Myapp'
},
headers: {
'Authorization': ' ... '
}
};
request(options, function (error, response) {
if (!error && response.statusCode === 200) return true;
return false;
});
};
And in another place:
var sendEmail = require('../email');
...
for( ... ){
if(sendEmail(subject, email)){
//do something
}
}
After success i want to do something but request return nothing. Is any way to check if post was success and do something in place where I require email module? I'm sending emails in loop and i must check one by one.
You can do something like this
var request = require('request');
var Promise = require('bluebird');
var email = function () {
return {
send: function (subject, email) {
var message = "test";
var options = {
method: 'POST',
url: 'https://api.emails.net.com/api/sendmail',
form: {
smtp_account: "smtp",
to: {},
subject: subject,
html: message,
from: 'myapp#email.com',
from_name: 'Myapp'
},
headers: {
'Authorization': ""
}
};
return new Promise(function (resolve, reject) {
request(options, function (error, response) {
if (!error && response.statusCode === 200) {
resolve(true);
} else {
reject(err);
}
});
})
}
}
};
module.exports = email();
And then you can consume this as below
for (var i = 0; i < 10; i++) {
console.log(i);
email
.send("test", "test#gmail.com")
.then(function (status) {
///Do what ever u want to do
})
.catch(function (err) {
console.log(err);
});
}
I've tested the interface and made some changes to use with my sailsjs (version 11) on the backend. In this case I use a sails service and things seem fine but I always get back a pending status.
```
SinchService.js
var sinchAuth = require('sinch-auth');
var request = require('request');
var sinchMessaging = {};
var Promise = require('bluebird');
//sinchMessaging.sendMessage = function (phoneNumber, message) {
//exports.sendMessage = function (phoneNumber, message) {
module.exports = {
sendMessage: function (phoneNumber, message) {
var auth = sinchAuth();
if (!auth) {
throw new Error("No Authorization was provided");
}
var options = {
method: 'POST',
url: "https://messagingApi.sinch.com/v1/sms/" + phoneNumber,
headers: {
"Content-Type": "application/json",
"Authorization": auth
},
body: "{\"Message\":\"" + message + "\"}"
// body: {"Message" : message }
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
sails.log("Finished with call");
if (error) {
sails.log(error);
throw error;
}
else {
sails.log("Finished with body ", body);//.MessageId
return resolve(response.body);
}
});
})
},
getStatus: function (messageId) {
var auth = sinchAuth();
if (!auth) {
throw new Error("No Authorization was provided");
}
var options = {
method: 'GET',
url: "https://messagingApi.sinch.com/v1/sms/" + messageId,
headers: {
"Content-Type": "application/json",
"Authorization": auth
}
};
return new Promise(function (resolve, reject) {
request(options, function (error, response, body) {
sails.log("Finished with call");
if (error) {
sails.log(error);
throw error;
}
else {
return resolve(response.body);
}
});
})
}
};
```
agendaService.js
var jsonObjS;
SinchService.sendMessage(phoneNumber, message).then(function (results) {
var jsonObj = JSON.parse(results);
console.log('results sendMessage ', jsonObj.messageId);
if (jsonObj.messageId!==undefined){
SinchService.getStatus(jsonObj.messageId).then(function (results_s) {
jsonObjS = JSON.parse(results_s);
console.log('results getStatusS ', jsonObjS.status);
SinchService.getStatus(jsonObjS.messageId).then(function (results_s) {
var jsonObjS = JSON.parse(results_s);
console.log('results getStatusS ', jsonObjS.status);
});
});
```
Pending will always be the first status, query again after some time to see the status of of the message.
I have a function upload file:
handleUpload: function (e) {
e.preventDefault();
var files = this.refs.fileupload.getDOMNode().files; //get all the files to be upload.
if (files.hasOwnProperty(0) //get the first file.
&& files[0] instanceof File) {
var formData = new FormData();
formData.append('fileUpload',
files[0]); //append file to be sent to the server.
formData['fileUpload'] = files[0]
Service.uploadContact(null,null,formData)
.then(function(data){
if (data.statusCode == 0) { //upload success
this.setState({
validationmessage: data.message,
success: true,
showSpinner: false
})
React.findDOMNode(this.refs.fileupload).value = '' //clear upload file.
this.filterContact_Paging(1); //refresh the contact list.
} else {
this.setState({
validationmessage: data.error, //upload fail error
success: false,
showSpinner: false
})
React.findDOMNode(this.refs.fileupload).value = '' //clear upload file.
}
}, function(_err){
console.log('err: ', _err)
})
}
},
And code process upload use request module:
serviceInterface[item.name] = function (params, csrfToken, formData) {
return new Promise(function (resolve, reject) {
var token = storage.get('token') || '',
request_params = {
url: CONFIG.apiPath + item.path,
method: item.method,
crossOrigin: true,
withCredentials: false,
json: true,
headers: {'Authorization': 'Bearer ' + token},
};
//This is where you append form data and in most cases for file upload.
if (formData) {
request_params['formData'] = formData;
}
//This is a cross site reference token
if (csrfToken) {
headers['x-csrf-token'] = csrfToken.token;
headers['x-csrf-sec'] = csrfToken.secret;
}
//This is where you determine post/get to send the data through body or querystring.
if (item.method == 'GET') {
request_params['qs'] = params;
} else {
request_params['body'] = params;
}
request(request_params, function (error, response, body) {
if (error) {
return reject(error);
}
if (response.statusCode === 418) {
return reject(body);
}
return resolve(body);
});
});
};
But when I run code, it throw a error:
TypeError: self._form.on is not a function