Nodejs sending file through http - node.js

I'm trying to send a file to another node.js service. So for that i'm using http and form-data modules.
This is the code i wrote
function uintToString(uintArray) {
return String.fromCharCode.apply(null, new Uint8Array(uintArray));
}
function (file) {
var data = uintToString(file.buffer);
//
var crlf = "\r\n",
boundaryKey = Math.random().toString(16),
boundary = `--${boundaryKey}`;
delimeter = `${crlf}--${boundary}`,
preamble = "", // ignored. a good place for non-standard mime info
epilogue = "",
headers = [
'Content-Disposition: form-data; name="file"; filename="' + name + '"' + crlf
],
closeDelimeter = `${delimeter}--`,
multipartBody = Buffer.concat(
new Buffer(preamble + delimeter + crlf + headers.join('') + crlf),
data,
new Buffer(closeDelimeter + epilogue)
);
var options = {
host: 'localhost',
port: 3000,
method: 'POST',
path: '/data/get',
headers: {
'Content-Type': 'multipart/form-data; boundary=' + boundary,
'Content-Length': formData._valueLength
}
};
//make request
return httpsRequest(formData, options)
.then((result) => {
console.log(result);
}, (err) => {
console.log(err);
});
};
function httpsRequest(data, options) {
return new Promise(function (resolve, reject) {
// request object
var req = https.request(options, function (res) {
var result = '';
res.on('data', function (chunk) {
result += chunk;
});
res.on('end', function () {
console.log("https end result - " + result);
resolve(result);
});
res.on('error', function (err) {
reject(err);
})
});
// req error
req.on('error', function (err) {
reject(err);
});
//send request witht the postData form
req.write(data);
req.end();
});
}
It is giving ""list" argument must be an Array of Buffers" this error. It looks like something is wrong on the httpsRequest function.

Don't reinvent the wheel, needle/request can do this for you. If you want to promisify things, use bluebird
const Promise = require('bluebird')
const needle = Promise.promisifyAll(require('needle'))
function send (file) {
let url = 'https://localhost/data/get'
let data = {
zip_file: {
buffer : file.buffer,
filename : name,
content_type : 'application/octet-stream'
}
}
return needle.postAsync(url, data, { multipart: true })
}

Related

How to handle large file processing through AJAX and express

I'm uploading a PDF file using AJAX and express and sending the data to external REST API for some file processing.
Whenever I upload more than 3MB file I will get an error in AJAX before the REST API response.
I get correct results from the external REST API, but processing takes more time so AJAX is not receiving the success msg.
How do I handle this?
Tried with AXIOS and other promise handler but same result
module.exports = (req, res) =>
{
var pdf = [],
form = new formidable.IncomingForm(),
pdfFile,
dirname = process.cwd();
form.multiples = false;
// Upload directory for the images
form.uploadDir = path.join(dirname, 'tmp_uploads');
form.on('file', function(name, file) {
let token;
console.log('FORM ON')
if (req.cookies.userData.token) {
token = req.cookies.userData.token;
let buffer = null,
type = null,
filename = '',
renameFile = '';
// Read a chunk of the file.
buffer = readChunk.sync(file.path, 0, 262);
// filename = Date.now() + '-' + file.name;
filename = file.name;
renameFile = path.join(dirname, 'uploads/' + filename);
fs.rename(file.path, renameFile, function(err) {
if (err) throw err;
console.log('renamed complete');
pdfFile = path.join(dirname, 'uploads/' + filename);
let readFileStream = fs.createReadStream(pdfFile),
data = '',
formData = {};
formData = {
name: filename,
my_file: readFileStream,
Width: 1024,
Height: 768
};
function postData() {
// Setting URL and headers for request
var options = {
url: EXTERNALURL,
headers: {
"Authorization": 'bearer ' + token,
"content-type": "multipart/form-data"
},
formData: formData
};
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.post(options, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
function getData(url) {
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.get({ url: url, encoding: null }, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
var errHandler = function(err) {
console.log(err);
}
var filePath;
function main() {
var dataPromise = postData();
// Get user details after that get followers from URL
dataPromise.then(JSON.parse, errHandler)
.then(function(result) {
fData = result;
var fileName = fData.Id,
file_url = fData.PresentationZipUrl;
filePath = path.join(dirname, fData.HtmlPath);
// Do one more async operation here
var downloadPromise = getData(file_url).then();
return downloadPromise;
}, errHandler)
.then(function(data) {
//console.log(data);
var zip = new AdmZip(data);
if (!fs.existsSync(filePath)) {
fs.mkdirSync(filePath);
zip.extractAllTo(filePath, true);
}
}, errHandler)
.then(function(data) {
console.log('Done');
res.status(200).json({ 'status': "success" });
}, errHandler);
}
// console.log('before');
main();
}); // END RENAME FILE
} else { //
console.log('ERROR')
res.redirect('/');
}
});
form.on('error', function(err) {
console.log('Error occurred during processing - ' + err);
});
// Invoked when all the fields have been processed.
form.on('end', function() {
console.log('All the request fields have been processed.');
});
// Parse the incoming form fields.
form.parse(req, function(err, fields, files) {})}
AJAX:
$.ajax({
enter code here
url: '/presentation/uploadFiles',
method: 'post',
data: formData,
processData: false,
contentType: false,
timeout: 0}).done(function(d) {
console.log(d);
if (d.status == 'success') {
location.reload()
}}).fail(function(e, t) {
console.log(e);
console.log(t)})
.always(function() {
});

Azure Function ignoring https.request

I have an azure function with this line of code.
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
But my code seems to just skip this part of code, with no errors. I am new to Azure and node.js so I might have missed some basic parts in setting this up.
Any ideas?
Edit:
Here is my full code
const https = require('https');
const querystring = require('querystring');
module.exports = async function (context, req) {
if (req.query.accessCode || (req.body && req.body.accessCode)) {
context.log('JavaScript HTTP trigger function processed a request.');
var options = {
host: 'httpbin.org',
port: 80,
path: '/post',
method: 'POST'
};
var postData = querystring.stringify({
client_id : '1234',
client_secret: 'xyz',
code: req.query.accessCode
});
var body = "";
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
context.log("help");
context.res = {
status: 200,
body: "Hello " + (body)
};
} else {
context.res = {
status: 400,
body: "Please pass a name on the query string or in the request body"
};
}
};
Ideally it should work. You can also try using request module like below
const request = require('request');
request('http://www.google.com', function (error, response, body) {
console.error('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
Try and see if it helps.
Solved by doing await properly. Used this as guide.
var https = require('https');
var util = require('util');
const querystring = require('querystring');
var request = require('request')
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
/*if (req.query.name || (req.body && req.body.name)) {*/
var getOptions = {
contentType: 'application/json',
headers: {
'Authorization': <bearer_token>
},
};
var postData = {
"key": "value"
};
var postOptions = {
method: 'post',
body: postData,
json: true,
url: <post_url>,
headers: {
'Authorization': <bearer_token>
},
};
try{
var httpPost = await HttpPostFunction(context, postOptions);
var httpGet = await HttpGetFunction(context, <get_url>, getOptions);
return {
res: httpPost
};
}catch(err){
//handle errr
console.log(err);
};
};
async function HttpPostFunction(context, options) {
context.log("Starting HTTP Post Call");
return new Promise((resolve, reject) => {
var data = '';
request(options, function (err, res, body) {
if (err) {
console.error('error posting json: ', err)
reject(err)
}
var headers = res.headers;
var statusCode = res.statusCode;
//context.log('headers: ', headers);
//context.log('statusCode: ', statusCode);
//context.log('body: ', body);
resolve(body);
})
});
};
async function HttpGetFunction(context, url, options) {
context.log("Starting HTTP Get Call");
return new Promise((resolve, reject) => {
var data = '';
https.get(url, options, (resp) => {
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
})
// The whole response has been received. Print out the result.
resp.on('end', () => {
resolve(JSON.parse(data));
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(err.message);
});
});
};

var function calling for unknown reason

I have the code below, and it seems to call the var promiseFeedback is called and I don't know why... This means it is called even when an error occurs when I create document. Whereas is should only be called if there is no err in the createDocument.
Is anyone able to clear up why?
if (json) {
createDocument(documentUrl, context, json, function(res){
var promiseFeedback = callFB (context, res);
var collection = `mydb`
client.createDocument(collection, res, (err, result) => {
if(err) {
context.log(err);
return context.done();
} else {
Promise.all([promiseFeedback]).then(function(results){
context.log("promiseFeedback: " + results[0]);
context.done();
});
}
});
});
}
function callFB(context, res) {
return new Promise((resolve, reject) => {
var requestUrl = url.parse( URL );
var requestBody = {
"id": res.id
};
var body = JSON.stringify( requestBody );
const requestOptions = {
hostname: requestUrl.hostname,
path: requestUrl.path,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body),
}
};
var request = https.request(requestOptions, function(res) {
var data ="";
res.on('data', function (chunk) {
data += chunk
});
res.on('end', function () {
resolve(true);
})
}).on('error', function(error) {
context.log("request error:", error);
resolve(false);
});
request.write(body);
request.end();
});
}
var promiseFeedback = callFB (context, res);
This statement executes callFB immediately, not just assigns another name to the promise. This promise callFB is out of the callback(scope) of err and Promise.all([promiseFeedback]), it runs no matter what the result of client.createDocument is.
To fix this:
Remove var promiseFeedback = callFB (context, res); and change Promise.all([promiseFeedback]) to callFB(context, res). You don't need to use Promise.all as you only have one promise to resolve.
Or you can just move var promiseFeedback = callFB (context, res); into else segment.

Using Q library for HTTP api response testing in nodejs

how to use Q to make it wait until previous response has come from the server.
What I am looking to do here is compare the response from test server and production server for the same request.
I get the responses back from both the servers, but unable to compare them since the assert statement is executed before the response comes back.
Any one know what I am doing wrong. heres the code.
var Q = require('q');
var path='';
var prodResponse = '';
var tstReponse = '';
Q.fcall(readFile())
.then(secondFunction())
.then(thirdFunction())
.then(function(){
console.log("prodResponse: "+prodResponse);
console.log("tstResponse: "+tstResponse);
assert.strictEqual(prodResponse, tstResponse)
})
.catch(function(){
console.log('error occurred');
})
.done();
function readFile(){
fs.readFile('hostname.json', function (err, data) {
if (err) return console.error(err);
path = JSON.parse(data);
return JSON.parse(data);
});
}
function secondFunction(){
var prodOptions = {
hostname: 'somehostname.com',
port: 80,
path: "/path?"+path.path,
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=UTF-8'
},
auth : ''
};
return http.request(prodOptions, function(res) {
console.log('Prod');
res.setEncoding('utf8');
res.on('data', function (chunk) {
prodResponse = chunk;
return chunk;
});
res.on('end', function() {
console.log('No more data in response.');
})
}).on('error', function(e) {
console.log('problem with request: ' + e.message);
}).end();
}
function thirdFunction(){
// same a second, only difference is the response http.
}
There is multiple errors in your code
Q.fcall(readFile())
Your q variable is q and not Q. So this line will crash because Q is undefined (javascript is case sensitive).
Then, readFile doesn't return any promise (in fact, it returns nothing). So the q library can't use anything to wait the end of any asynchronous work. The then callbacks will be fired immediatly.
You can use Q.ninvoke to make your readFile function return a promise, and you can use Q.defer to create and return a promise from your secondFunction:
var Q = require('q');
var path='';
var prodResponse = [];
var tstReponse = '';
readFile()
.then(secondFunction())
.then(thirdFunction())
.then(function(){
console.log("prodResponse: "+prodResponse);
console.log("tstResponse: "+tstResponse);
assert.strictEqual(prodResponse, tstResponse)
})
.catch(function(){
console.log('error occurred');
})
.done();
function readFile(){
return Q.ninvoke(fs, 'readFile', 'hostname.json').then(function (data) {
path = JSON.parse(data);
return path;
}, function (err) {
console.error(err);
});
}
function secondFunction(){
var prodOptions = {
hostname: 'somehostname.com',
port: 80,
path: "/path?"+path.path,
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=UTF-8'
},
auth : ''
};
var defer = Q.defer();
var chunks = [];
http.request(prodOptions, function(res) {
console.log('Prod');
res.setEncoding('utf8');
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function() {
console.log('No more data in response.');
prodResponse = chunks.join('');
defer.resolve(prodResponse);
})
}).on('error', function(e) {
console.log('problem with request: ' + e.message);
defer.reject(e);
}).end();
return defer.promise;
}
function thirdFunction(){
// same a second, only difference is the response http.
}

Uploading video to Vimeo with nodejs

I'm trying to upload a video to vimeo with nodejs (https://developer.vimeo.com/apis/advanced/upload at step 3) . This is what I currently do:
Firstly I call the function to read the file:
var options = {
hostname : dataObject.ticket.host,
path : '/upload?ticket_id=' + dataObject.ticket.id,
port : 8080,
method: 'POST'
}
postMovie(options);
I get these parameters from my object:
{
"generated_in": "0.0308",
"stat": "ok",
"ticket": {
"endpoint": "http://126535.cloud.vimeo.com:8080/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958",
"endpoint_secure": "https://126535.cloud.vimeo.com/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958",
"host": "126535.cloud.vimeo.com",
"id": "9d818e8bd066dfd54e53f1be2fa3f958",
"max_file_size": "26843545600"
}
}
This function is called :
function postMovie(options){
// This is an async file read
fs.readFile('public/uploads/4363066343.mp4', function (err, data) {
if (err) {
console.log("FATAL An error occurred trying to read in the file: " + err);
process.exit(-2);
}
// Make sure there's data before we post it
if(data) {
PostData(data,options);
}
else {
console.log("No data to post");
process.exit(-1);
}
});
};
When the file is read:
function PostData(data,options) {
var headers = {
'Content-Type': 'video/mp4',
'Content-Length': data.length
}
options.headers = headers
console.log(options)
// Set up the request
var post_req = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
// post the data
post_req.write(data);
post_req.end();
post_req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
}
My post_req.on(error) logs this:
problem with request: write EPIPE
problem with request: write EPIPE
I understand this is because of a time-out at the serverside.
I assume my request is not well formed.
Can someone point out what I did wrong ?
The upload operation will be much simpler with the request module.
var inspect = require('eyespect').inspector();
var request = require('request')
var path = require('path')
var fs = require('fs')
var filePath = path.join(__dirname, '../public/uploads/foo.mp4')
fs.stat(filePath, function(err, stats) {
if (err) {
inspect(err, 'error stating file')
return
}
var fileSize = stats.size
var url = 'https://126535.cloud.vimeo.com/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958'
var opts = {
url: url,
method: 'post',
headers: {
'Content-Length': fileSize,
'Content-Type': 'foo'
}
var r = request(opts)
// pipe the file on disk to vimeo
var readStream = fs.createReadStream(filePath)
readStream.pipe(r)
readStream.on('error', function (err) {
inspect(err, 'error uploading file')
})
readStream.on('end', function (err) {
inspect('file uploaded correctly')
})
})
Request also allows you to set the timeout option as well if the file is big and thus takes a long time to upload

Resources