What is the equivalent of curl --upload-file in node-request - node.js

I'm using node-request and trying to send a file to IBM's HDFS, as per this documentation.
Passing this JSON object to request successfully uploads:
var options = {
method: 'put',
body: 'foo',
headers: {
'Content-Type': 'application/octet-stream',
'Transfer-Encoding': 'chunked'
}
};
And running this CURL command successfully uploads a file as well:
curl -v -X PUT -L -b cookie.jar "https://host:port/webhdfs/v1/tmp/myLargeFile.zip?op=CREATE&data=true" --header "Content-Type:application/octet-stream" --header "Transfer-Encoding:chunked" -T "file.txt"
However, trying to specify a file stream like so:
var options = {
method: 'put',
headers: {
'Content-Type': 'application/octet-stream',
'Transfer-Encoding': 'chunked'
},
multipart : [
{ body: fs.createReadStream(localFile) }
]
};
fails, and I don't know where I'm going wrong. How do I reproduce the '--upload-file' argument from CURL using node-request?

I threw up an example on Github that works.
The gist is that you need to pipe the file into Request:
fs.createReadStream(filePath).pipe(request.put(putURL,options,function(err, httpsResponse, body){
if ( err ) {
console.log('err', err);
} else {
console.log(body);
}
}));
Still not sure how to pass in a file stream as an option in the options params but this will do for me!
-- UPDATE --
Wow, I feel like an absolute idiot. The option would be file. Yes, it's that simple.

Related

How to send file using request?

I am trying to hit an API which I got as a curl request. But When I am trying to upload it throwing an error that file is not uploaded. This is my curl request which is working fine.
curl -v -u admin:geoserver -XPUT -H "Content-type: application/zip" --data-binary #/Users/aitplap24/Desktop/WebGIS/uploads/users/61adbbe3ececf2068d6eb862/1638776276622.zip http://localhost:8080/geoserver/rest/workspaces/61adbbe3ececf2068d6eb862/datastores/layer1/61a604021467d3152d5720fa.shp
Here is my NodeJS code for doing the same using requests
const url = process.env.GEOSERVER_URL;
var headers = {
'Content-type': 'application/zip'
};
var dataString = '#/Users/aitplap24/Desktop/WebGIS/uploads/users/61adbbe3ececf2068d6eb862/1638776276622.zip';
var options = {
url: 'http://localhost:8080/geoserver/rest/workspaces/61adbbe3ececf2068d6eb862/datastores/test1/file.shp',
method: 'PUT',
headers: headers,
body: dataString,
formData : {
"file" : fs.createReadStream("/Users/aitplap24/Desktop/WebGIS/uploads/users/61adbbe3ececf2068d6eb862/1638776276622.zip")
},
auth: {
'user': 'admin',
'pass': 'geoserver'
}
};
request(options, (error, response, body) => {
console.log(error, response, body);
});
Error which I am getting
Error [ERR_STREAM_WRITE_AFTER_END]: write after end
I am sure something I am doing wrong with file uploading. Any suggestions is of great help!

Turning a CURL request into a Node js format with request-promise

I have been trying to convert a working POST CURL request into Node JS format that i can automate. Im reaching a wall with converting it and hoping someone can help.
Here is the curl:
curl -POST https://api-xxxxxxxx/add
-H "Authorization: Bearer xxxxxxx"
-H "Content-Type: multipart/mixed"
-F 'metadata={ "domain_id": 11661, "container_id": 18569512, "name": "TestFile", "size": "1072902", "originalfilename" : "xxx-d6", "mime": "x-html-package",
"attributes":"main_html_file_path=index.html\nduration=10000" }; type=application/json' -F "file=#xxx-d6.x-html-package"
Im using request-promise module to make this request but i might be making a mistake converting it.
let step1Options = {
method: 'POST',
uri: 'https://api-xxx/add',
form: {
metadata: {
domain_id: domainID,
container_id: 1856951,
name: "TestFileNodeJS",
size: "1072902",
originalfilename : "xxx-d6",
mime: "x-html-package",
attributes:"main_html_file_path=index.html\nduration=10000"
},
file: fs.createReadStream("xxx.x-html-package")
},
headers: {
"content-type": "multipart/mixed",
Authorization: "Bearer " + authID
}
};
rp(step1Options)
.then(function (body) {
console.log("success")
console.log(body)
})
.catch(function (err) {
console.log("Error: ", err)
// POST failed...
});
The error im getting:
body:
'{"error":"Unexpected Content-Type header: application/x-www-form-urlencoded. Expecting: multipart/mixed; boundary=<boundary>.","status":"error"}' } }
As far as i know, im changing the Content-Type but still thinks im not. Reading a bit of the documentation of Request-Promise, i dont believe multipart/mixed is a supported? Though i could be wrong.
Does anyone have an idea on how i can get this working or suggest alternatives?

how to do this curl operation in Node.js using request library

How can I convert this curl operation using request Node.js library:
curl -L -X GET -H "Content-Type:application/json" -H "Authorization: authorization..." -H "Scope: 11111111" https://url/download >> file.gz
/*the -L is curl option which means --location Follow redirects (H)
--location-trusted Like '--location', and send auth to other hosts (H)*/
If you just want to download to a file, you can use a head request type.
The request will look like so:
request.head({
url: "https://url/download",
followAllRedirects: true,
headers: {
'Content-Type': 'application/json',
'Authorization':'authorization...',
'Scope': '11111111'
}
}, (err, res, body) => {
request('https://url/download')
.pipe(fs.createWriteStream(`tmp/${res.path}`)).on('close', (data, err) => {
if(err) console.log(`Unable to write to file ${err}`)
console.log('Done')
})
})
I have used a similar snippet which worked well
Use Postmans code generator
Click code on the top left
Paste your curl request
Select Node.js Request from dropdown on top left of popup
You should then get JS snippet converted from your working cURL request
Here is the solution we have to put request inside another because: so the first request returns the url and the second one will download the file
const options = {
url: url,
headers: {
Authorization: `auth`,
'scope': profileId,
'content-type': 'application/json'
},
};
const r = request.get(options, async (err, res, body) => {
const fileStream = fs.createWriteStream(`${path}.gz`);
request(res.request.uri.href).pipe(fileStream);
// path exists unless there was an error
});

sending multiform data with DUPLICATE KEYS in nodejs

I need to send a POST request with NodeJS to an API that requires the same multiform key be used more than once.
This is a CURL example of the required action:
curl -H "Authorization: Bearer MY_ACCESS_TOKEN" -i -X POST -F "whitespace=1" \
-F "terms[]=lait" -F "definitions[]=milk" -F "terms[]=petits pois" \
-F "definitions[]=peas" -F "title=My first set via the API" \
-F "lang_terms=fr" -F "lang_definitions=en" \
https://api.quizlet.com/2.0/sets
As you can see, the keys "terms[]" and "definitions[]" are used more than once in the same request.
I've tried using the nodejs request/http/multi-form libraries with no success, as most of them require a JavaScript object to define the form data, which of course cannot accept duplicate keys.
Other than resorting to an exec() command to cURL, is there any nodejs library that will enable me to send a request with duplicate multiform keys?
I'm really banging my head against a wall with this one..
Try this its an example with request library
let options = { method: 'POST',
url:url,
headers:
{
'cache-control': 'no-cache',
authorization: 'Bearer '+accessToken ,
'content-type': 'application/json'
},
body:
{ //your array here
terms:['terms'] ,
definitions:['milk']
},
json: true
};
request(options, function (error, response, body) {
if(error){
console.log("Error ",error);
}
console.log("Response",body);
})
With unirest:
var unirest = require('unirest');
var req = unirest('POST', 'YOUR URL')
.headers({
'Content-Type': 'multipart/form-data; boundary=--------------------------846713359653092950719061',
'Authorization': 'YOUR AUTH'
})
.field('AAA', 'VAL1')
.field('AAA', 'VAL2')
.field('AAA', 'VAL3')
.end(function (res) {
if (res.error) throw new Error(res.error);
console.log(res.raw_body);
});

How to write curl 'PUT' request in node with 'request' module

I have this curl request working.
curl -v "https://developer.api.autodesk.com/oss/v2/buckets/:bucketName/objects/"
-X "PUT" -H "Authorization: Bearer tokenGoesHere"
-H "Content-Type: application/octet-stream" -T "forupload.rvt"
How can I write this in node with npm request module.
I tried the following with 'request' and 'fs'.
I get back "Token is not provided in the request".
function uploadFile(bucketData){
var uri = 'https://developer.api.autodesk.com/oss/v2/buckets/' + bucketData['bucketKey'] + '/objects/'
var authorizationHeader = ' Bearer ' + bucketData['token'] // this works in other post/get requests
var contentTypeHeader = 'application/octet-stream'
var streamTarget = 'C:\\Users\\architech\\Desktop\\Forge Node\\Test.rvt';
console.log(uri)
console.log(authorizationHeader)
console.log(contentTypeHeader)
console.log(streamTarget)
// console output:
// https://developer.api.autodesk.com/oss/v2/buckets/bucketpqglrzt/objects/
// Bearer ....token....
// application/octet-stream
// C:\Users\architech\Desktop\Forge Node\Test.rvt
request.put(
{
url: uri,
// preambleCRLF: true,
// postambleCRLF: true,
multipart:
[
{
'Authorization': authorizationHeader,
'Content-Type': contentTypeHeader,
body: fs.createReadStream(streamTarget)
},
]
},
function(error, response, body){
if(!error){
console.log(body);
}else{
console.log(error);
}
})
}
After trying several approaches, while I couldn't reproduce your specific problem, the trouble I had was with the binary attachment loading properly. Because createReadStream() runs asynchronously, it doesn't really seem to work the way the request docs say it should when added to the multipart or formData keys. Not sure why this is?
I got it working first using http://requestb.in - comparing the curl request to the same request constructed with Node. Here is the final, working version:
var request = require('request')
fs = require('fs')
var options = {
uri: 'https://developer.api.autodesk.com/oss/v2/buckets/<yourBucket>/objects/<yourFile.ext>',
headers: {
'Content-Type': 'application/octet-stream',
'Authorization': 'Bearer <token>'
}
}
fs.createReadStream(__dirname + '/<fileName.ext>').pipe(request.put(options, function(err, response, body) {
console.log(body)
/*
{
"bucketKey" : "< yourBucket>",
"objectId" : "urn:adsk.objects:os.object:brandontestbucket2/skyscpr1.3ds",
"objectKey" : "<fileName.ext>",
"sha1" : "...redacted...",
"size" : 43791,
"contentType" : "application/octet-stream",
"location" : "https://developer.api.autodesk.com/oss/v2/buckets/< yourBucket>/objects/<fileName.ext>"
}
*/
}))

Resources