nodejs modio api "add modfile" failing to upload - node.js

Im trying to upload modfiles with the api but it keeps saying that Im not including filedata. Ive tried with fetch like in the docs but it just gives the same error. If I try http it just gives a list of the files as if it was a GET request.
var zip = `./mod.zip`; // it exists!
var body = {
//filedata: `#${zip}`,
filedata: fs.readFileSync(zip, `binary`),
//filehash: crypto.createHash('md5').update(fs.readFileSync(zip, `binary`)).digest('hex'),
//version: version,
//active: active,
//changelog: changelog,
//metadata_blob: meta,
};
var res = await fetch(`https://api.mod.io/v1/games/${config.modio.gameid}/mods/${config.modio.modid}/files`, { // actually HTTP
method: 'POST',
headers: {
'Authorization': `Bearer ${config.modio.token}`,
'Content-Type': 'multipart/form-data',
'Accept': 'application/json',
},
body: JSON.stringify(body),
});
//console.log(body.filedata);
res = await res.json();
if (res.error)
console.log(res.error);
else
console.log(res);
{
"error": {
"code": 422,
"error_ref": 13009,
"message": "Validation Failed. Please see below to fix invalid input:",
"errors": {
"filedata": "The filedata field is required when upload id is not present.",
"upload_id": "The upload id field is required when filedata is not present."
}
}
}
Yes, ive submitted a bug report to them already, twice. Now they are ghosting me. (I'll probably submit a link to this as well)

They replied!
Hi
As mentioned, you would need to send data using FormData, not JSON.
You can find details at https://developer.mozilla.org/en-US/docs/Web/API/FormData/Using_FormData_Objects
Thanks,
Danny King
Developer Relations Specialist
So, I remade my code
const https = require(`https`);
var crypto = require('crypto');
var FormData = require('form-data');
function ObjectToForm(obj = {}) {
var form = new FormData();
Object.keys(obj).forEach(key => {
var val = obj[key];
switch (typeof val) {
case `boolean`:
val = String(val);
break;
}
form.append(key, val);
});
return form;
}
if (fs.statSync(zip).size > 5368709120) return consolelog(`Zip bigger then 5gb`);
var body = {
filedata: fs.createReadStream(zip),
filehash: crypto.createHash('md5').update(fs.readFileSync(zip)).digest('hex'),
version: version,
active: active,
changelog: changelog,
metadata_blob: meta,
};
var form = ObjectToForm(body);
var options = {
hostname: 'api.mod.io',
port: 443,
path: `/v1/games/${config.modio.gameid}/mods/${config.modio.modid}/files`,
method: 'POST',
headers: {
'Authorization': `Bearer ${config.modio.token}`,
...form.getHeaders(),
'Accept': 'application/json',
},
};
var req = https.request(options, (res) => {
var data = [];
res.on('data', (d) => data.push(d));
req.on(`close`, () => {
var buffer = Buffer.concat(data);
var resp = JSON.parse(buffer.toString());
if (resp.error)
console.log(resp.error);
else if (res.statusCode == 201)
r(true);
});
});
req.on('error', (e) => {
console.log(`Error publishing:`);
console.log(e);
r(e);
});
form.pipe(req)
.on(`close`, () => req.end());
And it worked. Thanks King.

Related

Azure Face API - Node JS Cannot sent local file in form of binary

I have tried to send the base64 form image into the Azure Face-API, with a config like this
var config = {
method: 'post',
url: endpoint,
params: {
returnFaceId: true,
returnFaceLandmarks: false,
returnFaceAttributes: 'age,gender,headPose,smile,facialHair,glasses,emotion,hair,makeup,occlusion,accessories,blur,exposure,noise'
},
body:Buffer(facesBase64, 'base64'),
headers: {
'Ocp-Apim-Subscription-Key': subscriptionKey,
'Content-Type': 'application/octet-stream'
}
};
but it always gets error 400. Is the binary form I sent wrong? facesBase64 is already in Base64 form.
EDIT
facesBase64 is full of base64 like this value
data:image/jpeg;base64,/9j/4AAQSkZJRgABAQ......
I think you can't send the base64 image into the Azure Face-API.
UPDATE
Send local image to faceapi.
'use strict';
const request = require('request');
const fs = require("fs");
// Replace <Subscription Key> with your valid subscription key.
const subscriptionKey = "eb8d5ea******8877c0" ;
const uriBase = 'https://pan***api.cognitiveservices.azure.com/face/v1.0/detect';
const imageBuffer = fs.readFileSync('jason.jpg');
// Request parameters.
const params = {
'returnFaceId': 'true',
'returnFaceLandmarks': 'false',
'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' +
'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise'
};
const options = {
uri: uriBase,
qs: params,
body: imageBuffer,
headers: {
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key' : subscriptionKey
}
};
request.post(options, (error, response, body) => {
if (error) {
console.log('Error: ', error);
return;
}
let jsonResponse = JSON.stringify(JSON.parse(body), null, ' ');
console.log('JSON Response\n');
console.log(jsonResponse);
});
I tried the method you mentioned for testing and got the following results.
{
"error": {
"code": "InvalidImage",
"message": "Decoding error, image format unsupported."
}
}
Below is my test code.
'use strict';
//const request = require('request');
const fs = require("fs");
const axios = require("axios");
const request = require('request').defaults({ encoding: null });
// Replace <Subscription Key> with your valid subscription key.
const subscriptionKey = "eb8d5ea0********8877c0" ;
const uriBase = "https://pans***api.cognitiveservices.azure.com"+ '/face/v1.0/detect'
const imgUrl="https://pan***torage.blob.core.windows.net/ja**b/ja**.jpg";
var imageBuffer;
request.get(imgUrl, function (error, response, body) {
if (!error && response.statusCode == 200) {
var data;
data = "data:" + response.headers["content-type"] + ";base64," + Buffer.from(body).toString('base64');
imageBuffer=data;
aa();
}
});
function aa(){
// Request parameters.
const params = {
'returnFaceId': 'true',
'returnFaceLandmarks': 'false',
'returnFaceAttributes': 'age,gender,headPose,smile,facialHair,glasses,' +
'emotion,hair,makeup,occlusion,accessories,blur,exposure,noise'
};
const options = {
uri: uriBase,
qs: params,
body: imageBuffer,
headers: {
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key' : subscriptionKey
}
};
request.post(options, (error, response, body) => {
if (error) {
console.log('Error: ', error);
return;
}
let jsonResponse = JSON.stringify(JSON.parse(body), null, ' ');
console.log('JSON Response\n');
console.log(jsonResponse);
});
}
Suggest.
If you really want to implement your function in this way, it is recommended to raise a support on the portal. Consult the official answer, which image formats currently supported by faceapi.
You can also submit suggestions to the official product group.

Uploading blob/file in react-native, contents is empty

I am able to succesfully upload a blob with proper contents from my web browser, but when I do it from react-native, the upload file is empty. Here is the code:
async function doit() {
const data = new FormData();
data.append('str', 'strvalue');
data.append(
'f',
new File(['foo'], 'foo.txt', {type: 'text/plain'}),
);
await fetch('http://localhost:3002/upload', {
method: 'POST',
body: data
});
}
However doing this same code from react-native, it uploads, but the file is empty.
Here is the node.js server I am using to test this. Loading http://localhost:3002 gives you a button called "upload it". Clicking it does the upload from the web. Screenshots of results are below.
var multiparty = require('multiparty');
var http = require('http');
http
.createServer(function (req, res) {
if (req.url === '/upload' && req.method === 'POST') {
console.log('multipart here');
var form = new multiparty.Form();
form.parse(req, function (err, fields, files) {
console.log(require('util').inspect({ fields, files }, false, null, true));
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ bar: true }));
});
return;
}
console.log('here');
// show a file upload form
res.writeHead(200, { 'content-type': 'text/html' });
res.end(
`
<script>
async function doit() {
const data = new FormData();
data.append('str', 'strvalue');
data.append(
'f',
// new File([new Blob(['asdf'], {type : 'text/plain'})], 'filename.txt'),
new File(['foo', 'what', 'the', 'hell'], 'foo.txt', {type: 'text/plain'}),
);
const res = await fetch('http://localhost:3002/upload', {
method: 'POST',
body: data
});
console.log(JSON.stringify(res, null, 4));
}
document.addEventListener('DOMContentLoaded', () => {
document.getElementById('b').addEventListener('click', doit, false)
}, false);
</script>
<button type="button" id="b">upload it</button>
`
);
})
.listen(3002);
From web browser we see the node server logs this, notice file size is 14.
However from react-native we see file size is 0:
I faced the same problem recently while posting an image from a react-native app to a server. However, I was able to make it work by appending the name and type of the file to the formData instance.
Here, the uri argument to uploadImageAsync is passed as a route parameter from the previous screen.
const postShoutHandler = async () => {
setShoutUploadStatus("Started Upload");
const response = await uploadImageAsync(route.params.captures);
const uploadResult = await response.json();
if (uploadResult === "Upload successful") {
setShoutUploadStatus("Success");
navigation.navigate("Home");
} else {
setShoutUploadStatus("Failed");
}
};
/* <--Upload image function --> */
const uploadImageAsync = (uri: string) => {
const apiUrl = "https://www.yourserver.com/image";
let uriParts = uri.split(".");
let fileType = uriParts[uriParts.length - 1];
let formData = new FormData();
formData.append("img", {
uri,
name: `photo.${fileType}`,
type: `image/${fileType}`,
});
formData.append("description", "HEY");
let options = {
method: "POST",
body: formData,
headers: {
Accept: "application/json",
"Content-Type": "multipart/form-data",
Authorization: "Bearer " + accessToken,
},
};
return fetch(apiUrl, options);
};
/* <--Upload image function --> */
Here is the Image configuration.
const photoData = await camera.takePictureAsync({
base64: true,
exif: false,
});

TypeError: Cannot read property 'name' of null NodeJs

I am learning and experimenting on NodeJs. I am using request-promise to call another api from NodeJs. I am using form-data to create a form and send it to another api.
My snippet:
const requestPromise = require('request-promise');
const FormData = require('form-data');
....
var sendToAPI = async (fileObjBuffer, myId, timestamp) => {
let formData = new FormData();
formData.append('fileData', fileObjBuffer);
formData.append('myId', myId);
formData.append('fileName', timestamp);
let options = {
method: 'POST',
uri: '<URL>',
formData: formData,
headers: {
'Access-Control-Allow-Origin': '*',
'enctype': 'multipart/form-data',
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS,DELETE,PUT'
},
json: true
};
try {
let apiResult = await requestPromise(options).promise();
console.log('\n\napiResult: ', apiResult);
} catch (error) {
console.log('error in sending to api: ',error);
}
}
var fetchAllData = async () => {
let query = 'select * from demo_db.demo_table;';
let fileObject = "";
var result;
try {
//cassandra query
result = await client.execute(query, [], { prepare: true });
} catch (error) {
console.log('error in fetching data from Cassandra: ',error);
}
result.rows.forEach(resultObj => {
fileObject = fileObject +resultObj['room_id'] +":"+resultObj['message_id']+":"+resultObj['original_message'] +":"+resultObj['send_date'] +":"+ resultObj['sender'] +"%";
});
let fileObjBuffer = new Buffer(fileObject);
let myId = uuidv4();
let timestamp = date.format(new Date(), 'YYYYMMMDDhhmmss', false);
sendToAPI(fileObjBuffer,myId,timestamp);
}
My error:
error in sending to api: TypeError: Cannot read property 'name' of null
at FormData._getContentDisposition (/home/bhushan/NodeJS-Scheduler/node_modules/request/node_modules/form-data/lib/form_data.js:226:40)
at FormData._multiPartHeader (/home/bhushan/NodeJS-Scheduler/node_modules/request/node_modules/form-data/lib/form_data.js:177:33)
at FormData.append (/home/bhushan/NodeJS-Scheduler/node_modules/request/node_modules/form-data/lib/form_data.js:70:21)
at appendFormValue (/home/bhushan/NodeJS-Scheduler/node_modules/request/request.js:326:21)
at Request.init (/home/bhushan/NodeJS-Scheduler/node_modules/request/request.js:337:11)
at Request.RP$initInterceptor [as init] (/home/bhushan/NodeJS-Scheduler/node_modules/request-promise-core/configure/request2.js:45:29)
at new Request (/home/bhushan/NodeJS-Scheduler/node_modules/request/request.js:127:8)
at request (/home/bhushan/NodeJS-Scheduler/node_modules/request/index.js:53:10)
at sendToAPI (/home/bhushan/NodeJS-Scheduler/schedulerTest.js:52:25)
at fetchAllData (/home/bhushan/NodeJS-Scheduler/schedulerTest.js:95:2)
at process._tickCallback (internal/process/next_tick.js:68:7)
Please help me to solve this issue.
request-promise internally handles form-data. Therefore no need to use form-data explicitly. Instead I made normal object in following way:
var formData = {
fileData: {
value: fileObjBuffer,
filename: timestamp
},
fileName: timestamp,
myId: myId
}
This issue arises because in the formData key of requestPromise method you do not need to actually pass formData but an object.
request-promise internally creates formData from the object passed.
let formData = {
fileData: fileObjBuffer,
fileName: timestamp,
myId: myId
}
let options = {
method: 'POST',
uri: '<URL>',
formData: formData, // formData here is an object
headers: {
'Access-Control-Allow-Origin': '*',
'enctype': 'multipart/form-data',
'Content-Type': 'application/json',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS,DELETE,PUT'
},
json: true
};
I did not find any good solution. So, I have added a function which will test if the value is empty or not. if it is empty then don't add it into the formData object else add it.
const FormData = z.require('form-data');
const form = new FormData();
const addFormData = (key, value) => {
// If value is an array
if(Array.isArray(value) && value.length > 0) {
form.append(key, JSON.stringify(value));
return;
}
// If value is string or number
if(value) {
form.append(key, value);
}
// TODO: You can add aditional checks if you would like to verify if it is non-empty object etc
}
addFormData('name', 'Rahul')
addFormData('age', 28)

Nodejs Post attachment to JIRA

I am receiving http POST response OK 200, but I see no file present on JIRA issue. From my research I can understand that it could be some problem with formData I am sending with request. Below is my code:
var newBuffer = new Buffer(req.Payload, 'base64');
var myReadableStreamBuffer = new streamBuffers.ReadableStreamBuffer({
frequency: 10, // in milliseconds.
chunkSize: 2048 // in bytes.
});
// With a buffer
myReadableStreamBuffer.put(newBuffer);
var formData = {
'file': {
'content': myReadableStreamBuffer,
'filename': req.FileName,
'mimeType': req.MimeType //mimeType from JSON
}
};
var options = {
url: 'https://comapny.atlassian.net/rest/api/2/issue/' + req.ReferenceId + '/attachments',
method: "POST",
json: true,
headers: {
'ContentType': 'multipart/form-data',
'Authorization': 'Basic ' + new Buffer(config.jira.jiraUser.userName + ':' + config.jira.jiraUser.password).toString('base64'),
'X-Atlassian-Token': 'nocheck'
},
formData: JSON.stringify(formData)
};
request(options,
function (error, response, body) {
if (error) {
errorlog.error(`Error Message : PostAttachmentToCSMS : ${error}`);
return response.statusCode;
}
else {
successlog.info(`Attachment posted for issue Key: ${req.ReferenceId} ${response.statusMessage}`);
return response.statusCode;
}
});
I can write file from myReadableStreamBuffer, so that seems ok. Please help me to identify the problem. Many thanks!
After spending some more time on it, I have found the correct format for formData:
var newBuffer = new Buffer(req.Payload, 'base64');
var formData = {
file: {
value: newBuffer,
options: {
filename: req.FileName,
contentType: req.MimeType
}
}
};
For whom like me getting errors with this API.
After struggling so many hrs on this thing, I finally found this works like a charm. I've got "XSRF check failed" 403/404 error message before writing this code.
// Don't change the structure of formData.
const formData = {
file: {
value: fs.createReadStream(filepath),
options: {
filename: filename,
contentType: "multipart/form-data"
}
}
};
const header = {
"Authentication": "Basic xxx",
// ** IMPORTANT **
// "Use of the 'nocheck' value for X-Atlassian-Token
// has been deprecated since rest 3.0.0.
// Please use a value of 'no-check' instead."
"X-Atlassian-Token": "no-check",
"Content-Type": "multipart/form-data"
}
const options = {
url: "http://[your_jira_server]/rest/api/2/issue/[issueId]/attachments",
headers: header,
method: "POST",
formData: formData
};
const req = request(options, function(err, httpResponse, body) {
whatever_you_want;
};
I was able to post attachments to JIRA using axios in the following way:
const axios = require('axios');
const FormData = require('form-data')
const fs = require('fs');
const url = 'http://[your_jira_server]/rest/api/2/issue/[issueId]/attachments';
let data = new FormData();
data.append('file', fs.createReadStream('put image path here'));
var config = {
method: 'post',
url: url,
headers: {
'X-Atlassian-Token': 'no-check',
'Authorization': 'Basic',
...data.getHeaders()
},
data: data,
auth: {
username: '',
password: ''
}
};
axios(config)
.then(function (response) {
res.send({
JSON.stringify(response.data, 0, 2)
});
})
.catch(function (error) {
console.log(error);
});

Implementation of Box View API in Node.js gives 202

I'm currently building a node implementation of the new Box View API and I'm getting a 202 everytime I upload a document and retrieve a session. However, if I do a curl call, I dont get a 202. Is there anyone else experiencing this issue?
Here is my Ember Implementation:
export default Ember.View.extend({
document: null,
documentID: null,
session: null,
sessionID: null,
getDocument: function() {
var self = this;
return Ember.$.ajax({
url: 'http://localhost:3000/doc',
type: 'POST',
contentType: 'application/json',
dataType: 'json',
data: JSON.stringify({ "docURL": this.textField.value })
}).then(function(response){
self.set('document', response);
self.set('documentID', response.document_id);
});
},
getSession: function() {
var self = this;
return Ember.$.ajax({
url: 'http://localhost:3000/sess/',
type: 'POST',
contentType: 'application/json',
data: JSON.stringify({ "docID": this.get('documentID') })
}).
then(function(response) {
self.set('session', response);
self.set('sessionID', response.session_id);
});
}.observes('documentID'),
actions: {
upload: function() {
this.getDocument();
}
}
});
Here is my node implementation:
var https = require('https');
var requestCount = 0;
exports.doc = function(req, res) {
var docURL = req.body.docURL;
var httpReq;
var opts = {
hostname: 'view-api.box.com',
path: '/1/documents',
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Authorization': 'Token <my token>' }
};
res.header('Access-Control-Allow-Origin', '*');
httpReq = https.request(opts, function(preq, pres) {
var output = '';
preq.on('data', function(chunk) {
output += chunk;
});
preq.on('end', function() {
output = JSON.parse(output);
output.document_id = output.id;
delete output.id;
res.json(output);
});
});
httpReq.write(JSON.stringify({ "url": docURL }));
httpReq.end();
};
exports.sess = getSession;
function getSession(req, res) {
var docID = req.body.docID;
var httpReq;
var opts = {
hostname: 'view-api.box.com',
path: '/1/sessions',
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Authorization': 'Token <my token>' }
};
res.header('Access-Control-Allow-Origin', '*');
httpReq = https.request(opts, function(preq, pres) {
var output = '';
if(preq.statusCode === 202) {
setTimeout(function() {
console.log('Retrying Request :: Count(' + requestCount + ')');
if (requestCount >= 3) {
res.json({ 'error': "Retry Again.", 'time': preq.headers['retry-after'] });
return;
}
getSession(req, res);
requestCount += 1;
}, 2000);
return;
}
preq.on('data', function(chunk) {
output += chunk;
});
preq.on('end', function() {
console.log('Successful Request!');
requestCount = 0;
output = JSON.parse(output);
output.session_id = output.id;
delete output.id;
res.json(output);
});
});
httpReq.write(JSON.stringify({ "document_id": docID, "duration": 60 }));
httpReq.end();
}
But now I'm getting this error. Is there a UI that can help me remove the uploaded documents?
{
"message": "You have exceeded your document upload rate-limit.",
"type": "error",
"request_id": "49f8b480b304496987b8cf21f5850c90"
}
You have the correct approach with retry-after for sessions.
The rate limiting you're seeing is actually due to the 2-document rate limit in place for the View API beta. See the FAQ for more info.
You can use webhooks to be notified when your documents finish converting (allowing you to upload another), so you don't have to poll the /documents endpoint for status.

Resources