I am having issues with the amazon advertising api. I am using a nodejs library (https://github.com/humanstupidity/amazon-advertising-api) to fetch a report from amazon which returns me a body that seems to be in binary but I am not able to make it readable in JSON.
I tried the request in postman and it returns the same binary response. I have to download and unpacked the file from postman to make it readable but I am somehow not able to make it work in nodeJS.
This is the part inside the library which is important:
const request = require('request');
const util = require('util');
const requestPromise = util.promisify(request);
const postRequestPromise = util.promisify(request.post);
const getRequestPromise = util.promisify(request.get);
module.exports = class AdvertisingClient {
async getReport(reportId) {
while (true) {
let reportRequest = JSON.parse(await this._operation(`sp/reports/${reportId}`));
if (reportRequest.status === 'SUCCESS') {
return this._download(reportRequest.location);
}
await sleep(500);
}
}
_download(location, auth = true)
{
let headers = {}
if (auth) {
headers.Authorization = 'Bearer ' + this.options.accessToken;
}
if (this.options.profileId) {
headers['Amazon-Advertising-API-Scope'] = this.options.profileId;
}
let requestOptions = {
url: location,
headers: headers,
followRedirect: false,
gzip: true
}
return this._executeRequest(requestOptions);
}
async _executeRequest(requestOptions){
let response = await getRequestPromise(requestOptions);
if (response.statusCode == 307) {
let finalResult = await this._download(response.headers.location, false);
return finalResult;
}
let result={
success: false,
code: response.statusCode,
response: response.body,
requestId: 0
};
console.log(result);
if (!(response.statusCode < 400 && response.statusCode >= 200)) {
let data = JSON.parse(response.body);
if (data && data.requestId) {
requestId = data.requestId;
}
result = false;
} else {
result.success = true;
}
return result
}
async _operation(_interface, data, method = 'GET') {
let url = `https://${this.endpoint}/v2/${_interface}`;
let headers = {
'Authorization': 'Bearer ' + this.options.accessToken,
'Amazon-Advertising-API-ClientId': this.options.clientId,
'Content-Type': 'application/json',
}
if (this.options.profileId) {
headers['Amazon-Advertising-API-Scope'] = this.options.profileId;
}
let requestOptions={
url: url,
headers: headers,
method: method,
gzip: true,
}
if (method=="GET") {
requestOptions.qs=data;
}else{
requestOptions.json=data;
}
let response = await requestPromise(requestOptions)
let resData = response.body;
if (!resData.error) {
return resData;
} else {
throw resData.error;
}
}
}
I call the code via the function:
let reportResult = await client.getReport(reportID);
And this is the result I am getting:
{
success: false,
code: 200,
response: '\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\u0000���j�#\u0010\u0006�{�B<�\u0010�������P�\u0016�[���M�F\r\u001a\u000b����\u0002y�nӖ�!�7����7�O\u001faRT�p�Fa\u0012g�8}�g�p\u0002H\\\u0019�H�����.M^��4ۗ���"�Z�\u0016q��I8��MR:��y3�\u001a/�F�P3\u0004�C�\u0017�t\\\u0017Y\u001a��\u001b��v��Hf��(`��0�+֧�g�8��]\u000b\u0019\u0013�\u001d-I\u001b.%��g��`���$���\u001f(��f~�tʐ�`H�/C���\u0011�>\u0014M\u0000�Fw\f��\u001b\u0018���\f�71`.���]Ev9[4\r1��5��!�˥�i\u0018�m�\u001c�R�=3��I�VL(����t�~sm_��\\i!\u0005\n' +
'�٠�aU���=��e�\u0007KW�Ypk�z(��Q��\u0003\u0013$`�em\u0010�\u0018=�d�����}���y3��\u000b.��=9\u0004\u0000\u0000',
requestId: 0
}
I played around with zlib as well but only get the error Error: unexpected end of file
I am pretty new to nodeJS and really have no idea what the issue is.
Thanks a lot in advance for your help!
The body is a gzip'ed (compressed) JSON. You need to gunzip it first.
If you use amazon-advertising-api-sdk package, it will do this automatically for you.
Related
I am a bit stuck and require your help.
Any idea on how do I change this downloader function (yes, I need to keep this as a function returning a buffer) making sure it download and returns only the first N bytes?
...
async function getDataFromURL(url) {
const options = {
uri: url,
gzip: true,
encoding: null,
method: 'GET',
};
let response = await request(options);
if (response.statusCode == 200) {
return response.body;
} else {
throw response.statusCode + " " + response.statusMessage;
};
};
...
let downloadedData = await getDataFromURL('https://server/file');
Solution thanks to #kmoser
...
async function getDataFromURL(url) {
const options = {
uri: url,
gzip: true,
encoding: null,
method: 'GET',
headers: {
'Range': 'bytes=0-253' // Maximum number of bytes to download
}
};
let response = context.request(options);
if (response.statusCode == 206 || response.statusCode == 200) {
return response.body;
} else {
throw response.statusCode + " " + response.statusMessage;
};
};
...
let downloadedData = await getDataFromURL('https://server/file');
Hi i am creating a shopify api node js script which runs with async and await.Basically i am using this to fetch data form paginated page and i am getting the data properly in console log for each page. the issue is i am unable to make a array in which i get all the data in the end of function.
Here is the code
const fetch = require("node-fetch");
const priceRule = "641166639179"
const totalresult = []
async function findCodeId(price_rule, url=null){
//get(priceRuleId, id)
let urlneww = url ? url : `https://xxxxxxxxxxxx.myshopify.com/admin/api/2020-01/price_rules/${price_rule}/discount_codes.json`;
await fetch(urlneww, {
method: 'GET',
headers: {
"Content-Type": "application/json",
"Authorization": "Basic xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx="
}
})
//.then(response => response.json())
.then(result => {
let rrr = result.json() ;
rrr.then((data)=>{
totalresult.push(data)
console.log(data)
}).catch(error => error);
if(result.headers.get('link').includes("next")){
let str = result.headers.get('link');
let arrStr = str.split('<').pop().split('>')[0]; // returns 'two'
//console.log(arrStr);
findCodeId(priceRule,arrStr);
//console.log(totalresult)
}else{
}
//return totalresult
})
.catch(error => console.log('error', error));
}
findCodeId(priceRule)
i am trying to push the data in totalresult constant but it is not working. Could you please suggest how can i do this so that on each result it pushes the data in the totalresult and at end of function i got all result data collected in totalresult.
You are mixing promise/async-await style which is making it complex. Also you are not awaiting while recursively calling function. Try this
const fetch = require("node-fetch");
const priceRule = "641166639179";
const totalresult = [];
async function findCodeId(price_rule, url = null) {
try {
// get(priceRuleId, id)
const urlneww = url
? url
: `https://xxxxxxxxxxxx.myshopify.com/admin/api/2020-01/price_rules/${price_rule}/discount_codes.json`;
const result = await fetch(urlneww, {
"method": "GET",
"headers": {
"Content-Type": "application/json",
"Authorization": "Basic xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx="
}
});
const data = await result.json();
totalresult.push(data);
console.log(data);
if (result.headers.get("link").includes("next")) {
const str = result.headers.get("link");
const arrStr = str
.split("<")
.pop()
.split(">")[0]; // returns 'two'
// console.log(arrStr);
await findCodeId(priceRule, arrStr);
// console.log(totalresult)
} else {
console.log(totalresult);
}
} catch (Err) {
console.error(Err);
}
}
findCodeId(priceRule);
I using that API to create resumable Upload in Drive.
It worked but i can't receive response data from Promise
Here is my code:
var fs = require('fs');
var request = require('request');
var mime = require('mime');
var _ = require("underscore");
class resumableUpload {
constructor(credentials,token,options={}){
this.credentials = credentials; // object from json credentials
this.token = token; // object from token credentials
this.new_token = {};
this.expiry_token_time = 0;
this.options = Object.assign({
size:256*1024,
retry:-1
},options)
}
refresh_token(){
return new Promise((ok,faild)=>{
var now = new Date().getTime();
if(now>this.expiry_token_time){
request.post("https://www.googleapis.com/oauth2/v4/token",{
form:{
refresh_token:this.token.refresh_token,
client_id:this.credentials.client_id,
client_secret:this.credentials.client_secret,
redirect_uri:this.credentials.redirect_uris[0],
grant_type:'refresh_token'
}
},(err,httpResponse,body)=>{
if(err) faild(err);
this.new_token = JSON.parse(body);
this.expiry_token_time = now + this.new_token.expires_in*1000;
ok(this.new_token.access_token);
})
}else{
ok(this.new_token.access_token);
}
})
}
/*
Upload Resumable
file:{
name:'name of file',
path:'path/to/file'
}
*/
upload(file){
return new Promise((ok,faild)=>{
this.refresh_token().then((access_token)=>{
console.log(access_token);
this.file_len = fs.statSync(file.path).size;
var form = {};
if(!_.isEmpty(file.name)) form.name = file.name;
if(!_.isEmpty(file.parents)) form.parents = [file.parents];
request.post({
uri:'https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable',
headers:{
'Authorization':`Bearer ${access_token}`,
'Content-Type':'application/json; charset=UTF-8',
'X-Upload-Content-Type':mime.getType(file.path),
'X-Upload-Content-Length': this.file_len,
'Content-Length':new Buffer(JSON.stringify(form)).length
},
body:JSON.stringify(form)
},(err,res,body)=>{
if(err || _.isEmpty(res.headers.location)){
if ((this.retry > 0) || (this.retry <= -1)) {
this.retry--;
return this.upload(file); // retry
} else {
//console.log(body);
faild("Over retry");
}
}else{
var location = res.headers.location;
//console.log("URL UPLOAD: ",location);
this.sendUpload(location,file).then((data)=>{
console.log("UPLOAD DONE!!!");
ok(data);
}).catch((err)=>{
console.log("ERROR: ",err);
faild(err);
});
}
})
}).catch(faild)
})
}
sendUpload(api,file,start=0){
return new Promise((ok,faild)=>{
this.refresh_token().then((access_token)=>{
var end = parseInt(start+this.options.size-1);
if(start>=this.file_len) ok("DONE");
if(end>=this.file_len) end = this.file_len - 1;
var headers = {
'Authorization':`Bearer ${access_token}`,
'Content-Length': parseInt(end - start + 1),
'Content-Range':`bytes ${start}-${end}/${this.file_len}`
}
var uploadPipe = fs.createReadStream(file.path, {
start: start,
end: end
})
uploadPipe.pipe(request.put({uri:api,headers:headers}).on('response',(res)=>{
if(res.statusCode == 308){
//console.log(res.headers);
var new_start = parseInt(res.headers.range.split('-')[1]);
console.log("Percent: "+(new_start/this.file_len*100).toFixed(2)+" %");
return this.sendUpload(api,file,new_start+1);
}else if(res.statusCode == 200 || res.statusCode == 201){
console.log("percent: 100 %");
//ok({status:"OK"});
}
}).on('error',(err)=>{
if ((this.retry > 0) || (this.retry <= -1)) {
this.retry--;
console.log("Retry Again");
return this.sendUpload(api,file,start);
}else{
faild("Over retry");
}
}).on('data',(data)=>{
if(!_.isEmpty(data)){
console.log(data.toString());
ok(data.toString());
}
}))
}).catch(faild)
})
}
}
module.exports = resumableUpload;
In that code - sendUpload function, I split my file to many chunk and upload it in to drive.
when I upload the last chunk, server will response statusCode is 200 or 201.
It console.log "percent: 100 %" and received data from that response.
My data of response like it:
{
"kind": "drive#file",
"id": "xxxxxxxxxxxxxxxxxxxxxxxxx",
"name": "test-video",
"mimeType": "video/mp4"
}
But, the Resolve (ok function) not call, cause in upload function, i can't receive it in then function. it mean i can't see "UPLOAD DONE!!!" in screen.
Resloved: Thanks for Roamer-1888
In Pipe, return a promise not work. I change return this.sendUpload(...) to this.sendUpload(...).then(ok,faild) and it worked.
I need to make a HTTP GET request to foreign server to start receiving events. After request I immediately get multipart/x-mixed-replace response. When an event occurs, it's sent as XML message together with boundary indicating the end of this part.
Now I must implement it in Node.js. With normal request I use node-rest-client, call its get() method and put my logic in method's callback. Trouble is that callback is executed only when response is finished and with multipart/x-mixed-replace it isn't until connection closes.
Is there some other NPM module that does the trick? I searched NPM registry but results I found seem unappropriate to the task. Or is it better to do it in pure Node? I so, please provide an example.
Here is my own implementation:
const webStream = {
get: function (url, callback) {
let webClient;
if (url.startsWith("http://")) {
webClient = require("http");
} else if (url.startsWith("https://")) {
webClient = require("https");
} else {
throw "Unsupported protocol.";
}
let clientRequest = webClient.get(url, function (response) {
let context = {
url: url,
boundary: "",
contentType: "",
contentLength: 0
};
let headersCompleted = false;
let bodyCompleted = false;
let buffer = null;
let receivedBodyChunk = 0;
response.on("data", function (chunk) {
if (!headersCompleted) {
let headers = chunk.toString().split(/\r?\n/);
context.boundary = headers[0].substring(2);
context.contentType = headers[1].split(":")[1].trim();
context.contentLength = parseInt(headers[2].split(":")[1]);
buffer = Buffer.alloc(context.contentLength);
headersCompleted = true;
} else {
if (!bodyCompleted) {
if (receivedBodyChunk < context.contentLength) {
chunk.copy(buffer, receivedBodyChunk, 0, chunk.byteLength);
receivedBodyChunk += chunk.byteLength;
if (receivedBodyChunk === context.contentLength) {
bodyCompleted = true;
}
}
}
if (bodyCompleted) {
callback(buffer, context);
headersCompleted = false;
bodyCompleted = false;
buffer = null;
receivedBodyChunk = 0;
}
}
});
});
return {
url: url,
handler: clientRequest,
on: function (type, listener) {
clientRequest.on(type, listener);
},
abort: function () {
clientRequest.abort();
}
};
}
};
let stream = webStream.get("http://127.0.0.1:8090/", function (data, context) {
// data: Received content (Buffer)
// context: { url, boundary, contentType, contentLength }
// TODO: Do something here...
});
// stream.abort();
// stream.on("error", function(e) {
// console.log("Error: " + e.message);
// });
I'm making an http request using the response node library and I am trying to call it recursively (If the user made a commit on one day, check the previous day. If not, count up all the days to get the streak).
The problem is that the line
const githubResponse = await request(options);
Spits out the error
Unexpected token o in JSON at position 1
await request(options) doesn't seem to return the JSON GitHub API response I am expecting, but instead githubResponse seems to be an object that I can't use. I'm guessing I'm using async/await improperly, but I'm not sure how to fix it.
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await request(options)
// I get an error on the next line
if (JSON.parse(githubResponse).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
UPDATE: It seems like this is not a promise, so I need to format this differently (as a callback). When I try this:
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
request(options, async function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
if (JSON.parse(body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
});
}
The line
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
becomes the problem as streakCounter is undefined, making the log NaN.
As said in the comments request uses callbacks instead of returning a promise and you dont really need to promisify it by yourself since there's already a pacakge for that called request-promise.
Using it in your code should directly work out of the box with async/await
I used promisify example from here to convert it to this and it worked!
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await promisify(request)(options);
if (JSON.parse(githubResponse.body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
function promisify(fn) {
return function (...args) {
return new Promise((resolve, reject) => {
fn(...args, (err, result) => {
if (err) return reject(err);
resolve(result);
});
});
};
};
If you are upgraded to Node 8 LTS, then native util.promisfy can be used as below:
const { promisify } = require('util')
const request = promisify(require('request'))
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview',
'json':true
}
};
try{
const githubResponse = await request(options);
if (githubResponse.body.total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
catch(err){
console.error(err)
return 0;
}
}
Using json:true in the options, will reduce another step to parse as the response will be in JSON format.