Hey guys I am trying to pull some data from a http request in a firebase pub sub function. Here is my current function I have ommited my api key
async function requestPromise(path: string) {
return new Promise((resolve, reject) => {
const options = {
hostname: "pro-api.coinmarketcap.com",
path: "/v1/cryptocurrency/listings/latest?start=1&limit=10&convert=USD",
method:"GET",
headers: {
Authorization: "Omitted",
},
};
http.get(options, (resp) => {
let data = "";
resp.on("data", (chunk) => {
data += chunk;
});
resp.on("end", () => {
resolve(data);
});
}).on("error", (error) => {
reject(error);
});
});
}
exports.updateCoinPrices = functions.pubsub.schedule("every 5 minutes")
.onRun(async (context) => {
await (async function () {
try {
await requestPromise("https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?start=query=limit=").then((data) => {
console.log("the data is")
console.log(data)
});
} catch (error) {
console.error(error);
}
});
return null;
});
Here is what I get from firebase data is
and here is what the data should be coming in as any ideas ?
Related
I have my function that create requests to get recents tweets by a keyword (I'm using NodeJS), but I need to stop it after 10 tweets, how can I do that? From the twitter api doc I didn't find anything...
Here only mentions the limit but not how to set it
https://developer.twitter.com/en/docs/twitter-api/rate-limits
Here the code:
const rules = [{
value: //keyword
}]
function streamTweets() {
const stream = needle.get(streamURL, {
headers: {
Authorization: `Bearer ${TOKEN}`
}
})
stream.on('data', (data) => {
try {
const json = JSON.parse(data)
console.log(json)
} catch (error) {
}
})
}
(async () => {
let currentRules
try {
currentRules = await getRules()
await deleteRules(currentRules)
await setRules()
} catch (error) {
console.error(error)
process.exit(1)
}
streamTweets()
})()
I am having trouble sending an error to the front end when a csv file is uploaded and the numbers already exist in the database. The backend is logging an error that the primary key value already exist, but the code I have written tells the front end that the file uploaded just fine.
Code snippet:
router.post('/:program/upload', upload.single('testUpload'), (req, res, next) => {
try {
CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json({
message: 'File Uploaded Just fine :)'
});
} catch (error) {
return res.status(500).json({
message: error
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', () => {
csvData.forEach(item => {
queries.upload(program, item)
.then(() => {
console.log('QR Code Added: ' + item);
}).catch((err) => {
console.log(`oopsie: ${err}`);
});
})
});
stream.pipe(csvStream);
}
Pretty confident the issue is with my poor understanding of promises.
As expected, I wasn't handling my promises correctly. I've updated the code a bit and now it responds with 2 arrays of successful uploads and errored uploads.
router.post('/:program/upload', upload.single('testUpload'), async (req, res, next) => {
try {
const result = await CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json(result);
}
catch (error) {
return res.status(500).json({
message: error,
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
return new Promise((resolve) => {
const results = {
seccess: [],
error: [],
}
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', async () => {
await Promise.all(
csvData.map(async (item) => {
try{
await queries.upload(program, item);
results.success.push(item);
console.log('QR Code Added: ' + item);
}
catch (error) {
console.log(`oopsie: ${error}`)
results.error.push(item);
}
})
)
resolve(results);
});
stream.pipe(csvStream);
})
}
https://github.com/strawberries73/OSD600-Journey-on-rocky-trails
I need the data converted to show URLs only for my Link Check to work. I have tried another solution but the new code solution deletes the conversion before it gets read. If there is a way to pull the URL's and get them to be written to a different file or the same file, my link checker would work.
// ExitCode
process.on("SIGTERM", () => {
server.close(() => {
console.log("Program is terminated");
});
});
//Flag labeling each URL as a good or bad
const _label = ({
good: "GOOD",
bad: "BAD"
});
//const getTelescope = fetch('http:/') data
async function getTelescopeData(){
//fetching data from local host
fetch("http://localhost:3000/posts").then(response => {
return response.json();
}).then(data => {
console.log(data);
//truncate the data and write to file
fs.truncate("telescopeData.txt", 0, function() {
for(i = 0; i < data.length; i++) {
fetch(`http://localhost:3000${data[i].url}`)
.then(res => {
return res.json();
}).then(telescopeData => {
//Append to telescopeData.txt
fs.appendFile("telescopeData.txt", telescopeData.html,
(err) => {
if(err) {
console.log(err)
.then.process(); //Termination
}
});
});
}
});
}
);
}
getTelescopeData(); //Data fetched
if(process.argv.length==2){
greetingMessage();
console.log("Hello");
}
else{
const filePath=path.join(__dirname,process.argv[2]);
fs.readFile(filePath,"utf-8",(err,data)=>{
if(err){
console.log("Fail to read file",err);
}
else{
const validUrl=data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
validUrl.forEach((url)=>{
fetch(url,{method:"HEAD",timeout:2000})
.then((res)=>{
if(res.status==200)
console.log(res.status,url.green,_label.good.rainbow);
else if(res.status==400||res.status==404)
console.log(res.status,url);
else console.log(res.status,url)
.then.process(); //Termination
})
.catch((error)=>{
console.log("404",url.red, _label.bad.bgRed);
});
});
}
});
}
Output of Data in the console
New Code tried, only it deletes the input after it has been written to file.
// ExitCode
process.on("SIGTERM", () => {
server.close(() => {
console.log("Program is terminated");
});
});
//Flag labeling each URL as a good or bad
const _label = ({
good: "GOOD",
bad: "BAD"
});
//const getTelescope = fetch('http:/') data
async function getTelescopeData() {
//fetching data from local host
fetch("http://localhost:3000/posts").then(response => {
return response.json();
}).then(data => {
console.log(data);
//truncate the data and write to file
fs.truncate("telescopeData.txt", 0, function () {
for (i = 0; i < data.length; i++) {
fetch(`http://localhost:3000${data[i].url}`)
.then(res => {
return res.json();
}).then(telescopeData => {
//Append to telescopeData.txt
fs.appendFile("telescopeData.txt", telescopeData.html,
(err) => {
if (err) {
console.log(err)
.then.process(); //Termination
}
});
});
}
});
// its not reading the file for some reason. File is empty when I add this.
fs.readFile("telescopeData.txt", "utf-8", (err, data) => {
if (err) {
console.log("Fail to read file", err);
}
else {
console.log(data);
console.log("^^^");
const validUrl = data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
console.log(validUrl);
console.log("^^^");
validUrl.forEach((url) => {
fetch(url, { method: "HEAD", timeout: 2000 })
.then((res) => {
if (res.status == 200)
console.log(res.status, url.green, _label.good.rainbow);
else if (res.status == 400 || res.status == 404)
console.log(res.status, url);
else console.log(res.status, url)
.then.process(); //Termination
})
.catch((error) => {
console.log("404", url.red, _label.bad.bgRed);
});
});
}
});
} // then ends
);
}
if (process.argv.length == 2) {
greetingMessage();
console.log("Hello");
} else if (process.argv[2] == "--telescope") {
console.log("Telescope");
getTelescopeData();
}
else {
const filePath = path.join(__dirname, process.argv[2]);
fs.readFile(filePath, "utf-8", (err, data) => {
if (err) {
console.log("Fail to read file", err);
}
else {
const validUrl = data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
validUrl.forEach((url) => {
fetch(url, { method: "HEAD", timeout: 2000 })
.then((res) => {
if (res.status == 200)
console.log(res.status, url.green, _label.good.rainbow);
else if (res.status == 400 || res.status == 404)
console.log(res.status, url);
else console.log(res.status, url)
.then.process(); //Termination
})
.catch((error) => {
console.log("404", url.red, _label.bad.bgRed);
});
});
}
});
}
async function getTelescopeData(body, url){
//fetching data from local host
const localRegex= /https?:\/\/localhost:[0-9]*/;
const baseurl=url.match(localRegex);
return JSON.parse(body).map((i)=>{return baseurl[0]+i.url;
})
}
I have a class :
class RequestTimeout {
constructor(timeoutMilliseconds) {
this.timeoutMilliseconds = timeoutMilliseconds;
this.timeoutID = undefined;
}
start() {
return new Promise((resolve, reject) => {
this.timeoutID = setTimeout(() => reject(new Error(`Request attempt exceeded timeout of ${this.timeoutMilliseconds}`)), this.timeoutMilliseconds);
});
}
clear() {
if (this.timeoutID) clearTimeout(this.timeoutID);
}
}
module.exports = RequestTimeout;
This class is used in a module:
const RequestTimeout = require('./request-timeout');
function Request() {
...
async function withTimeout(request, ms) {
const timeout = new RequestTimeout(ms);
return Promise.race([
request(),
timeout.start(),
])
.then(
response => {
timeout.clear();
return response;
},
err => {
timeout.clear();
throw err;
}
);
}
...
}
How do i mock RequestTimeout in a test using Request? For example:
it('should clear the timeout following a successful response', async () => {
nock('http://example.com')
.get('/')
.reply(200, { example: true });
const response = await request.get({ ...baseOptions });
expect(response.example).toEqual(true);
});
// MOCK
let mockGetTimeOutId = jest.fn();
jest.mock('../request-timeout', () => {
return jest.fn().mockImplementation((ms) => {
let timeoutId = undefined;
return {
start: () => new Promise((resolve, reject) => {
timeoutId = setTimeout(() => reject(), ms);
}),
clear: () => mockGetTimeOutId(timeoutId),
}
})
});
// TEST
it('should clear the timeout following a successful response', async () => {
nock('http://example.com')
.get('/')
.reply(200, { example: true });
expect(mockGetTimeOutId).toHaveBeenCalledTimes(0);
const response = await request.get({ ...baseOptions });
expect(mockGetTimeOutId).toHaveBeenCalledTimes(1);
expect(response.example).toEqual(true);
});
I'm developing a simple chatbot on Amazon Alexa. The idea is to know if an item is on menu at a particular store.
function httpGet() {
return new Promise(((resolve, reject) => {
const options = {
headers: {
'auth_key':'xxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx'
}
};
http.get("http://example.com/lookup/api/getitemlist?item=cake&storeid=50", options, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
resolve(JSON.parse(data));
});
}).on('error', (err) => {
reject(err);
});
}));
}
I'm following documentation on nodejs docs
The API works well in postman where the auth_key is passed in header.
Here's the error from amazon cloudwatch
The API responds with error message when the auth_key isnt present. Am I missing something? From reading the documentation. I thought this would work.
GetItemIntentHandler. I have to write more to handle the response. For now I'm only logging it. This is where I call the function httpGet();
const GetItemIntentHandler = {
canHandle(handlerInput) {
return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
&& Alexa.getIntentName(handlerInput.requestEnvelope) === 'GetItemIntent';
},
async handle(handlerInput){
const item = handlerInput.requestEnvelope.request.intent.slots.Item.value;
const response = await httpGet();
console.log("response reached us");
console.log(response);
return handlerInput.responseBuilder
.speak(` ${item}`)
.reprompt("What would you like?")
.getResponse();
}
};
I used the http.get(options[, callback]) method to get around.
Updated httpGet function I'm using now.
function httpGet() {
return new Promise(((resolve, reject) => {
var options = {
host: 'example.com',
path: '/lookup/api/getitemlist?item=cake&storeid=50',
port: null,
headers: {
'auth_key':'xxxxxxx-xxxxx-xxxxx-xxxxx-xxxxxxxxxx'
}
};
http.get(options, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
resolve(JSON.parse(data));
});
}).on('error', (err) => {
reject(err);
});
}));
}