I am retrieving a csv-file from a cloud based storage as res.text and need to convert it to json.
I am wondering if I should do the parsing in the return of fetchUrl or if I should do it in the route (res.send)?
const fetchUrl = async () => {
const URL_1 = 'https://file.csv'
const res = await fetch(URL_1)
return res.text()
}
router.get('/data', async (req, res, next) => {
try {
const getAllData = await fetchUrl();
console.log(getAllData, 'fetching?');
res.send(getAllData);
} catch (err) {
next(err);
//res.send({ message: err })
// res.status(404).send(err)
console.log(err)
}
})
I made use of a custom function, that converts it to json in the 2nd .then. So that I won't have to fetch upon every call to the endpoint since the data does not change.
Like so:
let getAllData
fetch('https://file.csv')
.then(res => res.text())
.then(data => {
getAllData = csvToJSON(data)
getAllData.forEach((item) => {
item.startTime = new Date(item.startTime)
})
})
Related
I know for sure that my pullData module is getting the data back from the file read but the function calling it, though it has an await, is not getting the data.
This is the module (./initialise.js) that reads the data:
const fs = require('fs');
const getData = () => {
return new Promise((resolve, reject) => {
fs.readFile('./Sybernika.txt',
{ encoding: 'utf8', flag: 'r' },
function (err, data) {
if (err)
reject(err);
else
resolve(data);
});
});
};
module.exports = {getData};
And this is where it gets called (app.js):
const init = require('./initialise');
const pullData = async () => {
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
const start = async() => {
let data = await pullData();
console.log(data);
}
start();
putting 'console.log(data)' just before return(data) in the resolve part of the call shows the data so I know it's being read OK. However, that final console.log shows my data variabkle as being undefined.
Any suggestions?
It's either
const pullData = async () => {
return init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
or
const pullData = async () =>
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
Both versions make sure a promise returned by then/catch is passed down to the caller.
I have been trying to fix this but it gives me this: ReferenceError: json is not defined
this is my code:
const fetch = require('node-fetch');
function makeAFile(text){
fetch("http://bin.shortbin.eu:8080/documents", {
method: "post",
body: 'hey wassup'
})
.then(res => res.json())
.then(json => console.log(json))
.catch(err => console.log(err))
return json
}
console.log(makeAFile('nope'))
You need to understand, how async call works.
const fetch = require('node-fetch');
function makeAFile(text){
fetch("http://bin.shortbin.eu:8080/documents", { // this is async call
method: "post", // might give result later in time
body: 'hey wassup'
})
.then(res => res.json())
.then(json => console.log(json))
.catch(err => console.log(err))
return json // JSON is not defined and will be return just after 'Fetch' is called which is 'undefined'
console.log(makeAFile('nope'));
To make your code sync, you should use async await like this:
const fetch = require('node-fetch');
async function makeAFile(text){
let res = await fetch("http://bin.shortbin.eu:8080/documents", { // await helps you to wait here to make your code sync
method: "post",
body: 'hey wassup'
})
const json = res.json()
return json ;
}
try {
const response = await makeAFile('nope')); //use try catch to catch error
console.log(response);
}
catch (error) {
console.log(`Error ${error}`);
}
You seem to be confused with how to return data from an async call. You do that using callbacks like this:
const fetch = require('node-fetch');
function makeAFile(text){
return fetch("http://bin.shortbin.eu:8080/documents", {
method: "post",
body: text
})
.then(res => res.json())
.catch(err => { console.error(err) })
}
makeAFile('nope').then((result) => console.log(result));
You can read more about it here;
EDIT: Provide a solution using async/await
async function makeAFile (text) {
try {
const res = await fetch("http://bin.shortbin.eu:8080/documents", {
method: "post",
body: text
})
return res.json();
} catch (err) {
console.error(err);
}
}
// Define an anonymous async function to allow await statements inside
(async () => {
// this "await" here is important
const result = await makeAFile('nope');
console.log(result);
// now result is store inside of "result"
})();
You don't have access to json outside then callback. Just make your function async, that way you will have an access to it and outer context code can wait till your function will be resolved or rejected:
const fetch = require('node-fetch');
async function makeAFile(text){
const res = await fetch("http://bin.shortbin.eu:8080/documents", {
method: "post",
body: 'hey wassup'
})
const json = res.json()
console.log(json)
return json
}
and call it like:
try {
const result = await makeAFile('nope')
console.log(result)
} catch(err) {
console.log(err)
}
or
const result = makeAFile('nope').then(result => {
console.log(result)
})
.catch(err => {
console.log(err)
});
const fetch = require('node-fetch');
function makeAFile(text){
jsonOut = ''
fetch("http://bin.shortbin.eu:8080/documents", {
method: "post",
body: 'hey wassup'
})
.then(res => res.json())
.then(json => jsonOut = json)
.catch(err => console.log(err))
return jsonOut
}
console.log(makeAFile('nope'))
you cannot access value inside a promise. Just check this code
I'm trying to access the data array that has the information from the request, but I cannot seem to figure it out.
When I try to display it in my view, I get an undefined cannot read property of length;
const getData = () => {
axios.get("https://www.worldometers.info/coronavirus/")
.then(res => {
const data = [];
const $ = cheerio.load(res.data);
$('.maincounter-number').each((index, element) => {
const numberData = $(element).text();
data[0] = {numberData: numberData};
});
}).catch(err => {
console.log("Error fetching and parsing data: ", err);
});
}
app.get("/", (req, res) => {
const data = getData();;
res.render('index', {title: 'Home', data: data});
});
Pug View
p #{data.dataNumbers}
I've also tried this function as well but I get the same issue
async function scrapeWorldOMeter(){
try{
const worldOMeterResponse = await axios.get("https://www.worldometers.info/coronavirus/");
const data = [];
const $ = cheerio.load(worldOMeterResponse.data);
$('.maincounter-number').each((index, element) => {
const numberData = $(element).text();
data[0] = {numberData: numberData};
return data[0];
});
}
catch(err){
throw new Error(`Can't scrape WorldOMeter ${err}`)
}
}
app.get("/", async(req, res) => {
const data = await scrapeWorldOMeter()
res.render('index', {title: 'Home', data});
});
You can fix your first code snippet by using promises properly,
const getData = () => {
return axios
.get('https://www.worldometers.info/coronavirus/')
.then((res) => {
const data = [];
const $ = cheerio.load(res.data);
$('.maincounter-number').each((index, element) => {
const numberData = $(element).text();
data[0] = { numberData: numberData };
});
return data[0];
})
.catch((err) => {
console.log('Error fetching and parsing data: ', err);
});
};
app.get('/', (req, res) => {
const data = getData().then((res) => {
res.render('index', { title: 'Home', data: data });
});
});
The problem here was that you were not returning the axios promise in getData() and also you were not chaining the response with then in the get route.
Your second code snippet can be fixed by using async/await properly,
async function scrapeWorldOMeter() {
try {
const worldOMeterResponse = await axios.get(
'https://www.worldometers.info/coronavirus/'
);
const data = [];
const $ = cheerio.load(worldOMeterResponse.data);
$('.maincounter-number').each((index, element) => {
const numberData = $(element).text();
data[0] = { numberData: numberData };
});
return data[0];
} catch (err) {
throw new Error(`Can't scrape WorldOMeter ${err}`);
}
}
app.get("/", async(req, res) => {
const data = await scrapeWorldOMeter();
res.render('index', {title: 'Home', data});
});
The problem here was that you were returning data[0] inside the jquery each loop. You simply have to move it to the next line outside the loop
I am trying to set up a route that sends data from a nested promise to my vue app.
But i'm having trouble with the getting data from the nested promises.
i tried using a callback with no success
app.get('/notification', (req, res) => {
const getData = (data) => {
console.log(data)
}
scheduler(data)
})
const scheduler = (callback) => {
sftp
.connect({ credentials })
.then(() => {
return sftp.list(root);
})
.then(async data =>
{
const filteredFile = data.filter(file => {
let currentDate = moment();
let CurrentTimeMinusFive = moment().subtract(5, "minutes");
let allAccessTimes = file.accessTime;
let parsedAccessTimes = moment(allAccessTimes);
let filteredTime = moment(parsedAccessTimes).isBetween(
CurrentTimeMinusFive,
currentDate
);
return filteredTime;
});
for (const file of filteredFile) {
let name = file.name;
let filteredThing;
await sftp
.get(`Inbound/${name}`)
.then(data => {
csv()
.fromString(data.toString())
.subscribe(function (jsonObj) {
return new Promise(function (resolve, reject) {
filteredThing = new Notification(jsonObj);
filteredThing.save()
.then(result => {
console.log(result);
callback(result) **// THIS IS THE RESULT I NEED IN MY FRONT END**
})
.catch(err => {
console.log(err);
});
resolve();
});
});
});
}
})
When i go to localhost/notification i get:
ReferenceError: data is not defined
Thanks in advance!
This is my code for sending data to a database:
app.post('/thanks', function(req, res) {
if (atendees.checkin === req.body.dbstring) {
dbConn.then(client => {
delete req.body._id;
const db = client.db('mydata')
db.collection(atendees.checkin).insertOne(req.body);
})
(...)
This is how I display on the page after clicking on a href link:
app.get('/view-feedbacks', function(req, res) {
dbConn.then(client => {
const db = client.db('mydata')
db.collection(atendees.checkin).find({}).toArray().then(function(feedbacks) {
res.status(200).json(feedbacks);
atendees.checkin = ' '
}).catch(err => {
throw(err);
})
});
});
That works fine. How can I do something similar to display all collections from the database instead of just the individual ones?
This is what I tried to do:
app.get('/view-history', function(req, res) {
dbConn.then(client => {
const db = client.db('mydata')
db.listCollections().toArray().then(function(collInfos) {
res.status(200).json(collInfos);
atendees.checkin = ' '
}).catch(err => {
throw(err);
})
});
});
But it just gives me the name of each collection. I want to show all collections and all of their elements.
Edit: my question is different from this one: MongoDB Show all contents from all collections .I'm trying to do this on express.js, not on the terminal
Edit2: Using db.collection:
app.get('/view-history', function(req, res) {
dbConn.then(client => {
const db = client.db('mydata')
db.collections().then(function(feedbacks) {
res.status(200).json(feedbacks);
atendees.checkin = ' '
}).catch(err => {
throw(err);
})
});
But this gives the error: TypeError: converting circular structure to JSON
With async/await, this could be done:
app.get('/view-history', async (req, res) => {
try {
const client = await dbConn;
const db = client.db('mydata');
let collections = await db.collections();
let documents = await Promise.all(collections.map(async (collection) => {
let documents = await collection.find({}).toArray();
return Promise.resolve([collection.collectionName, documents]); // Retain collectionName
}));
// Format into an object that looks like `collectionName: documents`
let formatted = documents.reduce((obj, collection) => {
obj[collection[0]] = collection[1];
return obj;
}, {});
res.json(formatted);
} catch (e) {
console.error(e);
res.sendStatus(500);
}
});
A Promise-only approach:
app.get('/view-history', (req, res) => {
dbConn.then((client) => {
const db = client.db('mydata');
return db.collections();
}).then((collections) => {
return Promise.all(collections.map((collection) => {
return new Promise((resolve, reject) => {
collection.find({}).toArray().then((documents) => {
resolve([collection.collectionName, documents]);
}).catch(reject);
});
}));
}).then((documents) => {
let formatted = documents.reduce((obj, collection) => {
obj[collection[0]] = collection[1];
return obj;
}, {});
res.json(formatted);
}).catch((e) => {
console.error(e);
res.sendStatus(500);
});
});
The main reason this code is unnecessarily verbose is because instead of just returning a big array filled with arrays of documents, you probably want an object that retains the name of the collection, like so:
{
collection1: [...documents...],
collection2: [...documents...],
...
}
Instead of:
[
[...documents...],
[...documents...],
...
]
If you do want just a big array of each collection without caring about the names of the collections, it becomes much simpler:
async/await version:
app.get('/view-history', async (req, res) => {
try {
const client = await dbConn;
const db = client.db('mydata');
let collections = await db.collections();
let documents = await Promise.all(collections.map((collection) => collection.find({}).toArray()));
res.json(documents);
} catch (e) {
console.error(e);
res.sendStatus(500);
}
});
Promise-only version:
app.get('/view-history', (req, res) => {
dbConn.then((client) => {
const db = client.db('mydata');
return db.collections();
}).then((collections) => {
return Promise.all(collections.map((collection) => collection.find({}).toArray()));
}).then((documents) => {
res.json(documents);
}).catch((e) => {
console.error(e);
res.sendStatus(500);
});
});
Have you tried just db.collections()? If that also doesn't give what you need, you might have to invoke db.collection(<name>) on each of the names you get from listCollections.