I'm working on a bot used for Twitch Song Requests. The bot will read the Twitch chat, search for !sr command and get the name of the song. Then, it will search for the song in Spotify, get the URI of the song and add it in the streamer's playlist.
EDIT: I'm sorry if there are any "dumb" code problem (like the callback #ippi notified), I'm really new to programming and Node JS especially.
I now have two functions: one that is searching for the song and writing the received URI in a text file, and one getting the URI from the file. Here's the code:
main code (call of the two functions):
testSong(commandName, accessToken);
let uri = getUri();
console.log(uri);
search for the song:
function testSong(song, accessToken) {
let song1;
let song2;
song1 = song.replace("!sr", "");
song2 = song1.trim();
var uri = "";
axios.get('https://api.spotify.com/v1/search?q=' + encodeURIComponent(song2) + '&type=track&market=CH&limit=1', {
headers: {
Authorization: 'Bearer ' + accessToken
}
})
// handle success
.then(function (response) {
uri = response.data.tracks.items[0].uri;
console.log("yeet")
fs.writeFileSync('conf/uri.txt');
logger.log('info', 'Successfully obtained URI for track ' + song2);
})
// handle error
.catch(function (error) {
logger.log('error', 'Error while accessing Spotify.');
return error;
});
}
get the URI:
function getUri() {
try {
return fs.readFileSync('conf/uri.txt', 'utf-8');
} catch (e) {
logger.log('error', 'Error while reading the URI text file: ' + e.stack);
}
}
I'm having a problem at the reading time. When running the bot for the first time, the uri.txt file is empty.
When I'm sending the first !sr in the Twitch chat, the song is not added in the Spotify playlist beacause it seems that the testSong command is writing to text file AFTER the getUri function read the file.
Even after that, I have to send a new !sr to add the first song, so every request is shifted.
Any idea why this is happening ?
I've read about async functions, but as far as I've understood, it's not what I want because I want the execution of the program to be blocked when I'm writing to the text file, thus the getUri function can then read the current requested song URI, and not be shifted.
EDIT 2: as Felix said, I modified the code as follow:
testSong(commandName, accessToken).then(() => console.log(getUri()));
function testSong(song, accessToken) {
let song1;
let song2;
song1 = song.replace("!sr", "");
song2 = song1.trim();
var uri = "";
return axios.get('https://api.spotify.com/v1/search?q=' + encodeURIComponent(song2) + '&type=track&market=CH&limit=1', {
headers: {
Authorization: 'Bearer ' + accessToken
}
})
// handle success
.then(function (response) {
uri = response.data.tracks.items[0].uri;
console.log("yeet")
fs.writeFileSync('conf/uri.txt', uri, function (err) {
if (err) {
return console.log(err);
} else {
response = true;
}
});
logger.log('info', 'Successfully obtained URI for track ' + song2);
})
// handle error
.catch(function (error) {
logger.log('error', 'Error while accessing Spotify.');
return error;
});
}
Is that correct?
As I have already mentioned in my comments, you are having this problem because you are using promises, i.e. the file will be written some time in the future, after you are trying to read.
And as we discussed, there is no need to use a file to "transfer" the value at all. You can just return the value from testSong (wrapped in a promise):
function testSong(song, accessToken) {
song = song.replace("!sr", "").trim();
return axios.get('https://api.spotify.com/v1/search?q=' + encodeURIComponent(song2) + '&type=track&market=CH&limit=1', {
headers: {
Authorization: 'Bearer ' + accessToken
}
})
// handle success
.then(function (response) {
return response.data.tracks.items[0].uri;
});
// errors should probably be handled by the caller
}
and then:
testSong(commandName, accessToken)
.then(function(uri) {
console.log(uri);
})
.catch(function(error) {
// handle error
});
An async function makes working with promises a bit easier. So you could implement testSong also as
async function testSong(song, accessToken) {
song = song.replace("!sr", "").trim();
const response = await axios.get('https://api.spotify.com/v1/search?q=' + encodeURIComponent(song2) + '&type=track&market=CH&limit=1', {
// ^^^^^
headers: {
Authorization: 'Bearer ' + accessToken
}
});
return response.data.tracks.items[0].uri.
}
Related
I have a NODE.JS api using expressjs that connects to an SQL Server, and I want to use it in an angular project. I make use two files, a route file and a controllers file. My route file is as follows:
module.exports = (app) => {
const UsrContrllr = require('../Controllers/users.controllers');
//1. GET ALL USERS
app.get('/api/users', UsrContrllr.func1);
//2. POST NEW USER
app.post('/api/user/new', UsrContrllr.func2);
};
And my controllers file is given below:
const mssql = require('mssql');
exports.func1 = (req, res) =>
{
// Validate request
console.log(`Fetching RESPONSE`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `SELECT * FROM USERS`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() === '') {
res.send('Oops!!! Required data not found...');
}
else {
// send records as a response
res.send(recordset);
}
};
});
};
exports.func2 = (req, res) =>
{
// Validate request
console.log(`INSERTING RECORD ${req}`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `INSERT INTO GDUSERS (USERCODE, PASSWORD, LANGUAGE, USERCLASS, FIRSTNAME, LASTNAME, CONTACTNO) VALUES ('${req.body.usercode}', '${req.body.password}', 'EN', '0', '${req.body.firstname}', '${req.body.lastname}', '${req.body.contactno}');`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() == '') {
res.send('Oops!!! Required data not found...');
}
else {
// Send records as response
res.send(recordset);
}
};
});
};
The GET request works well, but when I try to run the POST request directly from the angular application, I get an error stating
Cannot GET URL/api/user/new
The angular code in my angular project is:
signup() {
let headers = new Headers({ 'Content-Type': 'application/json' });
let options = new RequestOptions({ headers: headers });
console.log(this.user); //User details come from a form
this.http.post(“URL", this.user, options)
.subscribe(
(err) => {
if(err) console.log(err);
console.log("Success");
});
}
I’m not sure whether the angular code I’m using, is right or not, and I don’t know where I’m going wrong. How does one exactly send a http POST request from an Angular project?
this i the way i handled my user signup with http.post calls. my approach is slightly different when signing up user because i am using a promise instead of observable (which i normally use for my servicecalls). but i will show you both ways.
createUser(user: User): Promise < string > {
const promise = new Promise < string > ((resolve, reject) => {
const userForPost = this.createUserForPost(user);
this.http.post(environment.backendUrl + '/api/user/signup', userForPost, this.config).toPromise < HttpConfig > ()
.then(createdUser => {
}).catch(error => {
console.log(error);
});
});
return promise;
}
here another example with an observable
createForumPost(forumPost: ForumPost) {
this.http.post < { message: string, forumPostId: string } > (environment.backendUrl + '/api/forumPosts', forumPost).subscribe((responseData) => {
const id = responseData.forumPostId;
forumPost.id = id;
});
}
i defined my URL somewhere else and then just use the environment.backedUrl + 'path' to define my path (the same as the path in your backend controller)
this is one of my first answers here on SO. i am sry if it is a bit messy
i hope i was able to help with my examples :)
I am making a node.js application and part of my code requests for data from 193 different urls to download the json data from each url. Here is one of those urls: https://www.gemeentegeschiedenis.nl/gemeentenaam/json/Apeldoorn For the some the downloaded json data is fine and is complete. However towards the end, corruptions happen for some of the files. Part of the data becomes nullified and then there are some that have database errors. I think it has to do with requesting data from so many urls in a short amount of time (which is why I tried the "setTimeout" function (but that doesn't really work)).
function writeToFile(url) {
// get name to make each new file unique
var name = url.split("json/")[1];
var fileStream = fs.createWriteStream(`jsonFiles/${name}.json`);
var options = {
url: `${url}`,
method: 'GET',
headers: {
'Accept': 'application/json',
'Accept-Charset': 'utf-8',
json: true
}
}
//request the data from the site and download to the file.
request.get(options).pipe(fileStream);
}
function getMunicipalityGeoJsonData(req, res) {
//Get all the urls pointing to the JSON data for the province, Gelderland
getGelderlandJsonUrls((err, jsonUrls) => {
//for all those urls, write the data to files.
for (url of jsonUrls) {
console.log(url);
writeToFile(url);
}
})
}
function getGelderlandJsonUrls(callback) {
getMunicipalityJsonUrls("Gelderland", (err, data) => {
jsonUrls = data;
callback(null, jsonUrls);
});
}
function getMunicipalityJsonUrls(provinceName, callback) {
request({ uri: `https://www.gemeentegeschiedenis.nl/provincie/json/${provinceName}` }, (error, response, body) => {
body = JSON.parse(body);
// extracting each json URL from all the municipalities in Gelderland
var jsonUrls = [];
var numberMun = body.length;
for (var i = 0; i < numberMun; i++) {
var url = body[i].uri.naam;
var urlSplit = url.split("gemeentenaam");
var jsonUrl = urlSplit[0] + "gemeentenaam/json" + urlSplit[1];
jsonUrl = jsonUrl.replace("http://", "https://");
jsonUrls.push(jsonUrl);
}
callback(null, jsonUrls);
});
}
The last json data downloaded into the file as an html page with a database error from the url: https://www.gemeentegeschiedenis.nl/gemeentenaam/json/Zutphen which actually just took just under 6 seconds to load up looking at the network tab on Chrome
the 1812 has null for its properties when it should have a bunch of coordinates https://www.gemeentegeschiedenis.nl/gemeentenaam/json/Winssen (took just over a second to load on chrome
I am a noob at node, but please help me fix this issue maybe with some sort of checking if the data is corrupted or something. Thanks for the help in advanced:)
EDIT: I am trying to do up to 200 urls at a time in the for loop.
First off, add proper error handling to getMunicipalityJsonUrls() and to getGelderlandJsonUrls(). This means:
Check err parameter everywhere it's present and propagate the error back to the caller.
Capture possible errors from JSON.parse()
Check http statusCode.
Here's that fixed up code:
function getMunicipalityJsonUrls(provinceName, callback) {
request({ uri: `https://www.gemeentegeschiedenis.nl/provincie/json/${provinceName}` }, (error, response, body) => {
if (err) {
callback(err);
return;
}
if (response.statusCode !== 200) {
callback(new Error(`http status code ${response.statusCode}`));
return;
}
try {
const jsonUrls = JSON.parse(body).map(url => {
let urlSplit = url.split("gemeentenaam");
let jsonUrl = urlSplit[0] + "gemeentenaam/json" + urlSplit[1];
return jsonUrl.replace("http://", "https://");
});
callback(null, jsonUrls);
} catch(e) {
callback(e);
}
});
}
function getGelderlandJsonUrls(callback) {
getMunicipalityJsonUrls("Gelderland", (err, data) => {
if (err) {
callback(err);
} else {
callback(null, data);
}
});
}
Then, in writeToFile(), add error handling and completion monitoring and I chose to wrap it in a promise rather than a plain callback because I want to use it with some utilities that work with promises.
function writeToFile(url) {
return new Promise((resolve, reject) => {
// get name to make each new file unique
var name = url.split("json/")[1];
var fileStream = fs.createWriteStream(`jsonFiles/${name}.json`);
fileStream.on('error', (e) => {
reject(e);
});
var options = {
url: `${url}`,
method: 'GET',
headers: {
'Accept': 'application/json',
'Accept-Charset': 'utf-8',
json: true
}
}
//request the data from the site and download to the file.
request.get(options).pipe(fileStream).on('error', (e) => {
reject(e);
}).on('finish', () => {
resolve(url);
});
});
}
Now, we need to decide how to loop through all the URLs. If any of the urls could ever be attempting to write to the same file (if that's even a remote possibility), then you have to serialize the URLs to prevent them from ever having more than one asynchronous operation trying to write to the same file at the same time because that will just mess up that file. So, if that was the case, you could serialize the writing to the file like this:
// option 1 - serialize writing to files
async function getMunicipalityGeoJsonData(req, res) {
//Get all the urls pointing to the JSON data for the province, Gelderland
getGelderlandJsonUrls((err, jsonUrls) => {
if (err) {
console.log(err);
res.sendStatus(500);
} else {
try {
//for all those urls, write the data to files.
for (url of jsonUrls) {
console.log(url);
await writeToFile(url);
}
res.send("All done");
} catch(e) {
console.log(e);
res.sendStatus(500);
}
}
});
}
If you are absolutely sure that none of these URLs will ever cause writing to the same file, then you can run N of them at a time where you determine what the lowest value of N is that gets you decent performance. Higher values of N consume more peak resources (memory and file handles). Lower values of N run less things in parallel. If the target hostnames are all the same server, then usually you don't want N to be more than about 5. If the target hosts you are retrieving data from are all different, you can experiment with values of N up to maybe 20.
// option 2 - run N at a time in parallel
function getMunicipalityGeoJsonData(req, res) {
//Get all the urls pointing to the JSON data for the province, Gelderland
getGelderlandJsonUrls((err, jsonUrls) => {
if (err) {
console.log(err);
res.sendStatus(500);
} else {
//for all those urls, write the data to files.
const numConcurrent = 5;
mapConcurrent(jsonUrls, numConcurrent, writeToFile).then(() => {
res.send("All done");
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
}
})
}
The mapConcurrent() function comes from this answer Promise.all consumes all my RAM and is as follows. It expects you to pass it an array of items to be iterated over, the max you want in flight at the same time and a function that will be passed an array item and will return a promise connected to when it's done or has an error:
function mapConcurrent(items, maxConcurrent, fn) {
let index = 0;
let inFlightCntr = 0;
let doneCntr = 0;
let results = new Array(items.length);
let stop = false;
return new Promise(function(resolve, reject) {
function runNext() {
let i = index;
++inFlightCntr;
fn(items[index], index++).then(function(val) {
++doneCntr;
--inFlightCntr;
results[i] = val;
run();
}, function(err) {
// set flag so we don't launch any more requests
stop = true;
reject(err);
});
}
function run() {
// launch as many as we're allowed to
while (!stop && inflightCntr < maxConcurrent && index < items.length) {
runNext();
}
// if all are done, then resolve parent promise with results
if (doneCntr === items.length) {
resolve(results);
}
}
run();
});
}
There are comparable functions in Bluebird's Promise.map() and in the Async library.
So, using this code you now have the ability to control how many of your requests/writeToFile() operations are in-process at the same time and you are capturing and logging all possible errors. Do, you can tune how many can be in flight at the same time for best performance and lowest resource use and, if there are any errors, you should be logging those errors so you can debug.
This code is currently set to stop processing any further URLs if it gets an error. You can change that if you want to continue on to the other URLs if you get an error by tweaking mapConcurrent(). But, I would still make sure you log any errors so you know when there are errors and can investigate why you are seeing errors.
One other note. If this was my code, I would convert everything to promises (no plain callbacks) and I'd use the got() library instead of the now deprecated request() library. I don't write any new code using the request() library.
The line res.send("Successfully saved the new address."); throws the
'ERR_HTTP_HEADERS_SENT' error.
I read through other posts concerning the same error and tried return res.send("Successfully saved the new address."), but that doesn't fix it. Any insights?
Note: I am new to this.
Please be kind.
Thanks.
My Code
app.post("/", function(req, res) {
const url = "http://geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer/findAddressCandidates?";
const street = req.body.street;
const city = req.body.city;
const state = req.body.state;
const zip = req.body.zip;
const yourAddress = "Address=" + street.replace(" ", "+") + "&City=" + city.replace(" ", "+") + "&Zip=" + zip;
const parameters = "&category=&outFields=*&forStorage=false&f=json";
request(url + yourAddress + parameters, function(error, response, body) {
const data = JSON.parse(body);
const newAddress = data.candidates[0].address;
const longitude = data.candidates[0].location.x;
const latitude = data.candidates[0].location.y;
const address = new Address({
address: newAddress,
latitude: latitude,
longitude: longitude
});
address.save(function(err) {
if (!err) {
res.send("Successfully saved the new address.");
} else {
res.send(err);
}
});
});
res.redirect("/");
});
You are doing both res.send() and res.redirect() in the same request handler. You can't send two responses to the same request. That's what generates the warning message you see.
Pick one of the other. You either want to send an appropriate status about the .save() or you want to redirect. One or the other, not both.
In this particular code, the res.redirect() happens first so it's what the client sees. Then, after that and when the request() and the address.save() have both completed, then you try to res.send(), but that is blocked because a response has already been sent and the http connection is already done.
If you just want to redirect upon successful save, you can remove the existing res.redirect() and change to this:
address.save(function(err) {
if (!err) {
res.redirect("/");
} else {
console.log(err);
res.sendStatus(500);
}
I would Agree With #jfriend00 answer , since there can be only one response for the request .
Here's what you can do:
address.save(function(err) {
if (!err) {
res.redirect('/success')
} else {
res.send(err);
}
});
//add a new route
app.get('/success',(req,res)=>{res.send('Successfully saved the new address.')})
And then you can redirect to the home page.
This is just a workaround in this case and can differ logically in each case depending on the requirement
The Other Way is to redirect on the client side itself
eg:If you're on jquery
$.ajax({
statusCode: {
200: function() {
$(location).attr('href', 'to the homepage')
}
}
});
So i have 2 Cloud Functions within the same file:
exports.Auth = functions.region('europe-west1').https.onRequest((req, res) =>
and
exports.IPN = functions.region('europe-west1').https.onRequest((req, res) =>
When adding the following code right at the start of my Auth function it adds a new document to the Firestore as expected, however, when i add the same code at the start of my IPN function, which is currently being called via Paypal's IPN Simulator, it does nothing, no errors.
let pin = RandomPIN(10, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
var userRef = db.collection('Users').doc(pin);
var setWithOptions = userRef.set({ Activated: false }, { merge: true });
console.log("PIN: "+pin);
What on earth is going on, i must be missing something?
Thanks in advance.
Update:
Here are the logs, first with the 2 middle lines commented and then uncommented It seems to be silently failing, i'm just not sure what is causing it.
Update with Complete function:
exports.IPN = functions.region('europe-west1').https.onRequest((req, res) =>
{
console.log("IPN Notification Event Received");
let pin = RandomPIN(10, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
var userRef = db.collection('Users').doc(pin);
var setWithOptions = userRef.set({ Activated: false }, { merge: true });
console.log("PIN: "+pin);
if (req.method !== "POST")
{
console.error("Request method not allowed.");
res.status(405).send("Method Not Allowed");
}
else
{
console.log("IPN Notification Event received successfully.");
res.status(200).end();
}
let ipnTransactionMessage = req.body;
// Convert JSON ipn data to a query string since Google Cloud Function does not expose raw request data.
let formUrlEncodedBody = querystring.stringify(ipnTransactionMessage);
// Build the body of the verification post message by prefixing 'cmd=_notify-validate'.
let verificationBody = `cmd=_notify-validate&${formUrlEncodedBody}`;
console.log(`Verifying IPN: ${verificationBody}`);
let options = {
method: "POST",
uri: getPaypalURI(),
body: verificationBody,
};
// POST verification IPN data to paypal to validate.
request(options, function callback(error, response, body)
{
if(!error && response.statusCode === 200)
{
if(body === "VERIFIED")
{
console.log(`Verified IPN: IPN message for Transaction ID: ${ipnTransactionMessage.txn_id} is verified.`);
SendPIN(ipnTransactionMessage.payer_email, pin);
}
else if(body === "INVALID")
console.error(`Invalid IPN: IPN message for Transaction ID: ${ipnTransactionMessage.txn_id} is invalid.`);
else
console.error("Unexpected reponse body.");
}
else
{
console.error(error);
console.log(body);
}
});
});
Indeed it is a problem of Promises chaining and also a problem due to the request library: request supports callback interfaces natively but does not return a promise, which is what you must do within a Cloud Function.
I would suggest that you watch these official Firebase videos from Doug : https://www.youtube.com/watch?v=7IkUgCLr5oA&t=28s and https://www.youtube.com/watch?v=652XeeKNHSk which explain this key concept.
You can use request-promise (https://github.com/request/request-promise) and the rp() method which "returns a regular Promises/A+ compliant promise".
It is not clear what SendPIN() is doing. Let's make the assumption it returns a Promise. If this is true, you could adapt your code along the following lines:
//....
const rp = require('request-promise');
//....
exports.IPN = functions.region('europe-west1').https.onRequest((req, res) => {
console.log('IPN Notification Event Received');
let pin = RandomPIN(
10,
'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
);
var userRef = db.collection('Users').doc(pin);
if (req.method !== 'POST') {
console.error('Request method not allowed.');
res.status(405).send('Method Not Allowed');
} else {
let ipnTransactionMessage;
userRef
.set({ Activated: false }, { merge: true })
.then(() => {
console.log('PIN: ' + pin);
ipnTransactionMessage = req.body;
// Convert JSON ipn data to a query string since Google Cloud Function does not expose raw request data.
let formUrlEncodedBody = querystring.stringify(ipnTransactionMessage);
// Build the body of the verification post message by prefixing 'cmd=_notify-validate'.
let verificationBody = `cmd=_notify-validate&${formUrlEncodedBody}`;
console.log(`Verifying IPN: ${verificationBody}`);
let options = {
method: 'POST',
uri: getPaypalURI(),
body: verificationBody
};
// POST verification IPN data to paypal to validate.
return rp(options);
})
.then(response => {
//Not sure what you will get within the response object...
console.log(
`Verified IPN: IPN message for Transaction ID: ${
ipnTransactionMessage.txn_id
} is verified.`
);
return SendPIN(ipnTransactionMessage.payer_email, pin); //It is not clear what SendPIN is doing, let's make the assumption it returns a Promise...
})
.then(() => {
res.send('Success');
return null;
})
.catch(err => {
console.error(
`Invalid IPN: IPN message for Transaction ID: ${
ipnTransactionMessage.txn_id
} is invalid.`
);
res
.status(500)
.send(
'Error: ' +
err +
` - Invalid IPN: IPN message for Transaction ID: ${
ipnTransactionMessage.txn_id
} is invalid.`
);
return null;
});
}
});
I updated the function to create the CSV file but now I'm getting an error:
In upload function
internal/streams/legacy.js:57
throw er; // Unhandled stream error in pipe.
^
Error: ENOENT: no such file or directory, open 'C:\Users\shiv\WebstormProjects\slackAPIProject\billingData\CSV\1548963844106output.csv'
var csvFilePath = '';
var JSONFilePath = '';
function sendBillingData(){
var message = '';
axios.get(url, {
params: {
token: myToken
}
}).then(function (response) {
message = response.data;
fields = billingDataFields;
// saveFiles(message, fields, 'billingData/');
saveFilesNew(message, fields, 'billingData/');
var file = fs.createReadStream(__dirname + '/' + csvFilePath); // <--make sure this path is correct
console.log(__dirname + '/' + csvFilePath);
uploadFile(file);
})
.catch(function (error) {
console.log(error);
});
}
The saveFilesNew function is:
function saveFilesNew(message, options, folder){
try {
const passedData = message;
var relevantData='';
if (folder == 'accessLogs/'){
const loginsJSON = message.logins;
relevantData = loginsJSON;
console.log(loginsJSON);
}
if(folder == 'billingData/'){
relevantData = passedData.members;
const profile = passedData.members[0].profile;
}
//Save JSON to the output folder
var date = Date.now();
var directoryPath = folder + 'JSON/' + date + "output";
JSONFilePath = directoryPath + '.json';
fs.writeFileSync(JSONFilePath, JSON.stringify(message, null, 4), function(err) {
if (err) {
console.log(err);
}
});
//parse JSON onto the CSV
const json2csvParser = new Json2csvParser({ fields });
const csv = json2csvParser.parse(relevantData);
// console.log(csv);
//function to process the CSV onto the file
var directoryPath = folder + 'CSV/' + date + "output";
csvFilePath = directoryPath + '.csv';
let data = [];
let columns = {
real_name: 'real_name',
display_name: 'display_name',
email: 'email',
account_type: 'account_type'
};
var id = passedData.members[0].real_name;
console.log(id);
console.log("messageLength is" +Object.keys(message.members).length);
for (var i = 0; i < Object.keys(message.members).length; i++) {
console.log("value of i is" + i);
var display_name = passedData.members[i].profile.display_name;
var real_name = passedData.members[i].profile.real_name_normalized;
var email = passedData.members[i].profile.email;
var account_type = 'undefined';
console.log("name: " + real_name);
if(passedData.members[i].is_owner){
account_type = 'Org Owner';
}
else if(passedData.members[i].is_admin){
account_type = 'Org Admin';
}
else if(passedData.members[i].is_bot){
account_type = 'Bot'
}
else account_type = 'User';
data.push([real_name, display_name, email, account_type]);
}
console.log(data);
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err;
fs.writeFileSync(csvFilePath, output, function(err) {
console.log(output);
if (err) {
console.log(err);
}
console.log('my.csv saved.');
});
});
} catch (err) {
console.error(err);
}
}
The upload file function is:
function uploadFile(file){
console.log("In upload function");
const form = new FormData();
form.append('token', botToken);
form.append('channels', 'testing');
form.append('file', file);
axios.post('https://slack.com/api/files.upload', form, {
headers: form.getHeaders()
}).then(function (response) {
var serverMessage = response.data;
console.log(serverMessage);
});
}
So I think the error is getting caused because node is trying to upload the file before its being created. I feel like this has something to do with the asynchronous nature of Node.js but I fail to comprehend how to rectify the code. Please let me know how to correct this and mention any improvements to the code structure/design too.
Thanks!
You don't wait for the callback provided to stringify to be executed, and it's where you create the file. (Assuming this stringify function really does acccept a callback.)
Using callbacks (you can make this cleaner with promises and these neat async/await controls, but let's just stick to callbacks here), it should be more like:
function sendBillingData() {
...
// this callback we'll use to know when the file writing is done, and to get the file path
saveFilesNew(message, fields, 'billingData/', function(err, csvFilePathArgument) {
// this we will execute when saveFilesNew calls it, not when saveFilesNew returns, see below
uploadFile(fs.createReadStream(__dirname + '/' + csvFilePathArgument))
});
}
// let's name this callback... "callback".
function saveFilesNew(message, options, folder, callback) {
...
var csvFilePath = ...; // local variable only instead of your global
...
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err; // or return callbcack(err);
fs.writeFile(csvFilePath , output, function(err) { // NOT writeFileSync, or no callback needed
console.log(output);
if (err) {
console.log(err);
// callback(err); may be a useful approach for error-handling at a higher level
}
console.log('my.csv saved.'); // yes, NOW the CSV is saved, not before this executes! Hence:
callback(null, csvFilePath); // no error, clean process, pass the file path
});
});
console.log("This line is executed before stringify's callback is called!");
return; // implicitly, yes, yet still synchronous and that's why your version crashes
}
Using callbacks that are called only when the expected events happen (a file is done writing, a buffer/string is done transforming...) allows JS to keep executing code in the meantime. And it does keep executing code, so when you need data from an async code, you need to tell JS you need it done before executing your piece.
Also, since you can pass data when calling back (it's just a function), here I could avoid relying on a global csvFilePath. Using higher level variables makes things monolithic, like you could not transfer saveFilesNew to a dedicated file where you keep your toolkit of file-related functions.
Finally, if your global process is like:
function aDayAtTheOffice() {
sendBillingData();
getCoffee();
}
then you don't need to wait for the billing data to be processed before starting making coffee. However, if your boss told you that you could NOT get a coffee until the billing data was settled, then your process would look like:
function aDayAtTheOffice() {
sendBillingData(function (err) {
// if (err) let's do nothing here: you wanted a coffee anyway, right?
getCoffee();
});
}
(Note that callbacks having potential error as first arg and data as second arg is a convention, nothing mandatory.)
IMHO you should read about scope (the argument callback could be accessed at a time where the call to saveFilesNew was already done and forgotten!), and about the asynchronous nature of No... JavaScript. ;) (Sorry, probably not the best links but they contain the meaningful keywords, and then Google is your buddy, your friend, your Big Brother.)