I want to read a .csv. The consensus seems to be to do something like this:
const myCsv = "./my.csv";
// Throws exception if file doesn't exist
// File permissions are also 777 (just to figure this out)
fs.statSync(myCsv);
fs.createReadStream(myCsv)
.pipe(csv())
.on("data", () => console.log('got data')
.on("end", () => console.log('the end')
.on("error", () => console.log('error')
);
When I run this, none of the console.log() statements execute, and no errors/warnings.
Indeed, to try to understand what's going on, I run the following test case:
(() => {
const readStream = fs.createReadStream(myCsv);
readStream.on("open", () => {
console.log("open");
});
readStream.on("data", (chunk) => {
console.log("data");
});
readStream.on("error", (err) => {
console.log("error");
console.log(err);
});
readStream.on("end", () => {
console.log("end");
});
readStream.on("finish", () => {
console.log("finish");
});
})();
After fs.createReadStream(), readStream.fd = null (that can't be good), and none of the console statements executes.
NodeJs version = v16.19.0, and I'm running it on Ubuntu 22.04.
What am I missing? Thanks!
I know for sure that my pullData module is getting the data back from the file read but the function calling it, though it has an await, is not getting the data.
This is the module (./initialise.js) that reads the data:
const fs = require('fs');
const getData = () => {
return new Promise((resolve, reject) => {
fs.readFile('./Sybernika.txt',
{ encoding: 'utf8', flag: 'r' },
function (err, data) {
if (err)
reject(err);
else
resolve(data);
});
});
};
module.exports = {getData};
And this is where it gets called (app.js):
const init = require('./initialise');
const pullData = async () => {
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
const start = async() => {
let data = await pullData();
console.log(data);
}
start();
putting 'console.log(data)' just before return(data) in the resolve part of the call shows the data so I know it's being read OK. However, that final console.log shows my data variabkle as being undefined.
Any suggestions?
It's either
const pullData = async () => {
return init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
or
const pullData = async () =>
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
Both versions make sure a promise returned by then/catch is passed down to the caller.
I'm trying to read frpm a json file folder withing my program and i want to use a GET list endpoint to read through browser or postman, but i'm getting the above TypeError. Here is my code:
model.js:
const fs = require('fs');
function loadTeams() {
return new Promise((resolve, reject) => {
fs.readFile('./json/prov-nodes.json', (err, data) => {
if (err) reject(err);
const teams = JSON.parse(data);
console.log(teams);
resolve(teams);
});
});
}
app.use(bodyParser.json());
app.get('/list', (req, res) => {
let teams = [];
loadTeams()
.then(function(data){
teams = JSON.stringify(data);
console.log(teams);
**res.send(teams);** //intended to send to browser/postman response
console.log('try...part ..read call');
})
.catch(error => console.log(error))
res.send("My root page");
console.log(teams);
});
The loadTeams function does not return a promise, and therefore you cannot call .then().
You can wrap the function in a promise like this:
function loadTeams() {
return new Promise(function(resolve, reject) {
fs.readFile('./json/prov-nodes.json', (err, data) => {
if (err) reject(err);
try {
const teams = JSON.parse(data);
return resolve(teams);
} catch(e) {
reject(e);
}
});
});
}
In order to use loadTeams as an async function you should turn it into a function that returns Promise with a callback results:
function loadTeams() {
return new Promise((resolve, reject) => {
fs.readFile('./json/prov-nodes.json', (err, data) => {
if (err) reject(err);
const teams = JSON.parse(data);
console.log(teams);
resolve(teams);
});
});
}
I'm trying to develop a relative simple test on NodeJS 11.6.x. I'm not really a developer, but sometimes try to do some coding.
My objective is to create a SQLite database and repeat some steps every time a run the code:
1. Drop a table if it exists
2. Create a table
3. Insert N lines
4. Count how many lines is in the database
5. Close the database
I tried first with a basic approach using callback, but couldn't figure out a way to make the step 3 (insert N lines) and looking for a solution, the promise async/await 'pattern' sounded the way to accomplish everything.
But, after refactoring the code, the step 1 (drop table) isn't running and I still not being able to execute step 3 (insert N lines) and have no idea what is happening. I also tried to use a promise package with no luck.
Could someone please have a look and help on this and if possible, explain and or give some advice?
Thanks in advance
Edited: Well, I'not used to post here at SO and don't know the 'right' way to update things here. I beleave I should had left the first code as reference, buy I don't have anymore.
Now I think I'm almoust there. All steps are executing in order. Just the step 3 (insert N lines) that I'm not able to make it work. Or it inserts and stops not going to the next '.then' or it just insert 1 line and I'm cannot visualize what is happening.
In the code, I commented in two lines with 'BUG 1:' and 'BUG 2:'.
If I both are commented, I get what is happening, it inserts only 1 line and don't continue the promise chain
If I comment BUG 1 and let BUG 2 active, it inserts just one line and continues. I think I understand why
If I comment BUG 2 and let BUG 1 active, it inserts all lines but don't continue and again, I think I understand why
If I uncomment both (the way I think should work. Don't work, and return an aditional error "Segmentation fault"
Bellow the code:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
waitTime = 1
process.stdout.write('Starting...\n')
var test = new Promise((resolve, reject) => {
process.stdout.write('Drop Table... ');
db.run(`DROP TABLE IF EXISTS test`, (err) => {
if (err) {
process.stdout.write(`Dropping Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Dropped!\n`)
resolve()
}, waitTime)
}
})
})
test.then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Create Table... ')
db.run(`CREATE TABLE IF NOT EXISTS test (data TEXT)`, (err) => {
if (err) {
process.stdout.write(`Creating Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Created!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Insert Line... ')
lines = 10
let loop = (async () => {
for (let i = 0; i < lines; i++) {
await new Promise(resolve =>
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
throw (err)
} else {
setTimeout(() => {
// process.stdout.write(`Line ${i} Inserted!\n`)
process.stdout.write(`, ${i+1}`)
resolve() // BUG 1: if this line is commented, comment it, it will insert only 1 line
}, waitTime)
}
})
)
}
})()
process.stdout.write(`, IDone\n`)
resolve() // BUG 2: If this line is commented, the promise chain stops here
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Count Line(s)... ')
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) {
process.stdout.write(`Count Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(` ${rows[0].totalLines} Count!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Select Line(s)... ')
db.all('SELECT data FROM test', [], (err, rows) => {
if (err) {
process.stdout.write(`Select Error ${err.message}\n`)
reject()
} else {
rows.forEach((row) => {
console.log(row.data);
})
setTimeout(() => {
process.stdout.write(`${rows[0].totalLines} Select!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Close DB... ')
db.close((err) => {
if (err) {
process.stdout.write(`Closing Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Closed!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
console.log('Finished')
})
After the great explanation from #CertainPerformance (Thanks a lot), I was able to get it running. I believe it is now "the right" way to do it. May be there are some better ways, but for now, it is ok for me, bellow the final code:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
lines = 10
process.stdout.write('Starting... ')
var test = new Promise((resolve, reject) => { process.stdout.write(`Promise Created...!\n`)
resolve()
})
test.then(() => { process.stdout.write('Drop Table... ')
return new Promise((resolve, reject) => {
db.run(`DROP TABLE IF EXISTS test`, (err) => {
if (err) {
reject(err)
} else { process.stdout.write(`Dropped!\n`)
resolve() }
})
})
}).then(() => { process.stdout.write('Create Table... ')
return new Promise((resolve, reject) => {
db.run(`CREATE TABLE IF NOT EXISTS test (data TEXT)`, (err) => {
if (err) {
reject(err)
} else {
process.stdout.write(`Created!\n`)
resolve() }
})
})
}).then(() => { process.stdout.write('Insert Line... ')
let insertLoop = (async () => {
for (let i = 0; i < lines; i++) {
await new Promise(resolve =>
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
reject(err)
} else { ( i == 0 ) ? process.stdout.write(`${i + 1}`) : process.stdout.write(`, ${i + 1}`)
resolve() }
})
)
}
process.stdout.write(`, Inserted!\n`)
})()
return insertLoop
}).then(() => { process.stdout.write('Count Line(s)... ')
return new Promise((resolve, reject) => {
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) {
reject(err)
} else { process.stdout.write(` ${rows[0].totalLines} Counted!\n`)
resolve()
}
})
})
}).then(() => { process.stdout.write('Close DB... ')
return new Promise((resolve, reject) => {
db.close((err) => {
if (err) {
reject(err)
} else { process.stdout.write(`Closed!\n`)
resolve()
}
})
})
}).then(() => {
console.log('Finished')
}).catch((err) => {
process.stdout.write(`The process did not finish successfully: ${err}`)
})
There are two main issues. First, in the second .then, you declare loop as an async function that is immediately invoked: this means that loop will resolve to a Promise. The trimmed code looks like:
}).then(() => {
return new Promise((resolve, reject) => {
let loop = (async () => {
// do some asynchronus stuff
})()
resolve() // BUG 2
})
}).then(() => {
Declaring a Promise alone will not cause the current thread to wait for it. The above code doesn't work as expected for the same reason that this code prints after immediately:
console.log('start');
const prom = new Promise((resolve) => {
setTimeout(resolve, 500);
});
console.log('after');
You must call .then on a Promise (or await the Promise) in order to schedule additional operations after the Promise completes. Or, if you're currently inside a .then, you can return the Promise, which will mean that the next .then will run as soon as the returned Promise resolves:
}).then(() => {
let loop = (async () => {
// do some asynchronus stuff
})();
return loop;
}).then(() => {
// this block will run once `loop` resolves
Note the lack of a new Promise((resolve... constructor above - inside a .then, just returning the next Promise is often the preferred way to go, since it means a lot less code and avoids an antipattern.
The other issue with the current code is that errors will not be caught. For example, if your
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
throw (err)
// else call resolve()
throws an error, the Promise that's currently being awaited at that point will never resolve, nor will it reject - it will remain pending and unfulfilled forever. You should pass reject as the second argument to the Promise constructor, and call it when there's an error (instead of throw), for example:
await new Promise((resolve, reject) => {
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
reject(err)
} else {
// ...
This way, the awaited Promise will get rejected, which means that the whole loop will reject, and if the loop is returned, it'll allow a .catch to catch the error, for example:
var test = new Promise((resolve, reject) => {
// ...
});
test.then(() => {
return new Promise(...
// ...
})
.then(() => {
return new Promise(...
// ..
})
.then(() => {
return new Promise(...
// ..
})
.catch((err) => {
process.stdout.write(`The process did not finish successfully:`, err)
// handle errors
});
Note that, unless each db. function call needs to execute serially, it would be better to make all the requests at once, and resolve once each request has finished - this can significantly reduce the time required for the script to run. Create an array of Promises for each asynchronous call, then call Promise.all on that array to get a Promise that resolves when all of those Promises are done (or, rejects as soon as one of those Promises rejects). For example, for the second .then:
}).then(() => {
process.stdout.write('Insert Line... ')
const proms = Array.from(
{ length: lines },
(_, i) => new Promise((resolve, reject) => {
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
reject(err)
} else {
setTimeout(() => {
// process.stdout.write(`Line ${i} Inserted!\n`)
process.stdout.write(`, ${i+1}`)
resolve()
}, waitTime);
}
});
})
);
return Promise.all(proms);
}).then(() => {
Nothing else in your code looks to deal with asynchronous loops, luckily.
You may also consider a utility function like Promisify which will turn callback-based functions to Promises without all the extra new Promise(... boilerplate every time there's an asynchronous call.
Further improvements can be made by promisifying db into reusable functions, and leveraging the full power of async/await instead of mixing it with then:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
function runDbAsync(sql) {
return new Promise((resolve, reject) => {
db.run(sql, (err) => {
if (err) reject(err);
else resolve();
});
});
}
function getDbAsync(sql, val) {
return new Promise((resolve, reject) => {
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) reject(err);
else resolve(rows);
});
});
}
function closeDbAsync() {
return new Promise((resolve, reject) => {
db.close((err) => {
if (err) reject(err);
else resolve();
});
});
}
function write(text) {
process.stdout.write(text);
}
function writeLn(text) {
write(text + "\n");
}
async function main() {
const lines = 10
writeLn('Starting... ')
write('Drop Table... ');
await runDbAsync(`DROP TABLE IF EXISTS test`);
writeLn(`Dropped!`);
write('Create Table... ');
await runDbAsync(`CREATE TABLE IF NOT EXISTS test (data TEXT)`);
writeLn(`Created!`);
write('Insert Line... ');
for (let i = 0; i < lines; i++) {
await runDbAsync(`INSERT INTO test (data) VALUES ('a')`);
write( i == 0 `${i + 1}` : `, ${i + 1}`);
}
writeLn(`, Inserted!`);
write('Count Line(s)... ')
const rows = getDbAsync(`SELECT COUNT(*) AS totalLines FROM test`, []);
writeLn(` ${rows[0].totalLines} Counted!`)
write('Close DB... ');
await closeDbAsync();
writeLn(`Closed!`);
}
main().then(() => {
console.log('Finished')
}, err => {
writeLn(`The process did not finish successfully: ${err}`)
});
I am trying to download tracks via the soundcloud API, and then launch a callback once an indeterminant amount of tracks is downloaded. When I run the below code, I see "All done" being console logged before anything else, even though I intend for it to be the last thing... What am I doing wrong?
// Deps
import fs from 'fs'
import SC from 'node-soundcloud'
import request from 'request'
// Write mp3 function
function writeMP3(track) {
return new Promise((resolve, reject) => {
console.log('Starting download: ', track.title)
request.get(track.download_url)
.on('error', err => {
// reject('Download error: ', err)
})
.on('finish', () => {
() => resolve('Download complete')
})
.pipe(fs.createWriteStream(`./data/temp/${track.title}_${track.user.username}.mp3`))
})
}
async function asyncTrackFetch(track) {
return await writeMP3(track)
}
// Array of promises to callback upon
const trackActions = []
SC.init({
id: 'MY_ID',
secret: 'MY_SECRET'
})
SC.get('/tracks', (err, tracks) => {
if (err) {
throw new Error(err)
} else {
console.log('Tracks fetched: ', tracks.length)
tracks.map(track => {
if (track.downloadable) {
console.log('downloadable')
trackActions.push(asyncTrackFetch(track))
}
})
}
})
// Perform requests async
Promise.all(trackActions).then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
Promise.all(trackActions) waits on whatever promises are in trackActions, but trackActions is empty at the time you make the call. You're only adding promises to the array after your SC.get callback gets called.
Try putting your Promise.all... block inside the SC.get callback like this:
SC.get('/tracks', (err, tracks) => {
if (err) {
throw new Error(err)
} else {
console.log('Tracks fetched: ', tracks.length)
tracks.map(track => {
if (track.downloadable) {
console.log('downloadable')
trackActions.push(asyncTrackFetch(track))
}
})
Promise.all(trackActions).then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
}
})
It's worth mentioning as well that your line throw new Error(err) will crash the program since there's nowhere for that error to be caught.
As Antonio Val mentioned, there are better ways to do this. If you promisify the node-soundcloud library then the last part of your code could look like this:
SC.get('/tracks').then(tracks => {
// No need for trackedActions array.
return Promise.all(tracks.filter(track => track.downloadable)
.map(track => asyncTrackFetch(track)))
}).then(fetchedTracks => {
console.log('All done fetching tracks', fetchedTracks)
}).catch(err => {
// Handle error.
})
Or inside an async function,
try {
const tracks = await SC.get('/tracks')
const fetchPromises = tracks
.filter(track => track.downloadable)
.map(track => asyncTrackFetch(track))
const fetchedTracks = await Promise.all(fetchPromises)
console('All done fetching tracks.', fetchedTracks)
} catch (err) {
// Handle error
}
I think the easiest way would be to move Promise.all after tracks.map loop finished.
A more elegant solution would be to promisify SC.get as well and use async await along all your code.
UPDATE:
Couldn't test it so not sure if it works, but it would be something like this:
import fs from 'fs'
import SC from 'node-soundcloud'
import request from 'request'
function writeMP3(track) {
return new Promise((resolve, reject) => {
console.log('Starting download: ', track.title)
request.get(track.download_url)
.on('error', err => {
// reject('Download error: ', err)
})
.on('finish', () => {
() => resolve('Download complete')
})
.pipe(fs.createWriteStream(`./data/temp/${track.title}_${track.user.username}.mp3`))
})
}
function getTracks() {
return new Promise((resolve, reject) => {
SC.get('/tracks', (err, tracks) => {
if (err) {
return reject(err)
}
console.log('Tracks fetched: ', tracks.length)
resolve(tracks)
})
})
}
SC.init({
id: 'MY_ID',
secret: 'MY_SECRET'
})
With async await:
async function start() {
const tracks = await getTracks();
for (let track of tracks) {
await writeMP3(track)
}
}
start()
.then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
.catch((err) => {
// insert error handler here
})
If you just want to use Promises:
getTracks
.then((tracks) => {
const promiseArray = tracks.map((track) => {
return writeMP3(track)
})
return Promise.all(promiseArray)
})
.then(() => {
console.log('All done')
console.log(fs.readdirSync('./data/temp'))
})
.catch((err) => {
// insert error handler here
})