My aim is to do something like that :
function writer1(data,file){
const w = fs.createWriteStream(file,{flags:'w'})
for(let i = 0; i< data.length; i++){
w.write(data[i])
}
w.end()
}
function writer2(data,file, *some-stuff*){
const w = fs.createWriteStream(file,{flags:'w'})
for(let i = 0; i< data.length; i++){
if(data[i] !== *some-stuff*){
w.write(data[i])
}
}
w.end()
}
writer1(data,"file.txt")
writer2(data,"file.txt", "some string")
IMPORTANT TO NOTE : in the true piece of code I'm writing, writer1 has a condition to run; it runs only if the file it needs to write does not exists
But here is my problem; if the according files does not exists, i.e. if the 'STATE' of the project is init-state, then writer1 is launched but somehow shadows the execution of writer2.
The result is a txt file filled with the content DATA.
On the second pass, then writer1 is not launched, does not shadow the execution of writer2, and the result is a txt file filled with the content of DATA MINUS the variable some-stuff.
Essentially, my question is :
Why is the first stream shadowing the second and how to prevent that ?
I do understand that there's something asynchronous to be dealed with or a request to be made to the stream object in order to allow for other streams to access the same file. What is missing ?
Writing to a stream is an asynchronous process. If you open the file again in writer2 before writer1 has closed it, the writings of writer2 may be lost.
The following variant of writer1 is an asynchronous function that resolves only after closing the file. You can await this before calling writer2.
function writer1(data, file) {
return new Promise(function(resolve, reject) {
const w = fs.createWriteStream(file, {flags: 'w'})
.on("close", resolve)
.on("error", reject);
for (let i = 0; i < data.length; i++)
w.write(data[i]);
w.end();
});
}
function writer2(...) // similar
await writer1(data, "file.txt");
await writer2(data, "file.txt", "some string");
But I second jfriend00's question about what problem you are trying to solve.
Related
I'm writing React app. After clicking one button, I want the file to be downloaded. Before that, the array that I have has to be modified in order to have the downloaded report in proper format.
The problem I have is that I don't know how to force getReports() to wait for setInOrder() to process the data. Therefore code doesn't enter the loop.
export const setInOrder = async (objects) => {
var sortedObjectsAll = new Object();
for (let i = 0; i < objects.length; ++i) {
if (sortedObjectsAll.hasOwnProperty(objects[i].addedBy)) {
sortedObjectsAll[objects[i].addedBy].push(objects[i]);
} else {
sortedObjectsAll[objects[i].addedBy] = new Array();
}
}
return sortedObjectsAll
}
export const getReports = async (objects) => {
const sortedObjectsAll = await setInOrder(objects) // This is correct but not available instantly
console.log(sortedObjectsAll) // this is correctly printed
const reports = new Array();
for (let j = 0; j < sortedObjectsAll.length; ++j) {
console.log("Never enters here")
reports.push(createReport(sortedObjectsAll[j]))
}
return reports
}
I'm trying to use await or async somehow, but can't solve it. I see some Promises advised but I don't know how to really return the resulting variable to the code that actually downloads the report.
First you do not need to write an async-await something like that, because it is not an async operation (and if you write one and do not have any await in it, it will wait for nothing).
Second you want to iterate through an object, and not through an array, and that is the problem. Replace with the following (there are other solutions as well):
for (const key in sortedObjectsAll) {
...
}
still new to JSON and while ive searched the net and created a function to create a init file if none exists i'm coming up blank for search and retrive the data of the new existing file or how I add new entries or update new entries
so far i can do a read file and export the resits in a console log so i know the assignment work, its a global variable so the data should persist out of the read file loop but when i try and access it later to make the local array i'll pull data from and use for updating later it reads as undefined.
fs.readFile(path, 'utf8', (error, data) => {
if(error){
console.log(error);
return;
}
//console.log(JSON.parse(data));
JSONData = JSON.parse(data);
for (let i = 0; i < JSONData.length; i++) {
console.log(i+": ["+JSONData[i].unique+"] "+JSONData[i].name);
}
});//fs.readFile
var playerKey = "KuroTO";
playerKey = playerKey.toLowerCase();
for (let i = 0; i < JSONData.length; i++) {
if (JSONData[i].unique.toLowerCase() == playerKey){
console.log("["+i+"] "+JSONData[i].unique.toLowerCase()+": "+playerKey);
PlayerCard1.push(JSONData[i].userid);//0
PlayerCard1.push(JSONData[i].username);//1
PlayerCard1.push(JSONData[i].unique);//2
PlayerCard1.push(JSONData[i].name);//3
PlayerCard1.push(JSONData[i].avatarurl);//4
PlayerCard1.push(JSONData[i].level);//5
PlayerCard1.push(JSONData[i].Rank);//6
PlayerCard1.push(JSONData[i].henshined);//7
PlayerCard1.push(JSONData[i].Strength);//8
PlayerCard1.push(JSONData[i].Perception);//9
PlayerCard1.push(JSONData[i].Endurance);//10
PlayerCard1.push(JSONData[i].Wisdom);//11
PlayerCard1.push(JSONData[i].Intelligence)//12;
PlayerCard1.push(JSONData[i].Luck)//13;
PlayerCard1.push(JSONData[i].Agility)//14;
PlayerCard1.push(JSONData[i].Flexability)//15;
PlayerCard1.push(JSONData[i].RatedSpeed)//16;
};//if unique matches
};//for
this is ther psudo code concept im trying to do
if (JSONData.stringify.unique == {SearchUID}){toonname = JSONData.stringify.name;}
as i understand it you cant really apend just rewrite the file over again with new data and i think i can figure that out on my own once i cand figure out how to real the file into an array i can search like above
To read JSON, simply require the file.
JSON:
{
"key": "H"
}
JS:
let jsonFile = require("./path/to/json");
console.log(jsonFile.key); // H
Editing is just as simple.
let jsonFile = require("./path/to/json");
jsonFile.key = "A"
console.log(jsonFile.key) // A
Saving edits requires use of FileSystem:
const fs = require("fs")
let jsonFile = require("./path/to/json");
jsonFile.key = "A"
// first argument is the file path
// second argument is the JSON to write - the file is overwritten already
// due to above, so just JSON.stringify() the required file.
// third argument is an error callback
fs.writeFile("./path/to/jsonFile", JSON.stringify(jsonFile), (err) => {
if (err) throw new Error(err);
});
This can also be used to slightly clean up your current init function if you wanted, but that's up to you of course.
I cannot append data from multiple Read Streams using createWriteStreams() in Node.
It just creates a new file after each iteration.
chunkNo = 4;
for (i = 0; i < chunkNo; i++) {
const response = await this.downloadChunk(downloadUrl, i); // response.data is a ReadStream
await new Promise<void>((resolve, reject) => {
const stream = response.data.pipe(fs.createWriteStream('C:\\big_file.mkv'), { flags: 'a'});
stream.on('finish', () => {
resolve();
});
});
}
What am I missing? Even though the append flag is there.
Thank you in advance
To better consolidate your intuition about what the a (append) flag purpose is supposed to indicate, first have a look at this answer: Difference between modes a, a+, w, w+, and r+ in built-in open function?
It details the different modes in which a file can be open. As many other languages, JavaScript reuses the same principle that in C.
From this man page, the relevant section in our case is:
``a'' Open for writing. The file is created if it does not exist. The
stream is positioned at the end of the file. Subsequent writes
to the file will always end up at the then current end of file,
irrespective of any intervening fseek(3) or similar.
So in your code example, a new file would indeed be created at each iteration of the loop.
Therefore, what you can do is move the fs.createWriteStream('C:\\big_file.mkv', { flags: 'a'}) outside of the for loop, assigned a name to this readable stream and use it the pipe(), something along the way of :
chunkNo = 4;
const dest = fs.createWriteStream('C:\\big_file.mkv', { flags: 'a'});
for (i = 0; i < chunkNo; i++) {
const response = await this.downloadChunk(downloadUrl, i); // response.data is a ReadStream
await new Promise<void>((resolve, reject) => {
const stream = response.data.pipe(dest);
stream.on('finish', () => {
resolve();
});
});
}
I'm having a problem that I can't seem to get around.
Imaging the following
if (users) {
for (var i = 0; i < users.length; i++) {
apiData.users = getUserDetails(users[i]);
}
}
and
if(profiles) {
for (var i = 0; i < profiles.length; i++) {
apiData.profiles = getProfileDetails(users[i]);
}
}
now, only once both of those blocks of code have been completed (i.e. the API calls have returned the results that are required), would I like to do something like this
for (var i = 0; i < users.length; i++) {
saveUserProfile(apiData)
}
Normally I handle a situation like this by wrapping the first block of code in a function that returns a callback and then running the third function afterwards, but how can I do this when there are effectively 2 separate operations happening?
To be clear, I would like to avoid saving users and profiles separately, there are various reasons for this.
I would like to know if there is any standard way of handling a situation like this as there may or may not be users or profiles, so using callbacks seems like it will not work?
You can use async module to achieve this.
Use async.parallel() function since your first and second snippets do not depend on each other, and can run asynchronously, but you wish to make third snippet run when first and second are done executing.
Assuming users, profiles, and apiData are already declared, your code would look like
function func1(callback){
if (users) {
for (var i = 0; i < users.length; i++) {
apiData.users = getUserDetails(users[i]);
}
}
//invoke callback
}
function func2(callback){
if(profiles) {
for (var i = 0; i < profiles.length; i++) {
apiData.profiles = getProfileDetails(users[i]);
}
}
//invoke callback
}
var asyncfunc = [];
asyncfunc.push(func1);
asyncfunc.push(func2);
async.parallel(asyncfunc,function(err,result){
//result is an array which receives the second argument of each function's callback.
for (var i = 0; i < users.length; i++) {
saveUserProfile(apiData)
}
})
EDIT
PS: you can also use async.series() here. Both are equivalent in this case, since the functions handled by async.parallel() here do not use timers neither perform I/O tasks.
The callback in each function inside the array takes two arguments. First argument represents error, which is null if there are no errors, second represents the result of the function, which is passed to the second argument of the async.parallel() function's callback.
This question already has answers here:
JavaScript closure inside loops – simple practical example
(44 answers)
Closed 7 years ago.
I am a beginner to node.js. I was trying out the examples from the 'learnyounode' tutorial. I am trying to write a program that takes three url parameters and fetches some data from those urls and displays the returned data in the order in which the urls were provided.
var http = require('http');
var bl = require('bl');
var url = [];
url[0] = process.argv[2];
url[1] = process.argv[3];
url[2] = process.argv[4];
var data = [];
var remaining = url.length;
for(var i = 0; i < url.length; i++){
http.get(url[i], function (response){
response.setEncoding('utf8');
response.pipe(bl(function (err, chunk){
if(err){
console.log(err);
}
else{
data[i] = chunk.toString();
console.log(data[i]);
remaining -= 1;
if(remaining == 0) {
for(var j = 0; j < url.length; j++){
console.log(data[j]);
}
}
}
}));
});
}
I have two console.log statements in the program. The output i get is as follows:
It'll be chunder where lets throw a ford. We're going durry where mad as a cooee
.
Shazza got us some apples with come a strides. Mad as a swag when get a dog up y
a roo. It'll be rapt piece of piss as cunning as a trackie dacks.
As cross as a bogged with watch out for the boardies. As cunning as a digger fla
min lets get some roo bar. As dry as a piker piece of piss he hasn't got a joey.
Lets throw a strides mate we're going digger.
undefined
undefined
undefined
It seems like the data is correctly fetched and stored in the 'data' array but it still displays undefined.
Any idea why this is happening?
Thanks in advance!
This is a very common issue in async programming in node.js or even in the browser. A main issue you have is that the loop variable i will not be what you want it to be some time later when the async callback is called. By then, the for loop will have run to the end of its loop and i will be at the end value for all response callbacks.
There are numerous ways to solve this. You can use a closure to close over the i value and make it uniquely available to each callback.
var http = require('http');
var bl = require('bl');
var url = [];
url[0] = process.argv[2];
url[1] = process.argv[3];
url[2] = process.argv[4];
var data = [];
var remaining = url.length;
for(var i = 0; i < url.length; i++){
// create closure here to uniquely capture the loop index
// for each separate http request
(function(index) {
http.get(url[index], function (response){
response.setEncoding('utf8');
response.pipe(bl(function (err, chunk){
if(err){
console.log(err);
}
else{
data[index] = chunk.toString();
console.log(data[index]);
remaining -= 1;
if(remaining == 0) {
for(var j = 0; j < url.length; j++){
console.log(data[j]);
}
}
}
}));
});
})(i);
}
If you do much node.js programming, you will find that you probably want to learn how to use promises because they are very, very handy for controlling the flow and sequence of async operations.