Nodejs check file exists, if not, wait till it exist - node.js

I'm generating files automatically, and I have another script which will check if a given file is already generated, so how could I implement such a function:
function checkExistsWithTimeout(path, timeout)
which will check if a path exists, if not, wait for it, util timeout.

Assuming you're planning on using Promises since you did not supply a callback in your method signature, you could check if the file exists and watch the directory at the same time, then resolve if the file exists, or the file is created before the timeout occurs.
function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
watcher.close();
resolve();
}
});
});
}

fs.watch() API is what you need.
Be sure to read all the caveats mentioned there before you use it.

import fs from 'node:fs'; //es6
//or
const fs = require('fs'); //commonjs
/**
*
* #param {String} filePath
* #param {Number} timeout
* #returns {Promise<Boolean>}
*/
const holdBeforeFileExists = async (filePath, timeout) => {
timeout = timeout < 1000 ? 1000 : timeout
try {
var nom = 0
return new Promise(resolve => {
var inter = setInterval(() => {
nom = nom + 100
if (nom >= timeout) {
clearInterval(inter)
//maybe exists, but my time is up!
resolve(false)
}
if (fs.existsSync(filePath) && fs.lstatSync(filePath).isFile()) {
clearInterval(inter)
//clear timer, even though there's still plenty of time left
resolve(true)
}
}, 100)
})
} catch (error) {
return false
}
}
(async()=>{
const maxTimeToCheck = 3000; //3 second
const fileCreated = '/path/filename.ext';
const isFile = await holdBeforeFileExists(fileCreated, maxTimeToCheck);
//Result boolean true | false
})();
It's work goodssssssssssss................!!!
Try it before giving bad comments.
Enjoy your Kopi mana kopi obat kagak ngantuk???
express js:
router.get('some_url', async(req, res)=>{
const fileCreated = someFunctionCreateFileWithResultStringPathName();
const maxTimeToCheck = 3000; //3 second
const isFile = await holdBeforeFileExists(fileCreated, maxTimeToCheck);
if(isFile){
res.sendFile(fileCreated)
} else {
res.send('Failed to generate file, because use a bad function to generate file. or too long to create a file');
}
})

Here is the solution:
// Wait for file to exist, checks every 2 seconds by default
function getFile(path, timeout=2000) {
const intervalObj = setInterval(function() {
const file = path;
const fileExists = fs.existsSync(file);
console.log('Checking for: ', file);
console.log('Exists: ', fileExists);
if (fileExists) {
clearInterval(intervalObj);
}
}, timeout);
};

You could implement it like this if you have node 6 or higher.
const fs = require('fs')
function checkExistsWithTimeout(path, timeout) {
return new Promise((resolve, reject) => {
const timeoutTimerId = setTimeout(handleTimeout, timeout)
const interval = timeout / 6
let intervalTimerId
function handleTimeout() {
clearTimeout(timerId)
const error = new Error('path check timed out')
error.name = 'PATH_CHECK_TIMED_OUT'
reject(error)
}
function handleInterval() {
fs.access(path, (err) => {
if(err) {
intervalTimerId = setTimeout(handleInterval, interval)
} else {
clearTimeout(timeoutTimerId)
resolve(path)
}
})
}
intervalTimerId = setTimeout(handleInterval, interval)
})
}

Here another version that works for me :
async function checkFileExist(path, timeout = 2000)
{
let totalTime = 0;
let checkTime = timeout / 10;
return await new Promise((resolve, reject) => {
const timer = setInterval(function() {
totalTime += checkTime;
let fileExists = fs.existsSync(path);
if (fileExists || totalTime >= timeout) {
clearInterval(timer);
resolve(fileExists);
}
}, checkTime);
});
}
You can simply use it :
await checkFileExist("c:/tmp/myfile.png");

function verifyFileDownload(extension) {
browser.sleep(150000); //waiting for file to download
const fs = require('fs');
let os = require('os');
var flag = true;
console.log(os.userInfo());
fs.readdir('/Users/' + require("os").userInfo().username + '/Downloads/', (error, file) => {
if (error) {
throw error;
}
console.log('File name' + file);
for (var i = 0; i < file.length; i++) {
const fileParts = file[i].split('.');
const ext = fileParts[fileParts.length - 1];
if (ext === extension) {
flag = false;
}
}
if (!flag) {
return;
}
throw error;
});
};

This is very much a hack, but works for quick stuff.
function wait (ms) {
var now = Date.now();
var later = now + ms;
while (Date.now() < later) {
// wait
}
}

Related

clearInterval of a external function not working - Node.JS

I have a setInterval function that's been called in another function, and I need to stop it when the proccess is done. I tried to set this setInterval function as a variable and call clearInterval, but the interval keeps running
const createInterval = (visibilityTimeout, startDateTime, message) => {
setInterval(() => {
const currentDateTime = moment().valueOf();
const timeDifference = (visibilityTimeout * 1000) - (currentDateTime - startDateTime);
if (timeDifference >= 600000) {
return;
}
if (timeDifference < 494983) {
const params = {
QueueUrl: 'http://localhost:4566/000000000000/test-queue2',
ReceiptHandle: message.ReceiptHandle,
VisibilityTimeout: visibilityTimeout,
};
sqs.changeMessageVisibility(params, (err, data) => {
if (err) logger.error(err, err.stack);
else logger.info(data);
});
// eslint-disable-next-line no-param-reassign
visibilityTimeout += 300;
}
}, 5000);
};
module.exports = async (message) => {
const startDateTime = moment().valueOf();
const {
noteId,
} = JSON.parse(message.Body);
logger.info(`Processing message [noteId=${noteId}]`);
try {
const note = await TestSessionNote.findById(noteId);
const testSession = await TestSession.findById(note.test_session_id);
logger.info(`Downloading video [key=${testSession.video_key}]`);
const isProcessing = true;
const interval = createInterval(500, startDateTime, message, isProcessing);
await sleep(20000);
clearInterval(interval);
logger.info(`Finished processing message [noteId=${noteId}]`);
} catch (ex) {
await TestSessionNote.update(noteId, { status: 'transcribe_error' });
logger.error(`Error processing message [noteId=${noteId}]`, ex);
}
};
I know that if i create a var test = setInterval(() => {console.log('blabla')}, 500) and call clearInterval(test) it works, but i don't know how can i do this calling a function
I think that you have to return from createInterval function the intervalId and after that it should work.
Can you check what value has your intervalId right now, with your current implementation?
https://developer.mozilla.org/en-US/docs/Web/API/setInterval
"The returned intervalID is a numeric, non-zero value which identifies the timer created by the call to setInterval(); this value can be passed to clearInterval() to cancel the interval."

Node.js split file lines

I want to write a script that divides the lines read from the file into packages of 25, unfortunately the sample package returns 40 codes. I would like to do so that, for example, he divided me into packages of 25 items. I mean, I have, for example, 60 codes, this creates 2 packages of 25, and one with 10 codes. Unfortunately, I can't handle it.
const fs = require('fs');
fs.readFile('code.txt', function (err, data) {
if (err) throw err;
const array = data.toString().split("\n");
let count = 0;
let items = [];
for (let i in array) {
items.push({
PutRequest: {
Item: {
code: array[i]
}
}
});
let params = {
RequestItems: {
'TABLE_NAME': items
}
};
if (count === 25) {
dynamoDB.batchWrite(params, function (err, data) {
if (err) {
console.log(err);
} else {
count = 0;
items = [];
}
});
}else{
count++;
}
}
});
code.txt content
https://0bin.net/paste/NA8-4hkq#1Ohwt5uUkQqE0YscwnxTX2gxEqlvAUVKp1JRipBCsZg
Any idea what I do wrong?
Your dynamoDB.batchWrite() is asynchronous. Thus its callback is executed only after the loop has completed. So items and count are never reset ...
The easiest would be, if you could switch to an promise based approach like the following
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
while (lines.length > 0) {
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
}
}
A callback based approach could look like this
const BATCHSIZE = 25;
fs.readFile("code.txt", "utf-8", (err, data) => {
const lines = data.split("\n");
function writeBatch() {
if (!lines.length) return;
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
dynamoDb.batchWrite(params, err => {
if (err) ...
else writeBatch();
});
}
writeBatch();
}
The function writeBatch takes a certain number of lines from your original array and writes them into the database. Only afer the write into the DB was successful, it recursively calls itself and handles the next batch. But be aware, that this approach may exceed the maximum call stack size and throw an error.
You can also make either of this approaches not manipulate the lines array (which may be quite expensive), but just get out the current slice
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
let currentIndex = 0;
while (currentIndex < lines.length) {
const items = lines.slice(currentIndex, currentIndex + BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
currentIndex += BATCHSIZE;
}
}
and
const BATCHSIZE = 25;
fs.readFile("code.txt", "utf-8", (err, data) => {
const lines = data.split("\n");
function writeBatch(currentIndex) {
if (currentIndex >= lines.length) return;
const items = lines.slice(currentIndex, currentIndex + BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
dynamoDb.batchWrite(params, err => {
if (err) ...
else writeBatch(currentIndex + BATCHSIZE);
});
}
writeBatch(0);
}
To prevent stumbling into a maximum callstack exception you may also add the next batch to the eventloop and not call it recursively. Ie
dynamoDb.batchWrite(params, err => {
if (err) ...
else setTimeout(()=> { writeBatch(currentIndex + BATCHSIZE);}, 0);
});
This way you won't build up a massive callstack from recursive calls.
To keep track of how many records are already saved to the db you could simply store the current counter in a file. When you restart the process, load that file and check how many lines to skip. Don't forget to remove the file, once all records have been saved ... For example with the first approach:
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
const skipLines = 0;
try {
skipLines = +(await fs.readFile("skip.txt", "utf-8"));
if (isNaN(skipLines)) skipLines = 0;
lines.splice(0, skipLines);
} catch (e) {
skipLines = 0;
}
while (lines.length > 0) {
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
skipLines += BATCHSIZE;
await fs.writeFile("skip.txt", `${skipLines}`);
}
try {
await fs.unlink("skip.txt");
} catch (e) {
}
}

Why doesn't my async function return any result?

I wrote this small program to fetch data. This however is done async. Since I nonetheless need to use the function holeVertreter(kzl) as a function in another module, I'd like to get a return value which I can eventually pass on.
Excuse my spaghetti code (I usually prettify the code when I am done with my task ...).
Credentials are stored in a file and are therefore not found in this file.
I'd like to end up with "vertreter" as a return value.
Thank you in advance.
const node = require("deasync");
const DSB = require('dsbapi');
const tabletojson = require('tabletojson');
const https = require('https');
const cred = require("./vertrCred");
const dsb = new DSB(cred["dsb"]["user"], cred["dsb"]["passw"]); //Sanitized - no Credentials here
//Stackoverflow 2332811
String.prototype.capitalize = function(lower) {
return (lower ? this.toLowerCase() : this).replace(/(?:^|\s)\S/g, function(a) { return a.toUpperCase(); });
};
function holePlan(kuerzel) {
dsb.fetch()
.then(data => {
const timetables = DSB.findMethodInData('timetable', data);
const tiles = DSB.findMethodInData('tiles', data);
var tilesStr = JSON.stringify(tiles["data"][0]["url"]);
var url = JSON.parse(tilesStr);
https.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end',() => {
var tableasjson = tabletojson.convert(data);
var erstetab = tableasjson[0];
var zweitetab = tableasjson[1];
var drittetab = tableasjson[2];
var viertetab = tableasjson[3];
var fuenftetab = tableasjson[4];
var sechstetab = tableasjson[5];
var siebtetab = tableasjson[6];
var achtetab = tableasjson[7];
if (typeof kuerzel === "undefined")
{
var regenechse = '(Aaa|Aaa[A-Za-z?]|[A-Za-z?]Aaa)';
}
else {
var name = kuerzel.capitalize(true);
var regenechse = '('+name+'|'+name+'[A-Za-z?]|[A-Za-z?]'+name+')';
}
const regex = new RegExp(regenechse,'g');
var sammel = Object.assign(drittetab,fuenftetab);
var z= 0;
var vertreter = {}
var y = JSON.parse(JSON.stringify(sammel));
for (i=0;i<y.length;i++) {
if (typeof y[i].Vertreter =='undefined') {
}
else {
if(y[i].Vertreter.match(regex))
{
z += 1;
vertreter[z] = y[i];
}
}
}
if (z == 0) {
// console.log("Es gibt nichts zu vertreten");
}
else {
//console.log("Es werden "+z+" Stunden vertreten");
return (vertreter);
} ;
});
})
})
.catch(e => {
// An error occurred :(
console.log(e);
});
}
//Stackoverflow
function warte(promise) {
var done = 0;
var result = null;
promise.then(
function (value) {
done = 1;
result = value;
return (value);
},
function (reason) {
done = 1;
throw reason;
}
);
while (!done)
node.runLoopOnce();
return (result);
}
function holeVertretung(kzl) {
var aufgabe = new Promise((resolve,reject) => {
setTimeout(resolve,1000,holePlan(kzl));
});
var ergebnis = warte(aufgabe);
if (typeof ergebnis === "undefined") {
console.log("Mist");
}
else {
console.log(ergebnis);
}
return ergebnis;
}
holeVertretung("Aaa");
That's not the right way to work with promises. If you do such infinite loop, it beats the whole purpose of using promises. Instead, return value from the promise, and use async-await like this:
function warte(promise) {
var done = 0;
var result = null;
return promise.then(
...
}
async function holeVertretung(kzl) {
var aufgabe = new Promise((resolve, reject) => {
setTimeout(resolve, 1000, holePlan(kzl));
});
var ergebnis = await warte(aufgabe);
...
If async-await does not work for some reason, use then clause:
warte(aufgabe).then(value => {
var ergebnis = value;
});

Polling a URL until certain value is set in JSON response : Mocha, Integration testing

I am working on automating an End to end scenario using Mocha.
I have a url endpoint which is to be polled until a certain value is obtained in the resulting response. Is there any way to do it ?
Example with request and callback approach:
const request = require('request');
describe('example', () => {
it('polling', function (done) {
this.timeout(5000);
let attemptsLeft = 10;
const expectedValue = '42';
const delayBetweenRequest = 100;
function check() {
request('http://www.google.com', (error, response, body) => {
if (body === expectedValue) return done();
attemptsLeft -= 1;
if (!attemptsLeft) return done(new Error('All attempts used'));
setTimeout(check, delayBetweenRequest);
});
}
check();
});
});
Example with got and async/await approach:
const utils = require('util');
const got = require('got');
const wait = utils.promisify(setTimeout);
describe('example', () => {
it('polling', async function (done) {
this.timeout(5000);
const expectedValue = '42';
const delayBetweenRequest = 100;
for (let attemptsLeft = 10; attemptsLeft; attemptsLeft -= 1) {
const resp = await got.get('http://www.google.com');
if (resp.body === expectedValue) return done();
await wait(delayBetweenRequest);
}
done(new Error('All attempts used'));
});
});
This is how I was able to do it with WebdriverIO and Mocha
describe("wait for value in content of page", () => {
it("should be able to wait to value in url", () => {
var max_seconds_to_wait = 10;
var seconds_counter = 0;
var should_continue = true;
while (should_continue) {
browser.url('http://your.url.com');
var response = JSON.parse(browser.getText("body"));
console.log(response)
if (response == 'something') {
should_continue = false;
}
browser.pause(1000);
seconds_counter++;
if (seconds_counter > max_seconds_to_wait) {
throw 'Waiting for json from url timeout error';
}
}
});
});

How do I run multiple node streams in sequence?

Given two streams, stream1, stream2, how can I run them in sequence, throwing if any fails?
I'm looking for something simpler than this:
stream1.on('end',function(){
stream2.on('end',done);
});
stream1.on('error',function(error){done(error);});
stream2.on('error',function(error){done(error);});
Thanks.
there are some ways to do that, check next link, it gonna help to how to write some code in node in a elegant way:
Node.js FLOW
Hope it what you need.
Depending on your use case you could combine the two streams into one by using the multistream module.
Multistreams are constructed from an array of streams
var MultiStream = require('multistream')
var fs = require('fs')
var streams = [
fs.createReadStream(__dirname + '/numbers/1.txt'), // contains a single char '1'
fs.createReadStream(__dirname + '/numbers/2.txt'), // contains a single char '2'
fs.createReadStream(__dirname + '/numbers/3.txt') // contains a single char '3'
]
MultiStream(streams).pipe(process.stdout) // => 123
In case combining streams does not fit the use case you could build your stream on end event sending functionality on your own
const fs = require('fs');
var number1 = fs.createReadStream('./numbers1.txt')
.on('data', d => console.log(d.toString()));
var number2 = fs.createReadStream('./numbers2.txt')
.on('data', d => console.log(d.toString()));
onEnd([number1, number2], function(err) {
console.log('Ended with', err);
});
function onEnd(streams, cb) {
var count = streams.length;
var ended = 0;
var errored = null;
function shouldEnd() {
ended++;
if (errored) { return; }
if (count == ended) {
cb();
}
}
function endWithError(err) {
if (errored) { return; }
errored = true;
cb(err);
}
streams.forEach(s => s
.on('end', shouldEnd)
.on('error', endWithError)
);
}
The onEnd function can be used to wait for a stream array to end or in case an error event is emitted for the first emitted error event.
Try do it with async functions:
const { createReadStream } = require("fs")
async function main() {
const stream1 = createReadStream(__dirname + "/1.txt")
await pipe(stream1, process.stdout)
const stream2 = createReadStream(__dirname + "/2.txt")
await pipe(stream2, process.stdout)
const stream3 = createReadStream(__dirname + "/3.txt")
await pipe(stream3, process.stdout)
}
async function pipe(tap, sink) {
return new Promise((resolve, reject) => {
tap.pipe(sink, { end: false })
tap.on("end", resolve)
tap.on("error", reject)
})
}
Try do it with Promise
function doStream1(cb) {
// put those operation on stream1 in the callback function...
cb && cb();
var p = new Promise(function(resolve, reject) {
stream1.on( 'end', resolve );
stream1.on( 'error', reject );
});
return p;
}
function doStream2(cb) {
// some operation on stream2 on callback
cb && cb();
var p = new Promise(function(resolve, reject) {
stream2.on( 'end', resolve );
stream2.on( 'error', reject );
});
return p;
}
doStream1(cb).then(function() {
return doStream2(cb);
}).catch(function(e) {
// error handling is here
});
Try the code below(sequenceStream function). The thing which I am worried about is error handling, but it should work.
Also, if you use Node < 10.* then you need end-of-stream instead of stream.finished
const stream = require('stream');
const process = require('process')
const { promisify } = require('util')
const fromList = (lst) => new stream.Readable({
read() {
if (lst.length) {
this.push(String(lst.shift()))
} else {
this.push(null)
}
}
})
const _finished = promisify(stream.finished)
const sequenceStream = (streams) => {
const resultingStream = new stream.PassThrough()
let isNext = Promise.resolve()
for(const [i, curStream] of streams.entries()) {
isNext = isNext.then(() => {
curStream.pipe(resultingStream, {end: i === streams.length -1})
return _finished(curStream)
}).catch((err) => {
resultingStream.write(err)
})
}
return resultingStream
}
sequenceStream([
fromList([1, 2, 3, 4]),
fromList([5, 6, 7, 8]),
fromList([9, 10])
]).pipe(process.stdout)

Resources