Trying to understand how to work with this async stream for a Firebase loader. I'm finding:
when I enable the Firebase Promise, the process stays open even though the last record is output to stdout.
With Firebase code commented as such, the process closes, but the final event is not written to console.
So I'm guessing Firebase is leaving some connection open that needs to be closed before the process exits, and that if I could figure out how to detect the end of the pipe, I can probably do that.
I've commented the Firebase stuff for now to remove it from the problem, but given this code, how do I run some code after the pipe completes? As it is the done function is not executing.
const csv = require('csv');
const fs = require('fs');
const firebase = require('firebase-admin');
const config = require('../../app-config');
const serviceAccount = require('../../service-account');
const jobConf = require('../../data/category.jobfile');
const _ = require('lodash');
// configure firebase
firebase.initializeApp({
credential: firebase.credential.cert(serviceAccount),
databaseURL: config.firebase.databaseURL,
});
console.log('Loading config', jobConf);
const counts = {
total: 0,
new: 0,
updated: 0,
}
const transformer = (record, done) => {
const newObj = {};
const fieldMaps = jobConf.fieldMaps;
Object.keys(record).forEach((key) => {
const destination = fieldMaps[key];
const value = _.get(record, key);
destination ? _.set(newObj, destination, value) : _.set(newObj, key, value);
});
Promise.resolve()
.then(() => {
done(null, JSON.stringify(newObj, null, 2));
});
// // now we look for existing row
// firebase.database().ref(`/categories`).orderByChild('identifier').equalTo(newObj.identifier)
// .once('value')
// .then((snap) => snap.val())
// .then((val) => {
// if (val) {
// console.log('Got value', val);
// newObj.lastUpdated = firebase.database.ServerValue.TIMESTAMP;
// } else {
// newObj.created = firebase.database.ServerValue.TIMESTAMP;
// newObj.lastUpdated = firebase.database.ServerValue.TIMESTAMP;
// counts.total++;
// counts.new++;
// done(null, JSON.stringify(newObj, null, 2));
// }
// })
// .catch((err) => {
// console.log(err);
// done(err);
// })
};
const stream = fs.createReadStream(`../../data/${jobConf.sourceFile}`);
const pipe = stream.pipe(csv.parse({columns: true})).pipe(csv.transform(transformer)).pipe(process.stdout);
pipe.on('finish', function () { // not executed
console.log('Done.');
console.log('Stats: ', counts);
process.exit(0);
});
Related
I have a setInterval function that's been called in another function, and I need to stop it when the proccess is done. I tried to set this setInterval function as a variable and call clearInterval, but the interval keeps running
const createInterval = (visibilityTimeout, startDateTime, message) => {
setInterval(() => {
const currentDateTime = moment().valueOf();
const timeDifference = (visibilityTimeout * 1000) - (currentDateTime - startDateTime);
if (timeDifference >= 600000) {
return;
}
if (timeDifference < 494983) {
const params = {
QueueUrl: 'http://localhost:4566/000000000000/test-queue2',
ReceiptHandle: message.ReceiptHandle,
VisibilityTimeout: visibilityTimeout,
};
sqs.changeMessageVisibility(params, (err, data) => {
if (err) logger.error(err, err.stack);
else logger.info(data);
});
// eslint-disable-next-line no-param-reassign
visibilityTimeout += 300;
}
}, 5000);
};
module.exports = async (message) => {
const startDateTime = moment().valueOf();
const {
noteId,
} = JSON.parse(message.Body);
logger.info(`Processing message [noteId=${noteId}]`);
try {
const note = await TestSessionNote.findById(noteId);
const testSession = await TestSession.findById(note.test_session_id);
logger.info(`Downloading video [key=${testSession.video_key}]`);
const isProcessing = true;
const interval = createInterval(500, startDateTime, message, isProcessing);
await sleep(20000);
clearInterval(interval);
logger.info(`Finished processing message [noteId=${noteId}]`);
} catch (ex) {
await TestSessionNote.update(noteId, { status: 'transcribe_error' });
logger.error(`Error processing message [noteId=${noteId}]`, ex);
}
};
I know that if i create a var test = setInterval(() => {console.log('blabla')}, 500) and call clearInterval(test) it works, but i don't know how can i do this calling a function
I think that you have to return from createInterval function the intervalId and after that it should work.
Can you check what value has your intervalId right now, with your current implementation?
https://developer.mozilla.org/en-US/docs/Web/API/setInterval
"The returned intervalID is a numeric, non-zero value which identifies the timer created by the call to setInterval(); this value can be passed to clearInterval() to cancel the interval."
below is the working code in which it can post images but is there any way i can also share videos as instagram story?
the error i get when i try to post video instead of image are:**
error image
PS D:\Softwares\programming\Insta Bot\story> node index.js
18:45:11 - info: Dry Run Activated
18:45:11 - info: Post() called! ======================
18:45:11 - debug: 1 files found in ./images/
18:45:11 - warn: Record file not found, saying yes to D:\Softwares\programming\Insta Bot\story\images\meme.mp4
18:45:11 - debug: Read File Success
18:45:11 - error: undefined
(MAIN CODE)
index.js
const logger = require("./logger.js")
const { random, sleep } = require('./utils')
require('dotenv').config();
const { IgApiClient, IgLoginTwoFactorRequiredError } = require("instagram-private-api");
const ig = new IgApiClient();
const Bluebird = require('bluebird');
const inquirer = require('inquirer');
const { CronJob } = require('cron');
const path = require("path");
const fs = require("fs");
const fsp = fs.promises;
const sharp = require("sharp");
//==================================================================================
const statePath = "./etc/state.conf";
const recordPath = "./etc/usedfiles.jsonl";
const imgFolderPath = "./images/";
const dryrun = true;
const runOnStart = true;
//==================================================================================
(async () => { // FOR AWAIT
// LOGIN TO INSTAGRAM
if (!dryrun) {
await login();
logger.info("Log In Successful");
} else {
logger.info("Dry Run Activated");
}
// SCHEDULER
// logger.silly("I'm a schedule, and I'm running!! :)");
const job = new CronJob('38 43 * * * *', post, null, true); //https://crontab.guru/
if (!runOnStart) logger.info(`Next few posts scheduled for: \n${job.nextDates(3).join("\n")}\n`);
else post();
// MAIN POST COMMAND
async function post() {
logger.info("Post() called! ======================");
let postPromise = fsp.readdir(imgFolderPath)
.then(filenames => {
if (filenames.length < 1) throw new Error(`Folder ${imgFolderPath} is empty...`)
logger.debug(`${filenames.length} files found in ${imgFolderPath}`);
return filenames;
})
.then(filenames => filenames.map(file => path.resolve(imgFolderPath + file)))
.then(filenames => pickUnusedFileFrom(filenames, filenames.length))
.then(filename => {
if (!dryrun) registerFileUsed(filename)
return filename
})
.then(fsp.readFile)
.then(async buffer => {
logger.debug("Read File Success "); //TODO move this to previous then?
return sharp(buffer).jpeg().toBuffer()
.then(file => {
logger.debug("Sharp JPEG Success");
return file
})
})
.then(async file => {
if (!dryrun) {
// await sleep(random(1000, 60000)) //TODO is this necessary?
return ig.publish.story({ file })
.then(fb => logger.info("Posting successful!?"))
}
else return logger.info("Data not sent, dryrun = true")
})
.then(() => logger.info(`Next post scheduled for ${job.nextDates()}\n`))
.catch(logger.error)
}
})();
//=================================================================================
async function login() {
ig.state.generateDevice(process.env.IG_USERNAME);
// ig.state.proxyUrl = process.env.IG_PROXY;
//register callback?
ig.request.end$.subscribe(async () => {
const serialized = await ig.state.serialize();
delete serialized.constants; // this deletes the version info, so you'll always use the version provided by the library
await stateSave(serialized);
});
if (await stateExists()) {
// import state accepts both a string as well as an object
// the string should be a JSON object
const stateObj = await stateLoad();
await ig.state.deserialize(stateObj)
.catch(err => logger.debug("deserialize: " + err));
} else {
let standardLogin = async function() {
// login like normal
await ig.simulate.preLoginFlow();
logger.debug("preLoginFlow finished");
await ig.account.login(process.env.IG_USERNAME, process.env.IG_PASSWORD);
logger.info("Logged in as " + process.env.IG_USERNAME);
process.nextTick(async () => await ig.simulate.postLoginFlow());
logger.debug("postLoginFlow finished");
}
// Perform usual login
// If 2FA is enabled, IgLoginTwoFactorRequiredError will be thrown
return Bluebird.try(standardLogin)
.catch(
IgLoginTwoFactorRequiredError,
async err => {
logger.info("Two Factor Auth Required");
const {username, totp_two_factor_on, two_factor_identifier} = err.response.body.two_factor_info;
// decide which method to use
const verificationMethod = totp_two_factor_on ? '0' : '1'; // default to 1 for SMS
// At this point a code should have been sent
// Get the code
const { code } = await inquirer.prompt([
{
type: 'input',
name: 'code',
message: `Enter code received via ${verificationMethod === '1' ? 'SMS' : 'TOTP'}`,
},
]);
// Use the code to finish the login process
return ig.account.twoFactorLogin({
username,
verificationCode: code,
twoFactorIdentifier: two_factor_identifier,
verificationMethod, // '1' = SMS (default), '0' = TOTP (google auth for example)
trustThisDevice: '1', // Can be omitted as '1' is used by default
});
},
)
.catch(e => logger.error('An error occurred while processing two factor auth', e, e.stack));
}
return
//================================================================================
async function stateSave(data) {
// here you would save it to a file/database etc.
await fsp.mkdir(path.dirname(statePath), { recursive: true }).catch(logger.error);
return fsp.writeFile(statePath, JSON.stringify(data))
// .then(() => logger.info('state saved, daddy-o'))
.catch(err => logger.error("Write error" + err));
}
async function stateExists() {
return fsp.access(statePath, fs.constants.F_OK)
.then(() => {
logger.debug('Can access state info')
return true
})
.catch(() => {
logger.warn('Cannot access state info')
return false
});
}
async function stateLoad() {
// here you would load the data
return fsp.readFile(statePath, 'utf-8')
.then(data => JSON.parse(data))
.then(data => {
logger.info("State load successful");
return data
})
.catch(logger.error)
}
}
async function registerFileUsed( filepath ) {
let data = JSON.stringify({
path: filepath,
time: new Date().toISOString()
}) + '\n';
return fsp.appendFile(recordPath, data, { encoding: 'utf8', flag: 'a+' } )
.then(() => {
logger.debug("Writing filename to record file");
return filepath
})
}
function pickUnusedFileFrom( filenames, iMax = 1000) {
return new Promise((resolve, reject) => {
let checkFileUsed = async function ( filepath ) {
return fsp.readFile(recordPath, 'utf8')
.then(data => data.split('\n'))
.then(arr => arr.filter(Boolean))
.then(arr => arr.map(JSON.parse))
.then(arr => arr.some(entry => entry.path === filepath))
}
let trythis = function( iMax, i = 1) {
let file = random(filenames);
checkFileUsed(file)
.then(async used => {
if (!used) {
logger.info(`Unused file found! ${file}`);
resolve(file);
} else if (i < iMax) {
logger.debug(`Try #${i}: File ${file} used already`);
await sleep(50);
trythis(iMax, ++i)
} else {
reject(`I tried ${iMax} times and all the files I tried were previously used`)
}
})
.catch(err => {
logger.warn("Record file not found, saying yes to " + file);
resolve(file);
})
}( iMax );
})
}
I am trying to run a series of tasks. Each task is dynamic, and could have different rules to follow. This will be executed on AWS-Lambda.
I have an array of JSON. It has a body with task name in it, and it also has attributes.
I need to dynamically load a javascript file with the name inside the body.
I need to wait until all is finished inside that task. Or it failed (regardless where). If the fail happens, I will need to write that data inside the current record inside the forEach loop.
I have old issue, where my forEach is finished first without waiting for the task to complete.
This is the forEach loop:
const jobLoader = require('./Helpers/jobLoader');
event.Records.forEach(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
job.runJob(messageAttributes).then(res => {
console.log('Show results');
return; // resume another record from forEach
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
});
console.log('All Done');
});
The problem is that message All Done is printed, and then the message show results is printed. I get results from the database once it comes for execution.
This is the file that loads a task:
exports.loadJob = (jobName) => {
const job = require(`../Tasks/${jobName}`);
return job;
};
This is the file that contains actual task:
const mySqlConnector = require('../Storage/mySql');
exports.runJob = async (params) => {
let payload = {};
let dataToSend = await getUserName(params.userId.stringValue);
payload.dataToSend = dataToSend;
let moreDataToSend = await getEvenMoreData(params.userId.stringValue);
payload.moreDataToSend = moreDataToSend;
return await sendData(payload);
};
const getUserName = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const getEvenMoreData = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const sendData = (payload) => {
//this should be Axios sending data
};
And this is the mySql connector itself:
const mysql = require('promise-mysql');
exports.handler = async (query) => {
return mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database: 'crm'
}).then(conn =>{
let result = conn.query(query);
conn.end();
return result;
}).then(rows => {
//console.log("These are rows:" + rows);
return rows;
}).catch(error => {
return error;
});
};
The task file can have any number of things it needs to complete, which will be different when I start adding tasks.
I need that job.runJob completes, or that it catches an error, from whatever location it originated, so I can continue with the forEach.
I have tried using map and what not, but the end result is always the same.
What am I doing wrong?
You can use Promise.all method :
const promises = event.Records.map(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
return job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
throw new Error('Your error !');
});
});
try {
const results = await Promise.all(promises);
console.log('All done');
} catch (e) {
console.log('Something has an error', e);
}
don't forget to make your function async !
I managed to solve it, and still keep details about the execution:
Something like this:
for (let prop in event.Records){
const { body: jobName } = event.Records[prop];
const { messageAttributes } = event.Records[prop];
const job = jobLoader.loadJob(jobName);
await job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
event.Records[prop].failed = true;
event.Records[prop].failed = err.message;
console.log('I errored');
});
}
I have a file in which mongoose is setup
const keys = require('../chat/config/keys');
const mongoose = require('mongoose');
const dbURI = keys.mongoURI;
mongoose.connect(dbURI, { useNewUrlParser: true });
mongoose.set('debug', true);
let fetchVideo;
mongoose.connection.once('open', function () {
console.log('Mongoose default connection open to ' + dbURI);
let connectToDB = mongoose.connection.db;
let videoChatDB = connectToDB.collection('videochat');
fetchVideo = ({ id }) => {
if (id !== '100') {
videoChatDB.findOne({'studentID': parseInt(id)}).then((user) => {
if (user) {
console.log(user);
return true;
} else {
console.log(user);
return false;
}
});
}
}
});
module.exports = { fetchVideo };
And I am requiring that file inside my index.js file like so:
let db = require('./db');
In my index file I have a socket connection and I need to check the database when a new user comes.
socket.on('new-user', async (user) => {
let checkAvail = await db.fetchVideo(user);
});
But I am getting this error:
TypeError: db.fetchVideo is not a function
I am guessing it is undefined since it is declared inside an asynchronous function.
How would I make this to work?
Because the function is created asynchronously, one option is to export a Promise that resolves to fetchVideo function. Because mongoose.connection.once is callback-based, you'll have to transform it into a Promise.
If you want fetchVideo to resolve to something (rather than nothing), you also have to properly chain the findOne call with the promise chain; to fix that, return videoChatDB.findOne....
const fetchVideoProm = new Promise((res, rej) => {
mongoose.connection.once('open', function () {
console.log('Mongoose default connection open to ' + dbURI);
let connectToDB = mongoose.connection.db;
let videoChatDB = connectToDB.collection('videochat');
const fetchVideo = ({ id }) => {
if (id !== '100') {
return videoChatDB.findOne({'studentID': parseInt(id)}).then((user) => {
if (user) {
console.log(user);
return true;
} else {
console.log(user);
return false;
}
});
}
}
res(fetchVideo);
});
});
module.exports = { fetchVideoProm };
Consume it by awaiting the creation of the fetchVideo function, and then calling it:
socket.on('new-user', async (user) => {
const fetchVideo = await db.fetchVideoProm;
const checkAvail = await fetchVideo(user);
});
Given two streams, stream1, stream2, how can I run them in sequence, throwing if any fails?
I'm looking for something simpler than this:
stream1.on('end',function(){
stream2.on('end',done);
});
stream1.on('error',function(error){done(error);});
stream2.on('error',function(error){done(error);});
Thanks.
there are some ways to do that, check next link, it gonna help to how to write some code in node in a elegant way:
Node.js FLOW
Hope it what you need.
Depending on your use case you could combine the two streams into one by using the multistream module.
Multistreams are constructed from an array of streams
var MultiStream = require('multistream')
var fs = require('fs')
var streams = [
fs.createReadStream(__dirname + '/numbers/1.txt'), // contains a single char '1'
fs.createReadStream(__dirname + '/numbers/2.txt'), // contains a single char '2'
fs.createReadStream(__dirname + '/numbers/3.txt') // contains a single char '3'
]
MultiStream(streams).pipe(process.stdout) // => 123
In case combining streams does not fit the use case you could build your stream on end event sending functionality on your own
const fs = require('fs');
var number1 = fs.createReadStream('./numbers1.txt')
.on('data', d => console.log(d.toString()));
var number2 = fs.createReadStream('./numbers2.txt')
.on('data', d => console.log(d.toString()));
onEnd([number1, number2], function(err) {
console.log('Ended with', err);
});
function onEnd(streams, cb) {
var count = streams.length;
var ended = 0;
var errored = null;
function shouldEnd() {
ended++;
if (errored) { return; }
if (count == ended) {
cb();
}
}
function endWithError(err) {
if (errored) { return; }
errored = true;
cb(err);
}
streams.forEach(s => s
.on('end', shouldEnd)
.on('error', endWithError)
);
}
The onEnd function can be used to wait for a stream array to end or in case an error event is emitted for the first emitted error event.
Try do it with async functions:
const { createReadStream } = require("fs")
async function main() {
const stream1 = createReadStream(__dirname + "/1.txt")
await pipe(stream1, process.stdout)
const stream2 = createReadStream(__dirname + "/2.txt")
await pipe(stream2, process.stdout)
const stream3 = createReadStream(__dirname + "/3.txt")
await pipe(stream3, process.stdout)
}
async function pipe(tap, sink) {
return new Promise((resolve, reject) => {
tap.pipe(sink, { end: false })
tap.on("end", resolve)
tap.on("error", reject)
})
}
Try do it with Promise
function doStream1(cb) {
// put those operation on stream1 in the callback function...
cb && cb();
var p = new Promise(function(resolve, reject) {
stream1.on( 'end', resolve );
stream1.on( 'error', reject );
});
return p;
}
function doStream2(cb) {
// some operation on stream2 on callback
cb && cb();
var p = new Promise(function(resolve, reject) {
stream2.on( 'end', resolve );
stream2.on( 'error', reject );
});
return p;
}
doStream1(cb).then(function() {
return doStream2(cb);
}).catch(function(e) {
// error handling is here
});
Try the code below(sequenceStream function). The thing which I am worried about is error handling, but it should work.
Also, if you use Node < 10.* then you need end-of-stream instead of stream.finished
const stream = require('stream');
const process = require('process')
const { promisify } = require('util')
const fromList = (lst) => new stream.Readable({
read() {
if (lst.length) {
this.push(String(lst.shift()))
} else {
this.push(null)
}
}
})
const _finished = promisify(stream.finished)
const sequenceStream = (streams) => {
const resultingStream = new stream.PassThrough()
let isNext = Promise.resolve()
for(const [i, curStream] of streams.entries()) {
isNext = isNext.then(() => {
curStream.pipe(resultingStream, {end: i === streams.length -1})
return _finished(curStream)
}).catch((err) => {
resultingStream.write(err)
})
}
return resultingStream
}
sequenceStream([
fromList([1, 2, 3, 4]),
fromList([5, 6, 7, 8]),
fromList([9, 10])
]).pipe(process.stdout)