nodejs process.exit() prevents function from executing properly - node.js

I am writing my first NodeJs script. Below is some code I have set up to test a database connection.
When I include process.exit() at the very end of the script, nothing is logged to the console - however, when I simply remove that line of code, the function logs the query results appropriately (output included also).
I am wondering why the process.exit() at the very end of the code prevents the function from functioning and if I am using the exit method wrong.
Code:
/*
* Runs every minute on an uptime agent
* Checks list of sites to see if they're up
*/
// strict mode (see http://stackoverflow.com/questions/8651415/what-is-strict-mode-and-how-is-it-used for information)
'use strict';
// cuz im lazy
String.prototype.lc = function() { return this.toLowerCase(); }
String.prototype.uc = function() { return this.toUpperCase(); }
/** EXCEPTIONS **/
var DbConnectError = function(m) {
this.name = 'DbConnectError';
this.message = m;
}
var DbConfigError = function(m) {
this.name = 'DbConfigError';
this.message = m;
}
/** DATABSE **/
/*
* change log
* 08/07/2015 Tyler J Barnes
* -- init dev
*/
var db = function() {
// error messages
this._errors = [];
// connection state
this._isConnected = false;
// db configuration
this._config = {
host: '',
user: '',
password: '',
database: ''
};
// reference
this._db = null;
// connection obj ref
this._con = null;
// sql statement
this._sqlStmt = '';
// is prepared statement -- needs data binded
this._isPrepared = false;
// data to bind to prepared stmts
this._sqlData = null;
// query result set
this._result = null;
/*
* initialize
* #param (object) : cofig prop and values
* #return void
*/
this.ini = function(config) {
// make sure config was passed
if(!config || (typeof config).lc() != 'object') {
throw new DbConnectError('Invalid DB Configuration');
}
// check for appropriate properties
// if exist, store value
for(var p in this._config) {
if(!(p in config)) {
this._errors.push('Missing database config: '+p+'\n');
} else {
this._config[p] = config[p];
}
}
// throw any errors before continue
if(this._errors.length > 0) {
var tmp = '';
for(var i = 0; i < this._errors.length; i++) {
tmp+=this._errors[i];
}
throw new DbConfigError(tmp);
}
this._db = require('mysql');
};
// create connection -- returns thread id
this.con = function() {
this._con = this._db.createConnection(this._config);
this._con.connect();
this._isConnected = true;
};
// sets sql statement
this.setSqlStmt = function(str, prepared, bindData) {
this._sqlStmt = str;
if(prepared) {
this._isPrepared = true;
this._sqlData = bindData;
} else {
this._isPrepared = false;
this._sqlData = null;
}
};
// kills connection
this.die = function() {
if(this._isConnected) {
this._con.end();
}
};
}
var c = {
host: 'asdfasdf',
user: 'asdfasdf',
password: 'asdfasdf',
database: 'asdfasdf'
};
var d = new db();
d.ini(c);
d.con();
d._con.query('SELECT * FROM Agents', function(err,rows,fields) {
if(err) console.log(err);
console.log(rows);
});
d._con.end();
// this is what upsets me
process.exit();
Output when process.exit() is removed:
[{agentid:1, host: 'asdfasdfadf'....etc}]

The database query operation is asynchronous. This means that it will only be concluded after the program executes all of the main module, including the exit system call.
Instead, you should only terminate the process when the results are obtained from the database:
var d = new db();
d.ini(c);
d.con();
d._con.query('SELECT * FROM Agents', function(err,rows,fields) {
if(err) console.log(err);
console.log(rows);
process.exit();
});
d._con.end();
This is one of the typical misunderstandings of how JavaScript's asynchronous function calls work. I would advise you to read on the subject. These two questions may have some useful content:
How does Asynchronous Javascript Execution happen? and when not to use return statement?
How does JavaScript handle AJAX responses in the background?

Related

How to connect my electron app using PouchDB (leveldb) with Cloudant or any other database that support couchDB and sync

I'm creating an electron app using pouchDB and I want the app to be able to diferents customers and sync the data between them. As an example I'm making the tutorial: https://github.com/nolanlawson/pouchdb-getting-started-todo, I adapt the code to electron and I created a noSQL database at cloudant.
At the moment I can save data but I cannot sync with my remote db that is in cloudant. Here is the endpoint I'm using to sync data between both database.
Here is the error that I'm getting.
Here is the code of my script.js
(function() {
'use strict';
var $ = document.querySelector.bind(document);
var ENTER_KEY = 13;
var newTodoDom = document.getElementById('new_todo');
var syncDom = document.getElementById('sync-wrapper');
// EDITING STARTS HERE (you dont need to edit anything above this line)
var NodePouchDB = require('pouchdb');
var db = new NodePouchDB('todos');
var couchdb = require('felix-couchdb')
var remoteCouch = couchdb.createClient(5984, 'https://ac725f4e-29ec-4614-8e96-02ebc74a529b-bluemix.cloudant.com/')
db.info(function(err, info) {
console.log("is working", info)
db.changes({
since: info.update_seq,
live: true
}).on('change', showTodos);
});
// We have to create a new todo document and enter it in the database
function addTodo(text) {
var todo = {
_id: new Date().toISOString(),
title: text,
completed: false
};
db.put(todo).then(function (result) {
console.log("everything is A-OK");
console.log(result);
}).catch(function (err) {
console.log('everything is terrible');
console.log(err);
});
}
// Show the current list of todos by reading them from the database
function showTodos() {
db.allDocs({include_docs: true, descending: true}).then(function(doc) {
redrawTodosUI(doc.rows);
}).catch(function (err) {
console.log(err);
});
}
function checkboxChanged(todo, event) {
todo.completed = event.target.checked;
console.log(todo);
db.put(todo);
}
// User pressed the delete button for a todo, delete it
function deleteButtonPressed(todo) {
db.remove(todo);
}
// The input box when editing a todo has blurred, we should save
// the new title or delete the todo if the title is empty
function todoBlurred(todo, event) {
var trimmedText = event.target.value.trim();
if (!trimmedText) {
db.remove(todo);
} else {
todo.title = trimmedText;
db.put(todo);
}
}
// Initialise a sync with the remote server
function sync() {
syncDom.setAttribute('data-sync-state', 'syncing');
var opts = {live: true};
db.sync(remoteCouch, opts, syncError);
}
// EDITING STARTS HERE (you dont need to edit anything below this line)
// There was some form or error syncing
function syncError() {
syncDom.setAttribute('data-sync-state', 'error');
}
// User has double clicked a todo, display an input so they can edit the title
function todoDblClicked(todo) {
var div = document.getElementById('li_' + todo._id);
var inputEditTodo = document.getElementById('input_' + todo._id);
div.className = 'editing';
inputEditTodo.focus();
}
// If they press enter while editing an entry, blur it to trigger save
// (or delete)
function todoKeyPressed(todo, event) {
if (event.keyCode === ENTER_KEY) {
var inputEditTodo = document.getElementById('input_' + todo._id);
inputEditTodo.blur();
}
}
// Given an object representing a todo, this will create a list item
// to display it.
function createTodoListItem(todo) {
var checkbox = document.createElement('input');
checkbox.className = 'toggle';
checkbox.type = 'checkbox';
checkbox.addEventListener('change', checkboxChanged.bind(this, todo));
var label = document.createElement('label');
label.appendChild( document.createTextNode(todo.title));
label.addEventListener('dblclick', todoDblClicked.bind(this, todo));
var deleteLink = document.createElement('button');
deleteLink.className = 'destroy';
deleteLink.addEventListener( 'click', deleteButtonPressed.bind(this, todo));
var divDisplay = document.createElement('div');
divDisplay.className = 'view';
divDisplay.appendChild(checkbox);
divDisplay.appendChild(label);
divDisplay.appendChild(deleteLink);
var inputEditTodo = document.createElement('input');
inputEditTodo.id = 'input_' + todo._id;
inputEditTodo.className = 'edit';
inputEditTodo.value = todo.title;
inputEditTodo.addEventListener('keypress', todoKeyPressed.bind(this, todo));
inputEditTodo.addEventListener('blur', todoBlurred.bind(this, todo));
var li = document.createElement('li');
li.id = 'li_' + todo._id;
li.appendChild(divDisplay);
li.appendChild(inputEditTodo);
if (todo.completed) {
li.className += 'complete';
checkbox.checked = true;
}
return li;
}
function redrawTodosUI(todos) {
var ul = document.getElementById('todo-list');
ul.innerHTML = '';
todos.forEach(function(todo) {
ul.appendChild(createTodoListItem(todo.doc));
});
}
function newTodoKeyPressHandler( event ) {
if (event.keyCode === ENTER_KEY) {
addTodo(newTodoDom.value);
newTodoDom.value = '';
}
}
function addEventListeners() {
newTodoDom.addEventListener('keypress', newTodoKeyPressHandler, false);
}
addEventListeners();
showTodos();
if (remoteCouch) {
sync();
}
})();
To get to where the problem sits, can you verify that you can speak to the Cloudant database normally, that is using curl from the command-line? Using curl, fetch a document by its _id, perhaps a document you created manually using the Cloudant dashboard. That should shake out any problems with authentication: I note you're using IAM, which isn't always straight-forward -- and to my knowledge, not supported by PouchDB (or wasn't, last time I looked).
If that is the problem, create a new Cloudant instance with IAM+Legacy credentials.

How to connect Javascript to a remote IBM MQ?

I'm making APIs with LoopBack( a Javascript Framework), pushing and getting messages with IBM MQ.
I followed this tut: Nodejs and MQ
I can do it with local Queue Manager, but I dont know how to connect to a remote Queue Manager.
So, can any one explain me how to do this ?
Tks all.
I can do it with that link, which from #JoshMc's comment.
This is my code, it works fine:
module.exports = function (server) {
var mq = require('ibmmq');
var MQC = mq.MQC; // Want to refer to this export directly for simplicity
// The queue manager and queue to be used. These can be overridden on command line.
var qMgr = "QM1";
var qName = "soa.log";
var mqmd = new mq.MQMD(); // Defaults are fine.
var pmo = new mq.MQPMO();
var cd = new mq.MQCD();
var cno = new mq.MQCNO();
cd.ConnectionName = "localhost(1414)";
cd.ChannelName = "CHAN1";
var csp = new mq.MQCSP();
cno.ClientConn = cd;
cno.Options = MQC.MQCNO_CLIENT_BINDING; // use MQCNO_CLIENT_BINDING to connect as client
function putMessage(hObj) {
var msg = Buffer.from(JSON.stringify(coff));
// Describe how the Put should behave
pmo.Options = MQC.MQPMO_NO_SYNCPOINT |
MQC.MQPMO_NEW_MSG_ID |
MQC.MQPMO_NEW_CORREL_ID;
mq.Put(hObj,mqmd,pmo,msg,function(err) {
if (err) {
console.log(formatErr(err));
} else {
console.log("MQPUT successful");
}
});
}
mq.Connx(qMgr, cno, function (err, hConn) {
if (err) {
console.log((err));
} else {
console.log("MQCONN to %s successful ", qMgr);
// Define what we want to open, and how we want to open it.
var od = new mq.MQOD();
od.ObjectName = qName;
od.ObjectType = MQC.MQOT_Q;
var openOptions = MQC.MQOO_OUTPUT;
mq.Open(hConn, od, openOptions, function (err, hObj) {
if (err) {
console.log(formatErr(err));
} else {
console.log("MQOPEN of %s successful", qName);
putMessage(hObj);
}
// cleanup(hConn, hObj);
});
}
});
};

Delete a Document with all Subcollections and Nested Subcollections in Firestore

How can you delete a Document with all it's collections and nested subcollections? (inside the functions environment)
In the RTDB you can ref.child('../someNode).setValue(null) and that completes the desired behavior.
I can think of two ways you could achieve the desired delete behavior, both with tremendously ghastly drawbacks.
Create a 'Super' function that will spider every document and delete them in a batch.
This function would be complicated, brittle to changes, and might take a lengthy execution time.
Add 'onDelete' triggers for each Document type, and make it delete any direct subcollections. You'll call delete on the root document, and the deletion calls will propagate down the 'tree'. This is sluggish, scales atrociously and is costly due to the colossal load of function executions.
Imagine you would have to delete a 'GROUP' and all it's children. It would be deeply chaotic with #1 and pricey with #2 (1 function call per doc)
groups > GROUP > projects > PROJECT > files > FILE > assets > ASSET
> urls > URL
> members > MEMBER
> questions > QUESTION > answers > ANSWER > replies > REPLY
> comments > COMMENT
> resources > RESOURCE > submissions > SUBMISSION
> requests > REQUEST
Is there a superior/favored/cleaner way to delete a document and all it's nested subcollections?
It ought to be possible considering you can do it from the console.
according to firebase documentation:
https://firebase.google.com/docs/firestore/solutions/delete-collections
Deleting collection with nested subcollections might be done easy and neat with node-JS on the server side.
const client = require('firebase-tools');
await client.firestore
.delete(collectionPath, {
project: process.env.GCLOUD_PROJECT,
recursive: true,
yes: true
});
Unfortunately, your analysis is spot on and indeed this use case does require a lot of ceremony. According to official documentation, there is no support for deep deletes in a single shot in firestore neither via client libraries nor rest-api nor the cli tool.
The cli is open sourced and its implementation lives here: https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js. They basically implemented option 1. you described in your question, so you can take some inspiration from there.
Both options 1. and 2. are far from ideal situation and to make your solution 100% reliable you will need to keep a persistent queue with deletion tasks, as any error in the long running procedure will leave your system in some ill-defined state.
I would discourage to go with raw option 2. as recursive cloud function calls may very easily went wrong - for example, hitting max. limits.
In case the link changed, below the full source of https://github.com/firebase/firebase-tools/blob/master/src/firestore/delete.js:
"use strict";
var clc = require("cli-color");
var ProgressBar = require("progress");
var api = require("../api");
var firestore = require("../gcp/firestore");
var FirebaseError = require("../error");
var logger = require("../logger");
var utils = require("../utils");
/**
* Construct a new Firestore delete operation.
*
* #constructor
* #param {string} project the Firestore project ID.
* #param {string} path path to a document or collection.
* #param {boolean} options.recursive true if the delete should be recursive.
* #param {boolean} options.shallow true if the delete should be shallow (non-recursive).
* #param {boolean} options.allCollections true if the delete should universally remove all collections and docs.
*/
function FirestoreDelete(project, path, options) {
this.project = project;
this.path = path;
this.recursive = Boolean(options.recursive);
this.shallow = Boolean(options.shallow);
this.allCollections = Boolean(options.allCollections);
// Remove any leading or trailing slashes from the path
if (this.path) {
this.path = this.path.replace(/(^\/+|\/+$)/g, "");
}
this.isDocumentPath = this._isDocumentPath(this.path);
this.isCollectionPath = this._isCollectionPath(this.path);
this.allDescendants = this.recursive;
this.parent = "projects/" + project + "/databases/(default)/documents";
// When --all-collections is passed any other flags or arguments are ignored
if (!options.allCollections) {
this._validateOptions();
}
}
/**
* Validate all options, throwing an exception for any fatal errors.
*/
FirestoreDelete.prototype._validateOptions = function() {
if (this.recursive && this.shallow) {
throw new FirebaseError("Cannot pass recursive and shallow options together.");
}
if (this.isCollectionPath && !this.recursive && !this.shallow) {
throw new FirebaseError("Must pass recursive or shallow option when deleting a collection.");
}
var pieces = this.path.split("/");
if (pieces.length === 0) {
throw new FirebaseError("Path length must be greater than zero.");
}
var hasEmptySegment = pieces.some(function(piece) {
return piece.length === 0;
});
if (hasEmptySegment) {
throw new FirebaseError("Path must not have any empty segments.");
}
};
/**
* Determine if a path points to a document.
*
* #param {string} path a path to a Firestore document or collection.
* #return {boolean} true if the path points to a document, false
* if it points to a collection.
*/
FirestoreDelete.prototype._isDocumentPath = function(path) {
if (!path) {
return false;
}
var pieces = path.split("/");
return pieces.length % 2 === 0;
};
/**
* Determine if a path points to a collection.
*
* #param {string} path a path to a Firestore document or collection.
* #return {boolean} true if the path points to a collection, false
* if it points to a document.
*/
FirestoreDelete.prototype._isCollectionPath = function(path) {
if (!path) {
return false;
}
return !this._isDocumentPath(path);
};
/**
* Construct a StructuredQuery to find descendant documents of a collection.
*
* See:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
*
* #param {boolean} allDescendants true if subcollections should be included.
* #param {number} batchSize maximum number of documents to target (limit).
* #param {string=} startAfter document name to start after (optional).
* #return {object} a StructuredQuery.
*/
FirestoreDelete.prototype._collectionDescendantsQuery = function(
allDescendants,
batchSize,
startAfter
) {
var nullChar = String.fromCharCode(0);
var startAt = this.parent + "/" + this.path + "/" + nullChar;
var endAt = this.parent + "/" + this.path + nullChar + "/" + nullChar;
var where = {
compositeFilter: {
op: "AND",
filters: [
{
fieldFilter: {
field: {
fieldPath: "__name__",
},
op: "GREATER_THAN_OR_EQUAL",
value: {
referenceValue: startAt,
},
},
},
{
fieldFilter: {
field: {
fieldPath: "__name__",
},
op: "LESS_THAN",
value: {
referenceValue: endAt,
},
},
},
],
},
};
var query = {
structuredQuery: {
where: where,
limit: batchSize,
from: [
{
allDescendants: allDescendants,
},
],
select: {
fields: [{ fieldPath: "__name__" }],
},
orderBy: [{ field: { fieldPath: "__name__" } }],
},
};
if (startAfter) {
query.structuredQuery.startAt = {
values: [{ referenceValue: startAfter }],
before: false,
};
}
return query;
};
/**
* Construct a StructuredQuery to find descendant documents of a document.
* The document itself will not be included
* among the results.
*
* See:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/StructuredQuery
*
* #param {boolean} allDescendants true if subcollections should be included.
* #param {number} batchSize maximum number of documents to target (limit).
* #param {string=} startAfter document name to start after (optional).
* #return {object} a StructuredQuery.
*/
FirestoreDelete.prototype._docDescendantsQuery = function(allDescendants, batchSize, startAfter) {
var query = {
structuredQuery: {
limit: batchSize,
from: [
{
allDescendants: allDescendants,
},
],
select: {
fields: [{ fieldPath: "__name__" }],
},
orderBy: [{ field: { fieldPath: "__name__" } }],
},
};
if (startAfter) {
query.structuredQuery.startAt = {
values: [{ referenceValue: startAfter }],
before: false,
};
}
return query;
};
/**
* Query for a batch of 'descendants' of a given path.
*
* For document format see:
* https://firebase.google.com/docs/firestore/reference/rest/v1beta1/Document
*
* #param {boolean} allDescendants true if subcollections should be included,
* #param {number} batchSize the maximum size of the batch.
* #param {string=} startAfter the name of the document to start after (optional).
* #return {Promise<object[]>} a promise for an array of documents.
*/
FirestoreDelete.prototype._getDescendantBatch = function(allDescendants, batchSize, startAfter) {
var url;
var body;
if (this.isDocumentPath) {
url = this.parent + "/" + this.path + ":runQuery";
body = this._docDescendantsQuery(allDescendants, batchSize, startAfter);
} else {
url = this.parent + ":runQuery";
body = this._collectionDescendantsQuery(allDescendants, batchSize, startAfter);
}
return api
.request("POST", "/v1beta1/" + url, {
auth: true,
data: body,
origin: api.firestoreOrigin,
})
.then(function(res) {
// Return the 'document' property for each element in the response,
// where it exists.
return res.body
.filter(function(x) {
return x.document;
})
.map(function(x) {
return x.document;
});
});
};
/**
* Progress bar shared by the class.
*/
FirestoreDelete.progressBar = new ProgressBar("Deleted :current docs (:rate docs/s)", {
total: Number.MAX_SAFE_INTEGER,
});
/**
* Repeatedly query for descendants of a path and delete them in batches
* until no documents remain.
*
* #return {Promise} a promise for the entire operation.
*/
FirestoreDelete.prototype._recursiveBatchDelete = function() {
var self = this;
// Tunable deletion parameters
var readBatchSize = 7500;
var deleteBatchSize = 250;
var maxPendingDeletes = 15;
var maxQueueSize = deleteBatchSize * maxPendingDeletes * 2;
// All temporary variables for the deletion queue.
var queue = [];
var numPendingDeletes = 0;
var pagesRemaining = true;
var pageIncoming = false;
var lastDocName;
var failures = [];
var retried = {};
var queueLoop = function() {
if (queue.length == 0 && numPendingDeletes == 0 && !pagesRemaining) {
return true;
}
if (failures.length > 0) {
logger.debug("Found " + failures.length + " failed deletes, failing.");
return true;
}
if (queue.length <= maxQueueSize && pagesRemaining && !pageIncoming) {
pageIncoming = true;
self
._getDescendantBatch(self.allDescendants, readBatchSize, lastDocName)
.then(function(docs) {
pageIncoming = false;
if (docs.length == 0) {
pagesRemaining = false;
return;
}
queue = queue.concat(docs);
lastDocName = docs[docs.length - 1].name;
})
.catch(function(e) {
logger.debug("Failed to fetch page after " + lastDocName, e);
pageIncoming = false;
});
}
if (numPendingDeletes > maxPendingDeletes) {
return false;
}
if (queue.length == 0) {
return false;
}
var toDelete = [];
var numToDelete = Math.min(deleteBatchSize, queue.length);
for (var i = 0; i < numToDelete; i++) {
toDelete.push(queue.shift());
}
numPendingDeletes++;
firestore
.deleteDocuments(self.project, toDelete)
.then(function(numDeleted) {
FirestoreDelete.progressBar.tick(numDeleted);
numPendingDeletes--;
})
.catch(function(e) {
// For server errors, retry if the document has not yet been retried.
if (e.status >= 500 && e.status < 600) {
logger.debug("Server error deleting doc batch", e);
// Retry each doc up to one time
toDelete.forEach(function(doc) {
if (retried[doc.name]) {
logger.debug("Failed to delete doc " + doc.name + " multiple times.");
failures.push(doc.name);
} else {
retried[doc.name] = true;
queue.push(doc);
}
});
} else {
logger.debug("Fatal error deleting docs ", e);
failures = failures.concat(toDelete);
}
numPendingDeletes--;
});
return false;
};
return new Promise(function(resolve, reject) {
var intervalId = setInterval(function() {
if (queueLoop()) {
clearInterval(intervalId);
if (failures.length == 0) {
resolve();
} else {
reject("Failed to delete documents " + failures);
}
}
}, 0);
});
};
/**
* Delete everything under a given path. If the path represents
* a document the document is deleted and then all descendants
* are deleted.
*
* #return {Promise} a promise for the entire operation.
*/
FirestoreDelete.prototype._deletePath = function() {
var self = this;
var initialDelete;
if (this.isDocumentPath) {
var doc = { name: this.parent + "/" + this.path };
initialDelete = firestore.deleteDocument(doc).catch(function(err) {
logger.debug("deletePath:initialDelete:error", err);
if (self.allDescendants) {
// On a recursive delete, we are insensitive to
// failures of the initial delete
return Promise.resolve();
}
// For a shallow delete, this error is fatal.
return utils.reject("Unable to delete " + clc.cyan(this.path));
});
} else {
initialDelete = Promise.resolve();
}
return initialDelete.then(function() {
return self._recursiveBatchDelete();
});
};
/**
* Delete an entire database by finding and deleting each collection.
*
* #return {Promise} a promise for all of the operations combined.
*/
FirestoreDelete.prototype.deleteDatabase = function() {
var self = this;
return firestore
.listCollectionIds(this.project)
.catch(function(err) {
logger.debug("deleteDatabase:listCollectionIds:error", err);
return utils.reject("Unable to list collection IDs");
})
.then(function(collectionIds) {
var promises = [];
logger.info("Deleting the following collections: " + clc.cyan(collectionIds.join(", ")));
for (var i = 0; i < collectionIds.length; i++) {
var collectionId = collectionIds[i];
var deleteOp = new FirestoreDelete(self.project, collectionId, {
recursive: true,
});
promises.push(deleteOp.execute());
}
return Promise.all(promises);
});
};
/**
* Check if a path has any children. Useful for determining
* if deleting a path will affect more than one document.
*
* #return {Promise<boolean>} a promise that retruns true if the path has
* children and false otherwise.
*/
FirestoreDelete.prototype.checkHasChildren = function() {
return this._getDescendantBatch(true, 1).then(function(docs) {
return docs.length > 0;
});
};
/**
* Run the delete operation.
*/
FirestoreDelete.prototype.execute = function() {
var verifyRecurseSafe;
if (this.isDocumentPath && !this.recursive && !this.shallow) {
verifyRecurseSafe = this.checkHasChildren().then(function(multiple) {
if (multiple) {
return utils.reject("Document has children, must specify -r or --shallow.", { exit: 1 });
}
});
} else {
verifyRecurseSafe = Promise.resolve();
}
var self = this;
return verifyRecurseSafe.then(function() {
return self._deletePath();
});
};
module.exports = FirestoreDelete;
For those who don't want or can't use cloud functions, I found a recursiveDelete function in the admin sdk:
https://googleapis.dev/nodejs/firestore/latest/Firestore.html#recursiveDelete
// Recursively delete a reference and log the references of failures.
const bulkWriter = firestore.bulkWriter();
bulkWriter
.onWriteError((error) => {
if (error.failedAttempts < MAX_RETRY_ATTEMPTS) {
return true;
} else {
console.log('Failed write at document: ', error.documentRef.path);
return false;
}
});
await firestore.recursiveDelete(docRef, bulkWriter);
i don't know how much helpful for you but test it and compare the execution time which i get use it from fire store doc
/** Delete a collection in batches to avoid out-of-memory errors.
* Batch size may be tuned based on document size (atmost 1MB) and application requirements.
*/
void deleteCollection(CollectionReference collection, int batchSize) {
try {
// retrieve a small batch of documents to avoid out-of-memory errors
ApiFuture<QuerySnapshot> future = collection.limit(batchSize).get();
int deleted = 0;
// future.get() blocks on document retrieval
List<QueryDocumentSnapshot> documents = future.get().getDocuments();
for (QueryDocumentSnapshot document : documents) {
document.getReference().delete();
++deleted;
}
if (deleted >= batchSize) {
// retrieve and delete another batch
deleteCollection(collection, batchSize);
}
} catch (Exception e) {
System.err.println("Error deleting collection : " + e.getMessage());
}
}
As mentioned above, you need to write good bit of code for this. For each document that is to be deleted you need to check if it has one or more collections. If it does, then you need to queue those up for deletion too. I wrote the code below to do this. It's not tested to be scalable to large data sets, which is fine for me as I'm using it to clean up after small scale integration tests. If you need something more scalable, feel free to take this as a starting point and play around with batching more.
class FirebaseDeleter {
constructor(database, collections) {
this._database = database;
this._pendingCollections = [];
}
run() {
return new Promise((resolve, reject) => {
this._callback = resolve;
this._database.getCollections().then(collections => {
this._pendingCollections = collections;
this._processNext();
});
});
}
_processNext() {
const collections = this._pendingCollections;
this._pendingCollections = [];
const promises = collections.map(collection => {
return this.deleteCollection(collection, 10000);
});
Promise.all(promises).then(() => {
if (this._pendingCollections.length == 0) {
this._callback();
} else {
process.nextTick(() => {
this._processNext();
});
}
});
}
deleteCollection(collectionRef, batchSize) {
var query = collectionRef;
return new Promise((resolve, reject) => {
this.deleteQueryBatch(query, batchSize, resolve, reject);
});
}
deleteQueryBatch(query, batchSize, resolve, reject) {
query
.get()
.then(snapshot => {
// When there are no documents left, we are done
if (snapshot.size == 0) {
return 0;
}
// Delete documents in a batch
var batch = this._database.batch();
const collectionPromises = [];
snapshot.docs.forEach(doc => {
collectionPromises.push(
doc.ref.getCollections().then(collections => {
collections.forEach(collection => {
this._pendingCollections.push(collection);
});
})
);
batch.delete(doc.ref);
});
// Wait until we know if all the documents have collections before deleting them.
return Promise.all(collectionPromises).then(() => {
return batch.commit().then(() => {
return snapshot.size;
});
});
})
.then(numDeleted => {
if (numDeleted === 0) {
resolve();
return;
}
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(() => {
this.deleteQueryBatch(query, batchSize, resolve, reject);
});
})
.catch(reject);
}
}
Solution using Node.js Admin SDK
export const deleteDocument = async (doc: FirebaseFirestore.DocumentReference) => {
const collections = await doc.listCollections()
await Promise.all(collections.map(collection => deleteCollection(collection)))
await doc.delete()
}
export const deleteCollection = async (collection: FirebaseFirestore.CollectionReference) => {
const query = collection.limit(100)
while (true) {
const snap = await query.get()
if (snap.empty) {
return
}
await Promise.all(snap.docs.map(doc => deleteDocument(doc.ref)))
}
}
There is now a simple way delete a document and all of its subcollections using NodeJS.
This was made available in nodejs-firestore version v4.11.0.
From the docs:
recursiveDelete()
Recursively deletes all documents and subcollections at and under the specified level.
import * as admin from 'firebase-admin'
const ref = admin.firestore().doc('my_document')
admin.firestore().recursiveDelete(ref)
You can write a handler which will recursive delete all nested descendants when triggers onDelete Firestore event.
Example of handler:
const deleteDocumentWithDescendants = async (documentSnap: FirebaseFirestore.QueryDocumentSnapshot) => {
return documentSnap.ref.listCollections().then((subCollections) => {
subCollections.forEach((subCollection) => {
return subCollection.get().then((snap) => {
snap.forEach((doc) => {
doc.ref.delete();
deleteDocumentWithDescendants(doc);
});
});
});
});
};
// On any document delete
export const onDocumentDelete = async (documentSnap: FirebaseFirestore.QueryDocumentSnapshot) => {
await deleteDocumentWithDescendants(documentSnap);
};
Tie it up with firestore event:
exports.onDeleteDocument = functions.firestore.document('{collectionId}/{docId}')
.onDelete(onDocumentDelete);
// You can add all the collection hierarchy to object
private collectionsHierarchy = {
groups: [
[
'groups',
'projects',
'files',
'assets',
'urls',
'members'
]
]
};
async deleteDocument(rootDocument: string) {
// if (!rootDocument.startsWith(`groups/${this.groupId()}`)) {
// rootDocument = `groups/${this.groupId()}/${rootDocument}`;
// }
const batchSize: number = 100;
let root = await this.db
.doc(rootDocument)
.get()
.toPromise();
if (!root.exists) {
return;
}
const segments = rootDocument.split('/');
const documentCollection = segments[segments.length - 2];
const allHierarchies = this.collectionsHierarchy[documentCollection];
for (let i = 0; i < allHierarchies.length; i = i + 1) {
const hierarchy = allHierarchies[i];
const collectionIndex = hierarchy.indexOf(documentCollection) + 1;
const nextCollections: [] = hierarchy.slice(collectionIndex);
const stack = [`${root.ref.path}/${nextCollections.shift()}`];
while (stack.length) {
const path = stack.pop();
const collectionRef = this.db.firestore.collection(path);
const query = collectionRef.orderBy('__name__').limit(batchSize);
let deletedIems = await this.deleteQueryBatch(query, batchSize);
const nextCollection = nextCollections.shift();
deletedIems = deletedIems.map(di => `${di}/${nextCollection}`);
stack.push(...deletedIems);
}
}
await root.ref.delete();
}
private async deleteQueryBatch(
query: firebase.firestore.Query,
batchSize: number
) {
let deletedItems: string[] = [];
let snapshot = await query.get();
if (snapshot.size === 0) {
return deletedItems;
}
const batch = this.db.firestore.batch();
snapshot.docs.forEach(doc => {
deletedItems.push(doc.ref.path);
batch.delete(doc.ref);
});
await batch.commit();
if (snapshot.size === 0) {
return deletedItems;
}
const result = await this.deleteQueryBatch(query, batchSize);
return [...deletedItems, ...result];
}
Another solution using Node.js Admin SDK with Batch.
const traverseDocumentRecursively = async (
docRef: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>,
accumulatedRefs: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>[],
) => {
const collections = await docRef.listCollections();
if (collections.length > 0) {
for (const collection of collections) {
const snapshot = await collection.get();
for (const doc of snapshot.docs) {
accumulatedRefs.push(doc.ref);
await traverseDocumentRecursively(doc.ref, accumulatedRefs);
}
}
}
};
import { chunk } from 'lodash';
const doc = admin.firestore().collection('users').doc('001');
const accumulatedRefs: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>[] = [];
await traverseDocumentRecursively(doc, accumulatedRefs);
await Promise.all(
// Each transaction or batch of writes can write to a maximum of 500 documents
chunk(accumulatedRefs, 500).map((chunkedRefs) => {
const batch = admin.firestore().batch();
for (const ref of chunkedRefs) {
batch.delete(ref);
}
return batch.commit();
}),
);
Not sure if this is helpful for anyone here, but I am frequently facing the error "Fatal error deleting docs <list of docs>" when using firebase-tools.firestore.delete method (firebase-tools version 9.22.0).
I am currently handling these deletion failures using the returned error message in order to avoid rewriting the code cited at Oleg Bondarenko's answer. It uses admin.firestore to effectively delete the failed docs.
It's a poor solution since it relies on the error message, but at least it doesn't force us to copy the whole FirestoreDelete code to modify just a few lines of it:
firebase_tools.firestore
.delete(path, {
project: JSON.parse(process.env.FIREBASE_CONFIG!).projectId,
recursive: true,
yes: true,
token: getToken(),
})
.catch((err: Error) => {
if (err.name == "FirebaseError") {
// If recursive delete fails to delete some of the documents,
// parse the failures from the error message and delete it manually
const failedDeletingDocs = err.message.match(
/.*Fatal error deleting docs ([^\.]+)/
);
if (failedDeletingDocs) {
const docs = failedDeletingDocs[1].split(", ");
const docRefs = docs.map((doc) =>
firestore.doc(doc.slice(doc.search(/\(default\)\/documents/) + 19))
);
firestore
.runTransaction(async (t) => {
docRefs.forEach((doc) => t.delete(doc));
return docs;
})
.then((docs) =>
console.log(
"Succesfully deleted docs after failing: " + docs.join(", ")
)
)
.catch((err) => console.error(err));
}
}
});
If you are looking to delete user data, a solution to consider in 2022 is the Delete User Data Firebase Extension.
Once this is active, you can simply delete the user from Firebase Auth to trigger the recursive deletion of the user documents:
import admin from "firebase-admin";
admin.auth().deleteUser(userId);
You can call firebase.firestore().doc("whatever").set() and that will delete everything in that document.
The only way .set does not erase everything is if you set the merge flag to true.
See Firestore Documentation on Add Data
var cityRef = db.collection('cities').doc('BJ');
var setWithMerge = cityRef.set({
capital: true
}, { merge: true });

Proper way to make callbacks async by wrapping them using `co`?

It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);

Tracking when several callbacks are complete in node.js with Mongoose and FeedParser

I'm writing a batch process to read an RSS feed, and store the contents in MongoDB via Mongoose. I would run the script, and it would process the contents just fine... but the script wouldn't return to the console. My hypothesis was that my database connection was still open, and that's why I it wasn't returning.
I couldn't just track when my parsing was complete, because I still might have some mongoose save operations going on.
So, I wrote a function to track my open database connections, and the status of my RSS parsing. But my code ended up being heinously verbose. I'm wondering if there is a better model/pattern for doing something like this.
var FeedParser = require('feedparser')
, mongoose = require('mongoose');
var TEST_RSS_URL = "./test/data/20120303-seattle.rss";
var OPEN_DB_CONNECTIONS = 0;
var PARSING_DONE = false;
/*
* Keeps track of open database connections, and closes the connection when done
*/
function track_and_close_database(mode) {
switch(mode)
{
case 'open':
OPEN_DB_CONNECTIONS++;
break;
case 'close':
OPEN_DB_CONNECTIONS--;
if (0 == OPEN_DB_CONNECTIONS && PARSING_DONE) conn.close();
break;
case 'parseStart':
PARSING_DONE = false;
break;
case 'parseEnd':
PARSING_DONE = true;
if (0 == OPEN_DB_CONNECTIONS && PARSING_DONE) conn.close();
break;
}
}
function parse_stuff(stuff) {
// do some stuff
setTimeout(console.log("parsed some stuff",20));
}
function main() {
parser = new FeedParser();
parser.on('article', function(article) {
track_and_close_database('open');
// check to see if we already have this listing
stuff_model = conn.model('stuff');
stuff = stuff_model.findOne({'href': article.link}, function (error, doc) {
if (error) {
track_and_close_database('close');
return;
}
// this one doesn't exist yet, parse and save
if (null == doc) {
listing = parse_stuff(article);
// if listing is valid, save it!
if (null != listing) {
listing.save(function (error) { track_and_close_database('close') });
}
// parsing failed
else track_and_close_database('close');
}
// nothing to do, already in the database
else track_and_close_database('close');
});
});
// Completed parsing the RSS file
parser.on('end', function(article) {
track_and_close_database('parseEnd');
});
track_and_close_database('parseStart')
parser.parseFile(TEST_RSS_URL);
}
// run this thing
main();
I too ran into this issue. I would think the preferred way to handle this would be with events, however upon looking into the source code there didn't seem to be anything in there that kept some type of operation count. I ended up hooking it up to an the EventEmitter. What would be better is if Mongoose emitted events for before and after saves so I didn't have to plug this into all of my models.
Here is a sample of how I did it:
/* lib/render_log.js */
/* Model for managing RenderLog */
var common = require("./common");
common.initialize_locals(global);
var mongoose = require("mongoose"),
Schema = mongoose.Schema;
var RenderLogSchema = new Schema({
renderer: String,
template: String,
content: {}
});
RenderLogSchema.pre('save', function(next){
this.db.model('RenderLog').emit('open_db_op', this);
next();
});
RenderLogSchema.post('save', function(){
this.db.model('RenderLog').emit('close_db_op', this);
});
mongoose.connect('mongodb://localhost/time-jr-development');
var RenderLog = mongoose.model("RenderLog", RenderLogSchema);
exports = module.exports = RenderLog;
Followed by my test executable:
/* bin/test_mongoose.js */
var async = require('async');
var RenderLog = require("../lib/render_log");
var ConnectionManager = {
open_db_ops: 0,
new_db_op: function(){
this.open_db_ops ++;
},
close_db_op: function(){
this.open_db_ops --;
},
close: function(self){
if(!self)
self = this;
if(self.open_db_ops > 0){
console.log("Waiting...")
process.nextTick(async.apply(self.close, self));
}else{
RenderLog.db.close();
}
}
};
RenderLog.on("open_db_op", function(model){
ConnectionManager.new_db_op();
});
RenderLog.on("close_db_op", function(model){
ConnectionManager.close_db_op();
})
new RenderLog({renderer: "foo", template: "foo.html", content: {"bar": 1}}).save();
new RenderLog({renderer: "foo", template: "foo.html", content: {"bar": 2}}).save();
new RenderLog({renderer: "foo", template: "foo.html", content: {"bar": 3}}).save();
new RenderLog({renderer: "foo", template: "foo.html", content: {"bar": 4}}).save();
new RenderLog({renderer: "foo", template: "foo.html", content: {"bar": 5}}).save();
// You have to push this to the next tick otherwise it gets called before the save
// events have been emitted
async.nextTick(async.apply(ConnectionManager.close, ConnectionManager));

Resources