CSV to Mongo using mongoose schema - node.js

Question

I'm attempting to get a CSV file to my mongodb collection (via mongoose) while checking for matches at each level of my schema.
So for a given schema personSchema with a nest schema carSchema:
repairSchema = {
date: Date,
description: String
}
carSchema = {
make: String,
model: String
}
personSchema = {
first_name: String,
last_name: String,
car: [carSchema]
}
and an object that I am mapping the CSV data to:
mappingObject = {
first_name : 0,
last_name: 1,
car : {
make: 2,
model: 3,
repair: {
date: 4,
description: 5
}
}
}
check my collection for a match then check each nested schema for a match or create the entire document, as appropriate.
Desired process:
I need to check if a person document matching first_name and last_name exists in my collection.
If such a person document exists, check if that person document contains a matching car.make and car.model.
If such a car document exists, check if that car document contains a matching car.repair.date and car.repair.description.
If such a repair document exists, do nothing, exact match to existing record.
If such a repair document does not exist, push this repair to the repair document for the appropriate car and person.
If such a car document does does not exist, push this car to the car document for the appropriate person.
If such a person document does not exist, create the document.
The kicker
This same function will be used across many schemas, which may be nested many levels deep (current database has one schema that goes 7 levels deep). So it has to be fairly abstract. I can already get the data into the structure I need as a javascript object, so I just need to get from that object to the collection as described.
It also has to be synchronous, since multiple records from the CSV could have the same person, and asynchronous creation could mean that the same person gets created twice.
Current solution
I run through each line of the CSV, map the data to my mappingObject, then step through each level of the object in javascript, checking non-object key-value pairs for a match using find, then pushing/creating or recursing as appropriate. This absolutely works, but it is painfully slow with such large documents.
Here's my full recursing function, which works:
saveObj is the object that I've mapped the CSV on to that matches my schema.
findPrevObj is initially false. path and topKey both are initially "".
lr is the line reader object, lr.resume simply moves on to the next line.
var findOrSave = function(saveObj, findPrevObj, path, topKey){
//the object used to search the collection
var findObj = {};
//if this is a nested schema, we need the previous schema search to match as well
if (findPrevObj){
for (var key in findPrevObj){
findObj[key] = findPrevObj[key];
}
}
//go through all the saveObj, compiling the findObj from string fields
for (var key in saveObj){
if (saveObj.hasOwnProperty(key) && typeof saveObj[key] === "string"){
findObj[path+key] = saveObj[key]
}
}
//search the DB for this record
ThisCollection.find(findObj).exec(function(e, doc){
//this level at least exists
if (doc.length){
//go through all the deeper levels in our saveObj
for (var key in saveObj){
var i = 0;
if (saveObj.hasOwnProperty(key) && typeof saveObj[key] === "string"){
i += 1;
findOrSave(saveObj[key], findObj, path+key+".", path+key);
}
//if there were no deeper levels (basically, full record exists)
if (!i){
lr.resume();
}
}
//this level doesn't exist, add new record or push to array
} else {
if (findPrevObj){
var toPush = {};
toPush[topKey] = saveObj;
ThisCollection.findOneAndUpdate(
findPrevObj,
{$push: toPush},
{safe: true, upsert: true},
function(err, doc) {
lr.resume();
}
)
} else {
// console.log("\r\rTrying to save: \r", saveObj, "\r\r\r");
ThisCollection.create(saveObj, function(e, doc){
lr.resume();
});
}
}
});
}

Answer 1

I'll update for clarity, but the person.find is to check if a person with a matching first and last name exists. If they do exist, I check each car for a match - if the car exists already, there's no reason to add this record. If the car doesn't exist, I push it to the car array for the matching person. If no person was matched, I'd save the entire new record.
Ah, what you want is to update with upsert:
replace
Person.find({first_name: "adam", last_name: "snider"}).exec(function(e, d){
//matched? check {first_name: "adam", last_name: "snider", car.make: "honda", car.model: "civic"}
//no match? create this record (or push to array if this is a nested array)
});
with
Person.update(
{first_name: "adam", last_name: "snider"},
{$push: {car: {make: 'whatever', model: 'whatever2'}}},
{upsert: true}
)
If a match is found, it will push into OR create the car field this subdoucment: {car_make: 'whatever', car_model: 'whatever2'}.
If a match is not found, it will create a new doc that looks like:
{first_name: "adam", last_name: "snider", car: {car_make: 'whatever', car_model: 'whatever2'}}
This cuts your total db round trips in half. However, for even more efficiency, you can use an orderedBulkOperation. This would result in a single round trip to the database.
Here's what that would look like (using es6 here for concision...not a necessity):
const bulk = Person.collection.initializeOrderedBulkOp();
lr.on('line', function(line) {
const [first_name, last_name, make, model, repair_date, repair_description] = line.split(',');
// Ensure user exists
bulk.update({first_name, last_name}, {first_name, last_name}, {upsert: true});
// Find a user with the existing make and model. This makes sure that if the car IS there, it matches the proper document structure
bulk.update({first_name, last_name, 'car.make': make, 'car.model': model}, {$set: {'car.$.repair.date': repair_date, 'car.$.repair.description': repair_description}});
// Now, if the car wasn't there, let's add it to the set. This will not push if we just updated because it should match exactly now.
bulk.update({first_name, last_name}, {$addToSet: {car: {make, model, repair: {date: repair_date, description: repair_description}}}})
});