I'm working on some code to process a csv in Node.js and store it in an Oracle database. So far, things are going well, but with large amounts of rows in the csv, I get "ORA-01000: maximum open cursors exceeded." I am connecting to Oracle once at the beginning of the script. For each record in the csv, I am preforming multiple SELECTs, INSERTs, and DELETEs and then moving on to the next entry to process all using the same connection. At the end, I close the connection. One thought I had was to get a new connection each time from a pool, but I read other posts saying I should use one connection. Perhaps I need to set a special setting to handle all these queries on one connection?
The script is kind of long, so I will post the important parts...I can post more if needed. Using Q, csvtojson, and oracledb.
...
var conn = null;
function connect() {
var deferred = Q.defer();
oracledb.outFormat = oracledb.OBJECT;
oracledb.getConnection(
{
user : 'foo',
password : 'bar',
connectString : 'foo.bar/bar',
},
function(err, c) {
if(err) deferred.reject(new Error(err));
// set global connection
conn = c;
deferred.resolve();
}
);
return deferred.promise;
}
function closeConnection(conn) {
var deferred = Q.defer();
conn.release(function(err){
if(err) deferred.reject(err);
else return deferred.resolve();
});
return deferred.promise;
}
/* Process All Data, Promise Loop */
function process(data) {
return processEntry(data.shift()).then(function(){
console.log('Finished processing entry.');
return data.length > 0 ? process(data) : true;
});
}
/* Process an Entry */
function processEntry(entry) {
var deferred = Q.defer();
var data = {};
entryExists(entry)
.then(function(result) {
if(result) return entryLogicError(null, 'Entry exists. Skipping.');
else return getUserFromReleaseCode(entry);
})
.then(function(result) {
if(typeof result != 'undefined' && result.length > 0) {
data.user = result[0];
return getPanelCode(entry);
}
else return entryLogicError(entry, 'No valid release code.');
})
.then(function(result){
if(typeof result != 'undefined' && result.length > 0) {
return createHeader(result[0].foo, result[0].bar);
}
else return entryLogicError(entry, 'No valid panel code.');
})
... More of the same kind of statements processing the entry ...
.then(function() {
return logEntry(entry);
})
.catch(function(error) { console.log("DATA ERROR: " + error) })
.done(function(){
deferred.resolve();
});
return deferred.promise;
}
function entryLogicError() {
// logs entry to be corrected, return a rejected promise to go to the next entry
}
/* Check if record has been processed */
function entryExists(entry) {
var deferred = Q.defer();
var foo = entry[ENTRY_CONST.FOO];
var bar = entry[ENTRY_CONST.BAR];
conn.execute(
'SELECT * FROM TBL_FOO ' +
'WHERE FOO = :foo AND ' +
'BAR = :bar',
[foo, bar],
function(err, result) {
if(err) deferred.reject(err);
else {
deferred.resolve(result.rows.length > 0);
}
});
return deferred.promise;
}
/* Get User from Release Code */
function getUserFromReleaseCode(entry) {
var deferred = Q.defer();
var foo = entry[ENTRY_CONST.FOO];
conn.execute(
'SELECT * FROM TBL_BAR ' +
'WHERE FOO = :foo',
[foo],
function(err, result) {
if(err) deferred.reject(err);
else {
deferred.resolve(result.rows);
}
});
return deferred.promise;
}
/* Create Header */
function createHeader(foo, bar) {
var deferred = Q.defer();
conn.execute(
'BEGIN INSERT INTO TBL_DR_FOO VALUES (NULL,:foo, :bar,' +
'1,NULL,1,NULL,NULL,NULL,NULL) RETURNING DR_FOO_ID INTO :DR_FOO_ID; COMMIT; END;',
{ foo: foo,
bar: bar,
DR_FOO_ID: { dir: oracledb.BIND_OUT, type: oracledb.NUMBER }
},
function(err, result) {
if(err) deferred.reject(err);
else deferred.resolve(result.outBinds);
});
return deferred.promise;
}
function cleanHistory() {
// statement that deletes records from a certain date using conn.execute(..)
}
/* Main */
connect().then(function(){
var converter = new Converter({ noheader: false });
converter.on('end_parsed', function(data) {
process(data).then(function(){
return cleanHistory();
})
.then(function(){
return closeConnection();
}).done();
});
fs.createReadStream(batch).pipe(converter);
}, function(err){
return console.error(err);
});
Related
Even though I have searched for the solution of this error and i found some answers but none of them helped me fix this error, Error: NJS-012: encountered invalid bind data type in parameter 2.Maybe, one error can occur in a different scenarios.
Stored procedure definition
create or replace PROCEDURE SP_MEAL_GETMEALTYPES
(
p_DataSource OUT Sys_RefCursor
)
AS
BEGIN
OPEN p_DataSource FOR
select mealtypeid,description from mealtypes;
END;
File name: menusStoredProc.js
"use strict";
var dbParams = require('../../oracle/dbParams');
function storedProcs() {
this.SP_USER_GETMENUS = {
name: 'sp_meal_getmealtypes',
params: {
dataSource: {val: null, type: dbParams.CURSOR, dir: dbParams.BIND_OUT}
},
resultSetColumns: ['mealTypeId','description']
}
}
module.exports = storedProcs;
File name: menus.js
var express = require('express');
var MenusStoreProc = require('../storedProcedures/menusStoredProc');
var oraDbAssist = require('../../oracle/oracleDbAssist');
var router = express.Router();
router.get('/getmenus', (req, res, next) => {
var sp = new MenusStoreProc().SP_USER_GETMENUS;
oraDbAssist.getConnection(function (err, conn) {
if (err)
return console.log('Connecting to db failed - ' + err);
oraDbAssist.executeSqlWithConn(sp, false, conn, function (err, menus) {
if (err)
return console.log('Executing ' + sp.name + ' failed - ' + err);
res.status(200).json(JSON.stringify(menus));
});
});
});
module.exports = router;
Function definition added - executeSqlWithConn
function executeSqlWithConn(sp, autoCommit, connection, next) {
var sql = createProcedureSqlString(sp.name, sp.params);
var params = buildParams(sp.params);
connection.execute(sql, params, {autoCommit: autoCommit}, function(err, result) {
if (err) {
next(err, null);
return;
}
var allRows = [];
var numRows = 50; // number of rows to return from each call to getRows()
for(var attributeName in result.outBinds) {
if(result.outBinds[attributeName] && result.outBinds[attributeName].metaData) { // db response is a result set
function fetchRowsFromResultSet(pResultSet, pNumRows) {
pResultSet.getRows(pNumRows, function(readErr, rows) {
if(err) {
pResultSet.close(function (err) { // always close the result set
next(readErr);
});
return;
}
allRows.push(rows);
if (rows.length === pNumRows) {
fetchRowsFromResultSet(result.outBinds[attributeName], numRows);
return;
}
var allRowsResult = Array.prototype.concat.apply([], allRows);
generateJsonFromDbResultSet(pResultSet.metaData, allRowsResult, sp, function(resultSet) {
pResultSet.close(function (err) { // always close the result set
next(null, resultSet);
});
});
});
}
fetchRowsFromResultSet(result.outBinds[attributeName], numRows);
return;
}
}
next(null, result.outBinds);
});
}
Function definition added - buildParams
function buildParams(params) {
for(var attributeName in params) {
params[attributeName].val = typeof params[attributeName].val === 'undefined' ? null : params[attributeName].val;
if(params[attributeName].type.is(dbParams.DATE))
params[attributeName].val = params[attributeName].val ? new Date(params[attributeName].val) : null;
params[attributeName].type = params[attributeName].type.value;
params[attributeName].dir = params[attributeName].dir.value;
}
return params;
}
Any help, dear members ?
Got this error when I run my program
TypeError: Callback is not a function
// Update data from a new file
lib.update = function(dir,file,callback){
//Open the file for writing
fs.open(lib.baseDir+dir+'/'+'.json','r+',function(err,fileDescriptor){
if(!err && fileDescriptor){
var stringData= JSON.stringify(data);
//Truncate the file before writing
fs.truncate(fileDescriptor,function(err){
if(!err){
//Write to the file and close it
fs.writeFile(fileDescriptor,stringData,function(err){
if(!err){
fs.close(fileDescriptor,function(err){
if(!err){
callback(false);
}else {
callback('Error closing existing file!')
}
})
}else {
callback('Error writing to existing file')
}
});
}else {
callback('Error Truncating file')
}
});
}else {
callback('Could not open file for updating! May not exist yet')
}
});
}
I propose you to refactor your code to be a little more clear about what you are doing. Something like this, it will help you find where errors come. Using promises better than use a lot of callbacks.
Just my contribution.
lib.update = function(dir, file, data) {
const updateP = new CustomPromise();
openFile()
.then(truncateFile)
.then(writeFile.bind(this, JSON.stringify(data)))
.then(closeFile)
.then(updateP.resolve) // at this point all the functions was successful
.catch((errorType) =>
errorType !== ERROR_TYPE.ERROR_CLOSING_FILE // something fail try to close the file
? closeFile().finally(() => updateP.reject(errorType))
: updateP.reject(errorType),
);
return updateP.promise;
};
// Constants
const ERROR_TYPE = Object.freeze({
ERROR_OPEN_FILE: 'error-open-file',
ERROR_TRUNCATING_FILE: 'error-truncating-file',
ERROR_WRITING_FILE: 'error-writing-file',
ERROR_CLOSING_FILE: 'error-closing-file',
});
// Private functions
function CustomPromise() {
this.promise = new Promise(function(resolve, reject) {
this.resolve = resolve;
this.reject = reject;
});
}
function openFile() {
const openP = new CustomPromise();
fs.open(lib.baseDir + dir + '/' + '.json', 'r+', function(err, fileDescriptor) {
(err || !fileDescriptor) && openP.reject(ERROR_TYPE.ERROR_OPEN_FILE);
openP.resolve(fileDescriptor);
});
return openP.promise;
}
function truncateFile(fileDescriptor) {
const truncateP = new CustomPromise();
fs.truncate(fileDescriptor, function(err) {
err && truncateP.reject(ERROR_TYPE.ERROR_TRUNCATING_FILE);
truncateP.resolve(fileDescriptor);
});
return truncateP.promise;
}
function writeFile(data, fileDescriptor) {
const writeFileP = new CustomPromise();
fs.writeFile(fileDescriptor, data, function(err) {
err && writeFileP.reject(ERROR_TYPE.ERROR_WRITING_FILE);
writeFileP.resolve(fileDescriptor);
});
return writeFileP.promise;
}
function closeFile(fileDescriptor) {
const closeP = new CustomPromise();
fs.close(fileDescriptor, function(err) {
err && closeP.reject(ERROR_TYPE.ERROR_CLOSING_FILE);
closeP.resolve();
});
return closeP.promise;
}
I am trying to retrieve attendance list along with user details.
I am using caminte.js(http://www.camintejs.com/) Cross-db ORM for database interaction.
Here is my code sample of model function "attendanceList".
exports.attendanceList = function (req, callback) {
var query = req.query;
var searchfilters = {};
if(!req.user){
callback({ code:400, status:'error', message: 'Invalid Request', data:{}});
}else{
searchfilters["vendor_id"] = parseInt(req.user._id);
}
if(query.location && parseString(query.location) != '') {
searchfilters["location"] = parseString(query.location);
}
if (query.device_details && parseString(query.device_details) != '') {
searchfilters["device_details"] = parseString(query.device_details);
}
if(query.created_on) {
searchfilters["created_on"] = query.created_on;
}
if(query.status) {
searchfilters["status"] = { regex: new RegExp(query.status.toLowerCase(), "i") };
}
var SkipRecord = 0;
var PageSize = 10;
var LimitRecord = PageSize;
var PageIndex = 1;
if(query.pagesize) {
PageSize = parseInt(query.pagesize);
}
if(query.pageindex) {
PageIndex = parseInt(query.pageindex);
}
if (PageIndex > 1) {
SkipRecord = (PageIndex - 1) * PageSize;
}
LimitRecord = PageSize;
var SortRecord = "created_on";
if(query.sortby && query.sorttype) {
var sortingBy = query.sortby;
var sortingType = 'ASC';
if(typeof query.sorttype !== 'undefined') {
sortingType = query.sorttype;
}
SortRecord = sortingBy + ' ' + sortingType;
}
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
};
I am getting only attendance list without user details. How do I force to push user details into attendance list? Any Help!!
Thank You
This behavior is asynchronous. When you're making request to DB, your code keeps running, while task to get data comes to task queue.
To keep things simple, you need to use promises while handling asynchronous jobs.
Rewrite your code from this:
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
To this:
const findAttendanceFirst = (searchFilters, SortRecord, LimitRecord, SkipRecord) => {
return new Promise((resolve, reject) => {
Attendance.find({ where: searchFilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, (err, result) => {
if(err) return reject(err);
resolve(result);
});
});
}
const findUserByIdForUserDetails = (userId) => {
return new Promise((resolve, reject) => {
User.findById(parseInt(userId), function(err, data){
if(err) return reject(err);
resolve(data);
})
});
}
const getAttendanceCount = (searchFilters) => {
return new Promise((resolve, reject) => {
Attendance.count({ where: searchFilters }, (err, count) => {
if(err) return reject(err);
resolve(count);
});
})
}
So, now we can use this separate functions to make async behavior looks like sync.
try {
const data = await findAttendanceFirst(searchFilters, SortRecord, LimitRecord, SkipRecord);
for(let userData of data){
try {
userData.userDetails = await findUserByIdForUserDetails(userData.user_id);
} catch(e) {
// Some error happened, so no user details.
// you can set here null or nothing to userDetails.
}
}
let count;
try {
count = await getAttendanceCount(searchFilters);
} catch(e){
// Same as before.
}
const callBackData = { code:200, status:'success', message:'OK', total:count, data:result };
// And here you can do whatever you want with callback data. Send to client etc.
} catch(e) {
}
NB: I've not tested this code, it will be easier for yu to play with your actual data and use Promises and async/await
Just remember that each request to db is asynchronous, and you need to make your code wait for this data.
So i have this 2-layer query in node.js, each query could return multiple results. My code actually just ignores that for now. This is the best i can get, it seems working.
How to correct it please, i don't know how to callback for the 2nd one.
Also the db.close() is always called before the 2nd query finishes, even i have serialize().
var getInfo1Db = function(callback) {
var db = new sqlite3.Database("DB.sqlite3");
var cnt = 0;
var info1JsonObj = [];
db.all("select * from Info1DB",
function(err, rows) {
db.serialize(function() {
for(var ii=0, len=rows.length; ii<len; ii++) {
var t2 = rows[ii].info1;
var doorId = ...
db.all("select * from DoorDB where ObjectID=" + doorId,
function(err, row2) {
if(err) {
} else {
var doorName = row2[0]...
var info1JsonElem = {
"DoorName" : doorName
};
info1JsonObj.push(info1JsonElem);
cnt++;
if(cnt === rows.length) {
callback(null, info1JsonObj);
}
}
}
); // for the only door info based on door id
} // for each row of info1
db.close(); // why this finishes before the 2nd db.all
} ); // end of serialize
});
};
You can't implement nested query in sqlite3's normal way. ( I mean you even can't do it in the callback hell way, because the sqlite3 need to close the connection before another query called. otherwise you will always got error)
You have to use Promise, async and await to do this.
( it's worth to spend 30 minutes to learn these 3 words )
Step1. define a async function like this:
async query_1() {
new Promise(resolve => {
db = ...
db.serialize( () => {
db.get('select .. from ... where id = 1', [], (error, row) => {
// here is the KEY: put the result into resolve
// this equals to the "return" statement in non-sync method.
resolve(row)
}
})
db.close()
})
}
and also implement your query_2 function like this:
async query_2() {
let query_1_result = await this.query_1()
db = ...
db.serialize( () => {
db.get('select .. from ... where dependency_id = ' + query_1_result, [], (error, row) => {
// other code here...
}
})
db.close()
}
refer to my answer: https://stackoverflow.com/a/67881159/445908
How about using 2 function to do these ?
function db_query1(your_param,...., callback){
// database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
function db_query2(your_param,...., callback){
// database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
And call these function:
db_query1(....,function(err,result1){
if(err) ...// return
// do the things with result1
// And then call query2
db_query2(....,function(err,result2){
if(err) ...// return
// do the things with result1
});
});
Hope this will help :)
You can use Promises.all, an array and the second callback for node sqlite3 db.each() that is executed when all rows have been fetched. Node Sqlite3 db.each usage to simplify the nested query and
I cannot really get the meaning of the variables you are using thus I assume that each row in Info1DB has a one-to-many relationship with DoorDB on the field doorId.
async function getInfo (callback) {
sql = "select * from Info1DB;";
numVersions = 0;
countVersions = 0;
info1JsonObj = [];
db.serialize(function() {
db.each(sql, [], (err, info1Row) => {
sql = "select * from DoorDB where ObjectID=?;";
info1Row.doors = [];
doorId = ...
db.each(sql, [doorId], (err, doorRow) => {
info1Row.doors.push(new Promise((resolve, reject) => {
if (err) {
reject(err);
} else {
resolve(doorRow);
}
}));
}, (err, num) => {
Promise.all(info1Row.doors)
.then((doors) => {
info1Row.doors = doors;
info1JsonObj.push(info1Row);
countVersions++;
if (countVersions == numVersions) {
callback(null, info1JsonObj);
}
}).catch((err) => {
callback(err, null);
});
});
}, (err, versions) => {
numVersions = versions;
});
});
}
I am new to javascript and node.js and this is my first post, so please bear with me.
I am using ntwitter to get all previous tweets of a specific user.
My problem is that if the user has more than 200 tweets, I need to create a loop and I am not sure if I do it right.
This is the async function that gets the 200 latest tweets:
exports.getUserTimeline = function(user, callback) {
twit.getUserTimeline({ screen_name: user, count: 200 }, function(err, data) {
if (err) {
return callback(err);
}
callback(err, data);
});
}
I found a solution to do this using a recursive function, but it's quite ugly.. How can I improve it ?
exports.getUserHistory = function(user, callback) {
recursiveSearch(user, callback);
function recursiveSearch(user, callback, lastId, data) {
var data = data || []
, args = {screen_name: user, count: 200};
if(typeof lastId != "undefined") args.max_id = lastId;
twit.getUserTimeline(args, function(err, subdata) {
if (err) {
console.log('Twitter search failed!');
return callback(err);
}
if (data.length !== 0) subdata.shift();
data = data.concat(subdata);
var lastId = parseInt(data[data.length-1].id_str);
if (subdata.length !== 0) {
recursiveSearch(user, callback, lastId, data);
} else {
callback(err, data);
}
});
}
}
Thank's a lot!
Update: This is the improved (refactored) function suggested by hunterloftis with two modifications:
property max_id should not be specified on the first iteration
the case where the user exists but no tweets have been posted must be handled
code:
function getUserHistory(user, done) {
var data = [];
search();
function search(lastId) {
var args = {
screen_name: user,
count: 200,
include_rts: 1
};
if(lastId) args.max_id = lastId;
twit.getUserTimeline(args, onTimeline);
function onTimeline(err, chunk) {
if (err) {
console.log('Twitter search failed!');
return done(err);
}
if (!chunk.length) {
console.log('User has not tweeted yet');
return done(err);
}
//Get rid of the first element of each iteration (not the first time)
if (data.length) chunk.shift();
data = data.concat(chunk);
var thisId = parseInt(data[data.length - 1].id_str);
if (chunk.length) return search(thisId);
console.log(data.length + ' tweets imported');
return done(undefined, data);
}
}
}
When retrieving tweets I noticed that my tweet count wasn't always the same as the 'statuses_count' property of the user. It took me some time to figure out that this difference corresponds to the number of deleted tweets :)
Does your recursive function work? Doesn't look too bad to me. I might refactor it just a little into something more like this:
function getUserHistory(user, done) {
var data = [];
search();
function search(lastId) {
var args = {
screen_name: user,
count: 200,
max_id: lastId
};
twit.getUserTimeline(args, onTimeline);
function onTimeline(err, chunk) {
if (err) {
console.log('Twitter search failed!');
return done(err);
}
if (data.length) chunk.shift(); // What is this for?
data = data.concat(chunk);
var thisId = parseInt(data[data.length - 1].id_str);
if (chunk.length) return search(thisId);
return done(undefined, data);
}
}
}