Node js writing bulk xml data to SQL Server - node.js

Need some assistance in node.js. Trying to update about 6000+ records into SQL Server. The records size maybe 5 to 10 MB. The xml records will be sent to the stored procedure which has a nvarchar(max) parameter. I am fetching records from a soap API call and push to SQL Server via a stored procedure.
I get an error when I run the below code if the record size is beyond 2000 in xml object. What is best way to send large xml objects of 5 to 10k to a SQL Server stored procedure?
Any nodejs library to handle this? Can nodejs handle such large records update ?
Note: this code will run inside aws lambda
var soap = require("strong-soap").soap;
var rfcUrl = https://xxxxxxxx/xxxx/xxxxx/xxxxx.asmx?WSDL;
var rfcRequestArgs = {
UserName:
Password:
};
var options = {};
console.log("Calling ervice...\n");
soap.createClient(rfcUrl, options, function(err, client) {
var method = client["GetLocationData"];
method(rfcRequestArgs, function(err, result, envelope, soapHeader) {
if(err) {
console.log('error: ' + err);
return;
}
else
{
//'result' is the response body
console.log('RFC Result: \n');
InitializeSQLConnection(JSON.stringify(result));
}
});
});
var connection = null;
function InitializeSQLConnection(LocData)
{
var Connection = require('tedious').Connection;
var config = {
server: 'ddddddddd', //update me
authentication: {
type: 'default',
options: {
userName: 'xxxxxxx', //update me
password: 'xxxxxxx' //update me
}
} ,
options: {
// If you are on Microsoft Azure, you need encryption:
encrypt: false,
database: 'DBName' //update me
}
};
connection = new Connection(config);
connection.on('connect', function(err) {
if (err) {
console.log('SQL Connection Failed!');
console.log(err);
throw err;
}
// If no error, then good to proceed.
console.log("SQL Server Connected");
executeStoredProcedure(LocData);
});
connection.connect();
}
function executeStoredProcedure(LocData)
{
console.log("calling executeStoredProcedure");
var Request = require('tedious').Request;
var TYPES = require('tedious').TYPES;
var loadTestState = 0;
var request = new Request('spname_new', function(err) {
connection.close();
});
request.addParameter('parameterrecords', TYPES.NVarChar, LocData);
request.addOutputParameter('returnVal', TYPES.Int);
request.on('returnValue', (paramName, value, metadata) => {
console.log(paramName + ' : ' + value);
});
connection.callProcedure(request);
console.log("Called SP");
}
'''

Related

Accessing Azure Synapse Analytics from Node.js Function

I am trying to access an Azure Synapse SQL Pool out of a Node.js Function. I took this article as a foundation and switch the authentication method to 'azure-active-directory-msi-app-service'.
If I run the function I can see the queryText context.log in the Function cmd but unfortunately nothing else happens. It runs without any kind of output.
I cant see any other context.log or get any response from the function.
const Request = require('tedious').Request;
const TYPES = require('tedious').TYPES;
const test = require('lodash')
module.exports = function (context, req) {
//get the Query Parameter
// Create connection to database
const config = {
authentication: {
type: "azure-active-directory-msi-app-service"
},
server: "database.sql.azuresynapse.net",
options: {
database: "database",
encrypt: true,
port: 1433
}
};
const connection = new Connection(config);
// Create array to store the query results
let result = [];
let rowData = {};
// req.query.color will be passed as a query variable in the URL
//const payload = [req.query.color];
// Create query to execute against the database
const queryText = "SELECT TOP (10) [id],[column1],[column2] FROM [dbo].[Table]";
context.log(queryText);
// Create Request object
request = new Request(queryText, function(err) {
context.log("Request")
if (err) {
// Error in executing query
context.log.error(err);
context.res.status = 500;
context.res.body = "Error executing the query";
} else {
context.res = {
status: 200,
isRaw: true,
body: result,
headers: {
'Content-Type': 'application/json'
}
}
}
context.done();
});
// Manipulate the results and create JSON
request.on('row', function(columns) {
rowData = {};
columns.forEach(function(column) {
// IMPORTANT: Change the conversion logic here to adjust JSON format
rowData[column.metadata.colName] = column.value;
});
result.push(rowData);
});
connection.on('connect', function(err) {
if (err) {
// Error in connecting
context.log.error(err);
context.res.status = 500;
context.res.body = "Error connecting to Azure Synapase";
context.done();
} else {
// Connection succeeded
connection.execSql(request);
}
});
}```
This should work...
const Connection = require('tedious').Connection;
const Request = require('tedious').Request;
const TYPES = require('tedious').TYPES;
const config = {
server: "SQL pool endpoint goes here",
authentication: {
type: "azure-active-directory-msi-app-service"
},
options: {
encrypt: true,
database: "pool name goes here",
port: 1433
}
}
var connection = new Connection(config);
This assumes that you appropriately assigned managed identities as external users in the SQL pool. I also have a PR for this change which you can track here

SQL select all statement in a node.js application returns a tedious deprecated error

So I went to the Microsoft documentation for node.js and trying to connect to a database and I went through step by step, installed tedious and when I try to run my code it's throwing an error saying:
tedious deprecated In the next major version of tedious, creating a new Connection instance will no longer establish a connection to the server automatically. Please use the new connect helper function or call the .connect method on the newly created Connection object to silence this message. internal\process\task_queues.js:79:11.
Does anyone know what this means?
CODE:
const Discord = require('discord.js');
const bot = new Discord.Client();
const token = 'HIDDEN';
bot.on('ready', () => {
console.log('This bot is online!');
var Connection = require('tedious').Connection;
var config = {
server: '', //update me
authentication: {
type: 'default',
options: {
userName: '', //update me
password: '' //update me
}
},
options: {
// If you are on Microsoft Azure, you need encryption:
encrypt: true,
database: '' //update me
}
};
var connection = new Connection(config);
connection.on('connect', function(err) {
// If no error, then good to proceed.
if(!err)
{
console.log("Connected");
executeStatement();
}
});
var Request = require('tedious').Request;
var TYPES = require('tedious').TYPES;
function executeStatement() {
request = new Request("SELECT * from tblCustomer;", function(err) {
if (err) {
console.log(err);}
});
var result = "";
request.on('row', function(columns) {
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
result+= column.value + " ";
}
});
console.log(result);
result ="";
});
request.on('done', function(rowCount, more) {
console.log(rowCount + ' rows returned');
});
connection.execSql(request);
}
})
bot.login(token);

How to connect with SQL Server in windows authentication mode using node.js

I want to connect SQL Server in a windows authentication mode using node.js with "mssql" package. Below code is working fine with SQL Server authentication by providing username & password.
var sql = require("mssql");
// config for your database
var config = {
server: XXXX',
database: 'XXX' ,
options: {
trustedconnection:true,
encrypt: false, // Use this if you're on Windows Azure
}
};
function GetDatatable()
{
return new Promise( (resolve, reject) => {
sql.connect(config, function (err) {
if (err) console.log(err);
// create Request object
var request = new sql.Request();
// query to the database and get the records
var query='select Id,statusCode,from test';
request.query(query, function (err, recordset) {
var recordSetItem = recordset.recordset;
for(var i=0;i<recordSetItem.length;i++) {
var row=recordSetItem[i];
var basePath=row['baseUri'];
if(fullPath.lastIndexOf(basePath) > -1)
{
operationId=row['Id'];
}
}
if(operationId)
{
query='select * from tbltest';
request.query(query, function (err, scenarioRecordset) {
var subRecordSet = scenarioRecordset.recordset;
var xmlResponse= validateScenario (subRecordSet,fullPath,bodyRequest);
console.log(xmlResponse);
// let responseObject = {
// "id": "1",
// "statusCode": "200"
// };
resolve('responseObject');
});
}
});
});
});
Getting error message "login failed"
I believe you'll need to use the msnodesqlv8 module, you must install with
npm install msnodesqlv8
Then (for example):
const sql = require('mssql/msnodesqlv8')
const pool = new sql.ConnectionPool({
database: 'database',
server: 'localhost',
options: {
trustedConnection: true
}
})
async function testConnection() {
await pool.connect();
let result = await pool.request().query("select 42 as the_answer_to_life_the_universe_and_everything");
console.log("Result: ", result);
}
testConnection();
See here for more: mssql v8driver

Azure Function connecting SQL server with blank page in browser

I am new to Azure Function and nodejs.
I have a very simple azure function which connects to SQL server by using Azure Function with nodejs. I use the package named "tedious" to connection to SQL server which hosted in Azure. The connection works fine as I can see the result in my Terminal panel in Visual Studio Code by using "context.log" when put the URL "http://localhost:7071/api/Company" into my browser. However, i see nothing in my browser.
I suspect that the "return" called before the function "queryDatabase" is completed but i have no clue how to do this. Any advice?
var Connection = require('tedious').Connection;
var Request = require('tedious').Request;
var rows = [];
var config = {
userName: 'xxx',
password: 'xxx',
server: 'xxx',
options: {encrypt: true, database: 'xxx'}
};
var res = {};
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
var str = "";
var connection = new Connection(config);
var querystatus = "";
connection.on('connect', function(err) {
if(err) {
context.log(err);
//context.res = {body : err};
} else {
context.log("************Connected*****************");
//context.res = {body : "Hello!"};
queryDatabase(connection);
// context.res = {
// body: "Connected"
// };
//context.done();
}
});
context.log("************BEFORE context.res*****************");
return {
body:rows.toString()
};
function queryDatabase(connection) {
context.log("queryDatabase....started!");
var request = new Request(
"SELECT [id],[CompanyName] ,[CreatedDate] FROM [dbo].[Company]",
function(err, rowCount, rows)
{
context.log(rowCount + ' row(s) returned');
//context.log("Final Result:" + str);
}
);
request.on('row', function(columns) {
var row = {};
columns.forEach(function(column) {
context.log("%s\t%s", column.metadata.colName, column.value);
row[column.metadata.colName] = column.value;
});
rows.push(row);
});
connection.execSql(request);
}
};
Finally, I remove the "async " and add a context.done after the result is returned from sql server.

before fetching data in db response is sent by node js

this is my set notification object
var sentNotificationObj = function (notification, eventid, addedorganizerpropic) {
this.notification = notification;
this.eventid = eventid;
this.addedorganizerpropic = addedorganizerpropic;
}
this is my array which is stored notification obect
var notificationSetArray2 = [];
this is my api of getnotification
router.post('/getnotification', function (req, res) {
console.log('in aside');
var id = req.body.userid;
console.log('pos id' + id);
User.findById({ _id: id }, function (err, result) {
if (err) {
console.log('err get notification');
res.statusCode = 500;
res.json({
success: false,
message: "severe error"
});
} else {
this is code fetchin data in data base
for (var i = 0; i < result.notification.length; i++) {
var addedevent = result.notification[i].addedevent;
var notification = result.notification[i].notification;
console.log('added event' + addedevent);
console.log('added noti' + notification);
User.findById({ _id: result.notification[i].addedorganizer }, function (err, addedorganizer) {
if (err) {
console.log('error get added user pofile pic');
} else {
convert image to base64
var base64str = base64_encode(addedorganizer.profileData.profileurl);
console.log(base64str);
console.log(notification);
console.log(addedevent);
var newsentNotificationObj = new sentNotificationObj(notification, addedevent, base64str);
notificationSetArray2.push(newsentNotificationObj);
console.log('succe get added user profile pic');
}
});
}
this is response
res.statusCode = 200;
res.json({
success: true,
notificationArray: notificationSetArray2
})
console.log(notificationSetArray2);
notificationSetArray2.length = 0;
}
});
});
The most simple solution in here to use async library here.
Node runs code in asynchronous way. So it is natural to send response before fetching any data from your database.
By using async you can execute this in a sequence. So it will be solved.
Use async.series method for solve this. For example
async.series(
[
function(callback){
// fetch your data here using mongo with your loop
//
//
callback(); // this will trigger next step (call this after you finished iterating array)
},
function(callback){
// after above code has been executed
// send response here
callback() // call this to proceed
}
],
function(err){
// handle any error in here
}
)
A good example of how to use asyncjs in node

Resources