Moving data from nodejs console to SQL Server - node.js

I am very new to node.js. Using the following code I am able to retrive data from the wesbite intrino into console.
var https = require("https");
var username = "********";
var password = "********";
var auth = "Basic " + new Buffer(username + ':' +
password).toString('base64');
var request = https.request({
method: "GET",
host: "api.intrinio.com",
path: "/companies?ticker=AAPL",
headers: {
"Authorization": auth
}
}, function (response) {
var json = "";
response.on('data', function (chunk) {
json += chunk;
});
response.on('end', function () {
var company = JSON.parse(json);
console.log(company);
});
});
request.end();
And the result is as follows:-
My question is: how do I transfer this data into SQL Server? I tried watching a few tutorials and videos. But I was not able to exactly understand how it works.
Thanks in advance.

This is a very broad question. Connecting MS-SQL server from node usually uses tedious package. You can directly use this package to insert data as described in https://learn.microsoft.com/en-us/sql/connect/node-js/step-3-proof-of-concept-connecting-to-sql-using-node-js
Following is a snippet from the above link.
var Connection = require('tedious').Connection;
var config = {
userName: 'yourusername',
password: 'yourpassword',
server: 'yourserver.database.windows.net',
// If you are on Azure SQL Database, you need these next options.
options: {encrypt: true, database: 'AdventureWorks'}
};
var connection = new Connection(config);
connection.on('connect', function(err) {
// If no error, then good to proceed.
console.log("Connected");
executeStatement1();
});
var Request = require('tedious').Request
var TYPES = require('tedious').TYPES;
function executeStatement1() {
request = new Request("INSERT SalesLT.Product (Name, ProductNumber, StandardCost, ListPrice, SellStartDate) OUTPUT INSERTED.ProductID VALUES (#Name, #Number, #Cost, #Price, CURRENT_TIMESTAMP);", function(err) {
if (err) {
console.log(err);}
});
request.addParameter('Name', TYPES.NVarChar,'SQL Server Express 2014');
request.addParameter('Number', TYPES.NVarChar , 'SQLEXPRESS2014');
request.addParameter('Cost', TYPES.Int, 11);
request.addParameter('Price', TYPES.Int,11);
request.on('row', function(columns) {
columns.forEach(function(column) {
if (column.value === null) {
console.log('NULL');
} else {
console.log("Product id of inserted item is " + column.value);
}
});
});
connection.execSql(request);
}
But using an ORM is the best practice. sequelize is one of the best in the node community.
$ npm install --save sequelize
$ npm install --save tedious // MSSQL
const Sequelize = require('sequelize');
const sequelize = new Sequelize('database', 'username', 'password', {
host: 'localhost',
dialect: 'mssql'
});
You must go through either of these modules documentation to understand better.

Related

Implement Redis in Node.js Bot Service

Bot Info
App ID: 776ba3b4-38e5-4582-809d-7c8d773cfe9b
SDK Platform: Node.js
SDK Version:
Active Channels: Direct Line
Deployment Environment: Auzure Bot Service
Issue Description
I Need Help implementing Redis to save Bot State. I'm working in a project that is really a requirement that we reduce as much latency as possibe. Right know we are using DocumentDB but since Redis works with memory this could be faster.
I've followed the tutorial using mongo DB, Microsoft Bot framework MongoDB as middle layer to store conversational states, data and context and I'm editing the file /lib/IStorageClient.js to connect, save and retrieve from redis.
Code Example
This is my implementation of the /lib/IStorageClient.js, instead of using MongoDB connection I've put Redis connection
"use strict";
var Consts = require('./Consts');
var redis = require('redis');
var IStorageClient = (function () {
function IStorageClient(options) {
this.options = options;
}
IStorageClient.prototype.initialize = function (callback) {
var _this = this;
var host = "MyRedis.redis.cache.windows.net";
var auth = "KEY";
var client = redis.createClient(6380,host , {auth_pass: auth, tls:
{servername: host}});
this.client = client;
callback(null);
};
IStorageClient.prototype.insertOrReplace = function (partitionKey, rowKey,
entity, isCompressed, callback) {
console.log("=========Insert IStorageClient===========")
var docDbEntity = { id: partitionKey + ',' + rowKey, data: entity,
isCompressed: isCompressed };
var host = "MyRedis.redis.cache.windows.net";
var auth = "KEY";
var client = redis.createClient(6380,host , {auth_pass: auth, tls:
{servername: host}});
client.set(partitionKey + ',' + rowKey, JSON.stringify(docDbEntity),
function(err, reply) {
console.log("=========SET===========");
console.log("ID: ",partitionKey + ',' + rowKey);
console.log("Result: ",docDbEntity);
});
};
IStorageClient.prototype.retrieve = function (partitionKey, rowKey,
callback) {
console.log("=========Retrieve IStorageClient===========")
var id = partitionKey + ',' + rowKey;
var host = "MyRedis.redis.cache.windows.net";
var auth = "KEY";
var client = redis.createClient(6380,host , {auth_pass: auth, tls:
{servername: host}});
//id
client.get(id, function(error, result){
console.log("=========Get===========");
console.log("Search: ",id);
console.log("Result: ",result);
if (error) {
console.log("Error:",error)
callback(error, null, null);
}
else if (result == null) {
callback(null, null, null);
}
else if (result.length == 0) {
callback(null, null, null);
}
else {
var finaldoc = JSON.parse(result);
callback(null, finaldoc, null);
}
});
};
IStorageClient.getError = function (error) {
if (!error)
return null;
return new Error('Error Code: ' + error.code + ' Error Body: ' +
error.body);
};
return IStorageClient;
}());
exports.IStorageClient = IStorageClient;
Reproduction Steps
Download Microsoft Bot framework MongoDB as middle layer to store conversational states, data and context
Replace /lib/IStorageClient.js with my implementation
Set a Redis account and key in the /lib/IStorageClient.js
Run in the bot emulator
Actual Results
I could see the json saving to Redis, also I could print the retrieve result in the console, but the thing is that the answer is not being received in the bot emulator.
You are looking for botbuilder-redis-storage middleware, available here:
GitHub - https://github.com/suttna/botbuilder-redis-storage
NPM - https://www.npmjs.com/package/botbuilder-redis-storage
Usage example:
var redis = require('redis')
var RedisStorage = require('botbuilder-redis-storage')
var builder = require('botbuilder')
// Initialize redis client
var redisClient = redis.createClient(process.env.REDIS_URL, { prefix: 'bot-storage:' });
// Create new storage with redis client
var storage = new RedisStorage(redisClient)
var connector = new builder.ChatConnector()
var bot = new builder.UniversalBot(connector)
// Configure bot to use the RedisStorage
bot.set('storage', storage)

How to invoke/use the ranker ID from within the nodeJs code to call Retrieve & Rank API?

I have tried a successful approach in Watson Retrieve and Rank, to set the 'ranker_id' from within the code snippet versus setting it as an environment variable.
Below is the code snippet:
var qs = require('querystring');
// search documents
var ranker_id = 'replace with ID';
var question = payload.input.text; //Only the question is required
var query = **qs.stringify({q: question, ranker_id: ranker_id, fl: 'id,title,contentHtml'});**
solrClient.get('fcselect', query, function(err, searchResponse) {...}.
In some versions of npm, qs also works-
var qs = require('qs');
**This would be the requirement to deploy in all production architecture, where the code would reside in production servers & make calls to the API. In such a scenario, env variable(ranker_id) could not be set in production environment, hence this approach
You can see in this documentation here and see the all examples how to use Retrieve and Rank - IBM Watson.
In this case the password and username is Service Credentials.
One example search and rank:
var watson = require('watson-developer-cloud');
var retrieve_and_rank = watson.retrieve_and_rank({
username: '{username}',
password: '{password}',
version: 'v1'
});
var params = {
cluster_id: 'sc1ca23733_faa8_49ce_b3b6_dc3e193264c6',
collection_name: 'example_collection'
};
// Use a querystring parser to encode output.
var qs = require('qs');
// Get a Solr client for indexing and searching documents.
// See https://github.com/watson-developer-cloud/node-sdk/blob/master/services/retrieve_and_rank/v1.js
solrClient = retrieve_and_rank.createSolrClient(params);
var ranker_id = 'B2E325-rank-67'; //PASTE YOUR RANKER_ID
var question = 'what is the basic mechanism of the transonic aileron buzz';
var query = qs.stringify({q: question, ranker_id: ranker_id, fl: 'id,title'});
solrClient.get('fcselect', query, function(err, searchResponse) {
if(err) {
console.log('Error searching for documents: ' + err);
}
else {
console.log(JSON.stringify(searchResponse.response.docs, null, 2));
}
});
See one example how Get information about a ranker:
var watson = require('watson-developer-cloud');
var retrieve_and_rank = watson.retrieve_and_rank({
username: '{username}', //username from Service Credentials Retrieve and Rank
password: '{password}', // password from Service Credentials Retrieve and Rank
version: 'v1'
});
var params = {
ranker_id: 'B2E325-rank-67', //PASTE YOUR RANKER_ID
};
retrieve_and_rank.rankerStatus(params,
function(err, response) {
if (err)
console.log('error:', err);
else
console.log(JSON.stringify(response, null, 2));
});
Example Index documents:
//require watson
var watson = require('watson-developer-cloud');
//call with your password and username from Service Retrieve and Rank Credentials
var retrieve_and_rank = watson.retrieve_and_rank({
username: '{username}',
password: '{password}',
version: 'v1'
});
//cluster id from your documentation
var params = {
cluster_id: 'sc1ca23733_faa8_49ce_b3b6_dc3e193264c6',
collection_name: 'example_collection',
};
// your doc here
var doc = {
id: 1,
author: 'brenckman,m.',
bibliography: 'j. ae. scs. 25, 1958, 324.',
body: 'experimental investigation of the aerodynamics of a wing in a slipstream. an experimental study of a wing in a propeller slipstream was made in order to determine the spanwise distribution of the lift increase due to slipstream at different angles of attack of the wing and at different free stream to slipstream velocity ratios.',
title: 'experimental investigation of the aerodynamics of a wing in a slipstream'
};
//Get a Solr client for indexing and searching documents with rankerid inside params variable
solrClient = retrieve_and_rank.createSolrClient(params);
console.log('Indexing a document...');
solrClient.add(doc, function (err, response) {
if (err) {
console.log('Error indexing document: ', err);
}
else {
console.log('Indexed a document.');
solrClient.commit(function(err) {
if(err) {
console.log('Error committing change: ' + err);
}
else {
console.log('Successfully committed changes.');
}
});
}
});

How to serve a PDF file stored using CollectionFS in Meteor?

I am working on a Meteor application.
Currently, I have a few PDFs on my server. To serve these already existing PDFs directly to the client, I do it this way and it works very well:
Router.route("/file/:fileName", function() {
var fileName = this.params.fileName;
// console.log(process.env.PWD);
var filePath = process.env.PWD + '/' + fileName;
var fs = Meteor.npmRequire('fs');
var data = fs.readFileSync(filePath);
this.response.writeHead(200, {
"Content-Type": "application/pdf",
"Content-Length": data.length
});
this.response.write(data);
this.response.end();
}, {
where: "server"
});
I save these PDFs to Mongo using CollectionFS (Later, I shall generate PDFs and save them. For now, I am just directly saving these already existing PDFs to Mongo as I first want to get the Mongo part to work.).
testCollection = new FS.Collection("testCollection", {
stores: [new FS.Store.GridFS("testCollection")]
});
testCollection.allow({
'insert': function () {
return true;
}
});
var file = new FS.File(process.env.PWD + '/PDFKitExampleServerSide.pdf');
file.encoding = 'binary';
file.name('myPDF.pdf');
var document = testCollection.insert(file);
console.log(document._id);
My question is, after I save these PDFs to Mongo using CollectionFS (like I do above), how do I retrieve and serve these PDFs?
Router.route("/database/:pdfId", function() {
//need help here
}, { where: "server"});
After a lot of searching and trying, I've finally gotten it to work.
Router.route("/database/:pdfId", function() {
var pdfId = this.params.pdfId;
var file = testCollection.findOne({_id: pdfId});
var readable = file.createReadStream("tmp");
var buffer = new Buffer(0);
readable.on("data", function(b) {
buffer = Buffer.concat([buffer, b]);
});
var response = this.response;
readable.on("end", function() {
response.writeHead(200, {
"Content-Type": "application/pdf",
"Content-Length": buffer.length
});
response.write(buffer);
response.end();
});
}, {
where: "server"
});
I know that this question is old, but I found an easier way to store and retrieve PDFs. Apparently if you store your PDFs in the database and they are smaller than 16MB (which is likely in this type of files) the performance is way slower than if you store the files in your server file system.
For doing that, you can use FS.Store.FileSystem instead of FS.Store.GridFS. The following code works for me:
// Client
var pdfStore = new FS.Store.FileSystem("uploadedFiles");
UploadedFiles = new FS.Collection("uploadedFiles", {
stores: [pdfStore],
filter: {
allow: {
extensions: ['pdf','doc','docx','xls','xlsx','ppt','pptx''jpg','png','jpeg']
}
}
});
// Server
var pdfStore = new FS.Store.FileSystem("uploadedFiles", {path: uploadFilesPath});
UploadedFiles = new FS.Collection("uploadedFiles", {
stores: [pdfStore],
filter: {
maxSize: 5242880, // 5MB in bytes
allow: {
extensions: ['pdf','doc','docx','xls','xlsx','ppt','pptx''jpg','png','jpeg']
},
deny: {
extensions: ['exe']
},
onInvalid: function (message) {
if (Meteor.isClient) {
alert(message);
} else {
console.log(message);
}
}
}
});
And then just use this little helper to retrieve the url to the file:
get_uploaded_link: function(id){
console.log(id);
var file = UploadedFiles.findOne({_id: id});
return file.url();}

UCWA Lync authentication - 500 web ticket is invalid

I'm trying to create a simple client for Lync using Nodejs.
Base on http://ucwa.lync.com/documentation/KeyTasks-CreateApplication I've made someting like this.
It works until last step #9 when I should register my app with UCWA. Server responds with code 500 and silly explanation
There is a problem with the resource you are looking for, and it cannot be displayed
And headers
x-ms-diagnostics': '28032;source="mysource";reason="The web ticket is invalid."'
var http = require('request-promise');
var lync = {};
lync.setup = function(email, password){
var self = this;
var hostname = email.split('#');
this.username = email;
//discover urls
return http.get('http://lyncdiscover.'+hostname[1])
.then(function(d) {
var parsed = JSON.parse(d);
self.urls = {
self: parsed._links.self.href,
user: parsed._links.user.href,
xframe: parsed._links.xframe.href
};
return http.get(self.urls.user);
}).catch(function(err){
if(err.statusCode == 401){
var toParse = err.response.headers['www-authenticate'];
var Oauth = toParse.match(/https:\/\/[\d\w\./_-]*/i)[0];
var loginPost = {
grant_type: 'password',
username: email,
password: password
};
return http.post(Oauth, {form:loginPost});
}
return false
}).then(function(data){
var parsed = JSON.parse(data);
//setup authorization
http = http.defaults({
headers: {Authorization: parsed.token_type + ' ' + parsed.access_token}
});
//console.log(self.urls.user);
//console.log('Authorization:'+ parsed.token_type + ' ' + parsed.access_token);
return http.get(self.urls.user);
}).then(function(data){
var parsed = JSON.parse(data);
self.urls.applications = parsed._links.applications.href;
var registerApp = {
culture : "en-us",
endpointId : "2d9dc28d-4673-4035-825c-feb64be28e4e",
userAgent : "Test"
};
var r = "{'userAgent': 'NodeJs', 'endpointId' : '2d9dc28d-4673-4035-825c-feb64be28e4e', 'culture': 'en-US'}";
return http.post(self.urls.applications, {body: registerApp, json:true});
})
.then(function(data){
console.log(data);
})
.catch(function(err){
console.log(err);
return false;
});
};
//run app
lync.setup('login#domain.com', 'password').then(function(ret){
});
One key point here. It's not my server. I just have an account over there and I can login with official Lync client or Pidgin plugin.
Are there some extra steps to "allow" my app to work with UCWA?
#ShelbyZ
I can easily authorize using Oauth. I'm receiving authorization token so I'm logged in.
I'm receiving json similar to
"_links":{
"self":{"href":"link"},
"applications":{"href":"i need this"},
"xframe":{"href":"link"}
} }
Now. I need to "register my application" doing POST.
In this last step I get 500 code response.
I hope It's not related with that #Matthew Proctor said..
becouse I cannot simple administrate the server
Thank you #ShelbyZ
You were right, it was split-domain scenario. Now authorization works, and I can register my app. Also example for future generations
var http = require('request-promise');
var lync = {};
lync._authorize = function(){
var self = this;
var orgDomain = self.urls.user.match(/https:\/\/([\w\d\.]+)/i)[0];
//console.log(orgDomain);
http.get(self.urls.user).catch(function(err){
if(err.statusCode == 401){
var toParse = err.response.headers['www-authenticate'];
var Oauth = toParse.match(/https:\/\/[\d\w\./_-]+/i)[0];
var loginPost = {
grant_type: 'password',
username: self.username,
password: self.password
};
return http.post(Oauth, {form:loginPost});
}
}).then(function(data){
if(data) {
var parsed = JSON.parse(data);
//setup authorization
http = http.defaults({
headers: {Authorization: parsed.token_type + ' ' + parsed.access_token}
});
return http.get(self.urls.user);
}
}).then(function(data){
//check for split-domain scenario
var parsed = JSON.parse(data);
var domain = parsed._links.self.href.match(/https:\/\/([\w\d\.]+)/i)[0];
console.log('[1] '+orgDomain);
console.log('[2] '+domain);
if(domain!== orgDomain){
//split domain scenario
self.urls.user = self.urls.user.replace(orgDomain, domain);
http = http.defaults({
headers: {Authorization: null}
});
self._authorize();
} else { //create app
var parsed = JSON.parse(data);
self.urls.applications = parsed._links.applications.href;
var registerApp = {
culture : "en-us",
endpointId : "2d9dc28d-4673-4035-825c-feb64be28e4e",
userAgent : "NodeJs client"
};
return http.post(self.urls.applications, {body: registerApp, json:true});
}
}).then(function(app){
console.log(app);
});
};
lync.setup = function(email, password){
var self = this;
var hostname = email.split('#');
this.username = email;
this.password = password;
//discover urls
return http.get('http://lyncdiscover.'+hostname[1])
.then(function(d) {
var parsed = JSON.parse(d);
self.urls = {
self: parsed._links.self.href,
user: parsed._links.user.href,
xframe: parsed._links.xframe.href
};
return self._authorize();
});
};
//run app
lync.setup('username#domain.com', 'password');
I was getting the same error before I added my test domain to the list of Allowed Domains.
This can be updated via PowerShell, full details below:
Enabling UCWA and Configuring Allowed Domains
I've had clients see similar errors when running my code from http://localhost/, their fix was to test their code on a FQDN (ie http://testmyucwacode.mydomain.com/).

Accessing Cloudant using nodejs nano

Is there a gist which I can follow to use to connect to cloudant using nano / nodejs. Or, is there any other library which I can use to connect from nodejs to cloudant. Here is the error I encounter, when i try to connect.
description: 'one of _writer, _creator is required for this request',
Here is how my code looks
// require nano, point it at cloudant's root
var config = require('config');
var nano = require('nano')({url: config.cloudant.url})
, username = config.cloudant.username
, userpass = config.cloudant.password
, cookies = {}
, callback = console.log // this would normally be some callback
;
nano.auth(username, userpass, function (err, body, headers) {
if (err) {
return callback(err);
}
if (headers && headers['set-cookie']) {
cookies['user'] = headers['set-cookie'];
}
callback(null, "it worked");
});
var db = require('nano')({
'url': config.cloudant.url + '/' + config.cloudant.database,
'cookie': cookies['user']
});
After this, when i try to connect - i get the error
var db = require('nano')({
'url': config.cloudant.url + '/' + config.cloudant.database,
'cookie': cookies['user']
});
var doc = {"hello": "world"};
db.insert(doc, function (err, body, headers) {
if (err) {
return callback(err);
}
// change the cookie if couchdb tells us to
if (headers && headers['set-cookie']) {
cookies['user'] = headers['set-cookie'];
}
callback(null, "it worked");
});
You can try using this https://github.com/cloudant/nodejs-cloudant . It is a wrapper over nano to directly connect with cloudant.

Resources