i have an azure function with http trigger, then this function get data from url and process this data and parse to json.
What i need is when the function is called before return the answer store this data into azure table storage.
const axios = require('axios');
const azureStorage = require('azure-storage');
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
const name = (req.query.url || (req.body && req.body.url));
const responseMessage = name
? `Hello you are trying to download file from ${req.body.name} with url ${req.body.url}`
: "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.";
context.res = {
body: {
textParsed: await getData(req.body.url),
statusTable: saveDataOnTable(context),
conectionString: process.env.conectionString
}
};
}
async function getData(url) {
const request = await axios(url).then(response => { return response.data });
const jsonData = await csvJSON(request);
return jsonData;
}
async function csvJSON(csv) {
let regularQuotes = /"/g;
let csvArray = csv.split("\r\n");
let headers = csvArray[0].split(",");
let result = [];
//recorre el array con los datos del csv
for (let index = 1; index < csvArray.length; index++) {
//comprueba que el campo no este vacio
if (!csvArray[index]) continue;
let obj = {};
//guarda la linea actual
let currentline = csvArray[index];
//reemplaza los caracteres de escape como comillas
currentline = regularQuotes[Symbol.replace](currentline, '');
//separa los datos utilizando las comillas como caracter de escape
currentline = currentline.split(",");
for (let j = 0; j < headers.length; j++) {
//elimina espacios de las cabezeras
let head = headers[j].trim();
//elimina espacios de los datos
let value = currentline[j].trim();
//crea el objeto con los valores
obj[head] = value;
}
result.push(obj);
}
//devuelve una string en formato json
return JSON.stringify(result);
}
There is sdk available to connect and manipulate a table in azure called #azure/data-tables.
Install it along with uuid
npm install #azure/data-tables
npm install uuid
Import specific functions from the sdk such as:
const { v4: uuidv4 } = require('uuid');
const { TableClient, AzureNamedKeyCredential } = require("#azure/data-tables");
Now you will need the connection key of your storage entity. Also please create a table beforehand in that storage space.
const accountname = "";
const accountkey = "";
const tableName = "";
Now we create a tableclient which will connect to the table storage and add the entity to the table.
const credential = new AzureNamedKeyCredential(accountname, accountkey);
const client = new TableClient(`https://${accountname}.table.core.windows.net`, tableName, credential);
Now we have to create an entity which will be send to the table and then we will send it to table storage.
// I am assuming that the response of the url isin json
let testjsonobject = {};
testjsonobject.language = "javascript";
testjsonobject.platform = "azure functions"
context.log(testjsonobject);
let testjsonobjectString = JSON.stringify(testjsonobject);
let partitionkeynew = uuidv4();;//crypto.randomUUID();
context.log(partitionkeynew);
const entity = {
partitionKey : partitionkeynew,
rowKey:"1",
description:testjsonobjectString
}
let result = await client.createEntity(entity);
complete code :
const { TableServiceClient, odata } = require("#azure/data-tables");
const { v4: uuidv4 } = require('uuid');
const { TableClient, AzureNamedKeyCredential } = require("#azure/data-tables");
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
const accountkey = "";
const accountname = "";
const tableName = "";
const credential = new AzureNamedKeyCredential(accountname, accountkey);
const client = new TableClient(`https://${accountname}.table.core.windows.net`, tableName, credential);
let testjsonobject = {};
testjsonobject.language = "javascript";
testjsonobject.platform = "azure functions"
context.log(testjsonobject);
let testjsonobjectString = JSON.stringify(testjsonobject);
let partitionkeynew = uuidv4();;//crypto.randomUUID();
context.log(partitionkeynew);
const entity = {
partitionKey : partitionkeynew,
rowKey:"1",
description:testjsonobjectString
}
let result = await client.createEntity(entity);
context.log(result);
context.res = {
// status: 200, /* Defaults to 200 */
body: result
};
}
Here result will get a Etag if the operation is successful.
Related
when i run this code it says KustoAuthenticationError: Failed to get cloud
info for cluster https://clusterName.kusto.windows.net
The appId is the Application (client) ID,
appKey is the value of the client secret
and authorityId is the Directory (tenant) ID
I got this code from: https://github.com/Azure/azure-kusto-node/blob/master/packages/azure-kusto-ingest/example.js
code:
const IngestClient = require("azure-kusto-ingest").IngestClient;
const IngestStatusQueues = require("azure-kusto-ingest").IngestStatusQueues;
const IngestionProps = require("azure-kusto-ingest").IngestionProperties;
const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnectionStringBuilder;
const { DataFormat, JsonColumnMapping, IngestionMappingKind, CompressionType, ReportLevel, ReportMethod } = require("azure-kusto-ingest");
const { BlobDescriptor, StreamDescriptor } = require("azure-kusto-ingest").IngestionDescriptors;
const StreamingIngestClient = require("azure-kusto-ingest").StreamingIngestClient;
// const fs = require("fs");
const clusterName = "clusterName";
const authorityId = "authorityId";
const appId = "appId"
const appKey = "appKey"
var d = new Date();
var dateTimeNow = d.toJSON().slice(0,19).replace('T',':');
var data = {"Id" : "1314444525", "Temperature" : 32, "Battery" : 89, "Humidity" : 94, "Time" : dateTimeNow};
var s = require('stream');
var stream = new s.Readable();
stream.push(JSON.stringify(data));
stream.push(null);
// Streaming ingest client
const props2 = new IngestionProps({
database: "DBname",
table: "Tablename",
format: DataFormat.JSON,
ingestionMappingReference: "Pre-defined mapping name",
});
const streamingIngestClient = new StreamingIngestClient(
KustoConnectionStringBuilder.withAadApplicationKeyAuthentication(`https://${clusterName}.kusto.windows.net`, appId, appKey, authorityId),
props2
);
startStreamingIngestion();
async function startStreamingIngestion() {
try {
await streamingIngestClient.ingestFromStream(stream, props2);
console.log("Ingestion done");
await waitForStatus();
} catch (err) {
console.log(err);
}
}
async function waitForStatus(numberOFIngestions = 1) {
while ((await statusQueues.failure.isEmpty()) && (await statusQueues.success.isEmpty())) {
console.log("Waiting for status...");
await sleep(1000);
}
const failures = await statusQueues.failure.pop(numberOFIngestions);
for (let failure of failures) {
console.log('Failed: ${JSON.stringify(failure)}');
}
const successes = await statusQueues.success.pop(numberOFIngestions);
for (let success of successes) {
console.log('Succeeded: ${JSON.stringify(success)}');
}
}
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
This usually means you don’t have access to your ADX cluster’s endpoint. Verify whether your ADX cluster allows requests over the public network and that your Azure AD App doesn't have any outgoing network restrictions. In addition, make sure to grant 'ingestor' permission on your ADX database to your Azure AD App.
I am attempting to convert a HTTP request body in JSON format into a hashmap. Below is how my request looks:
{
"car": [{
"ford": "focus",
"audi": "a6"
}],
"food": [{
"soup": "leek"
}]
}
And below is the code I have tried so far:
const express = require('express');
const router = express.Router();
router.post('/items', async (req, res) => {
res.setHeader('Content-Type', 'application/json');
let jsonString = JSON.stringify(req.body);
let jsonObject = JSON.parse(jsonString);
let jsonMap = new Map(Object.entries(jsonObject));
var mainArray = {};
var carKey = 'car';
mainArray[carKey] = [];
for (var entry of jsonMap.entries()) {
let key = entry[0], value = entry[1];
let innerJson = JSON.stringify(value);
let innerJsonObj = JSON.parse(innerJson);
for(var innerKey in innerJsonObj) {
let tempStr = JSON.stringify(innerJsonObj[innerKey]);
let tempVal = JSON.parse(tempStr);
let attr = '';
let param = '';
for(var k in tempVal) {
attr = k;
param = tempVal[k];
logger.info(k + "=" + tempVal[k]);
}
let result = someMethod(param);
var carData = {
attr : JSON.stringify(result);
}
mainArray[carKey].push(carData);
}
}
});
I want to display the results in the same way as received in the request, but with the use of the binary result of someMethod(). Below is what im trying to create as a response.
{
"car": [
{
"ford": "binarydata",
"audi": "binarydata"
}
],
"food": [
{
"soup": "binarydata"
}
]
}
I have managed to achieve this using the code below. This allows me to specify the key and value in the JSON I add to the array via the push() command. Thank you for the guidance.
var carData = {};
carData[attr] = JSON.parse(JSON.stringify(`${result}`));
mainArray[carKey].push(carData);
I am writing a node JS web crawler class, and I have encountered the following error, this.textInvertedIndex[word].push is not a function. Upon further inspection I realised that for some reason this.textInvertedIndex[word] was written as a native object, function Object({ [native code] }). For the first few iterations, by console logging this.textInvertedIndex everything seemed fine as it was an object of arrays. But then suddenly this error occurred. Is there any part of the code where I am implicitly rewriting textInvertedIndex?
Here is the relevant class:
function Crawler(queue, maxIndexSize) {
this.queue = queue;
this.maxIndexSize = maxIndexSize;
this.findChunks = () => {
let currentChunk;
let minimumDistance = Infinity;
for (i = 1; i <= this.maxIndexSize; i++) {
if (this.maxIndexSize % i === 0) {
const newDistance = Math.abs(i - 30);
if (newDistance < minimumDistance) {
minimumDistance = newDistance;
currentChunk = i;
} else {
return currentChunk
};
};
};
};
this.chunks = this.findChunks();
this.chunkSize = this.maxIndexSize / this.chunks;
this.totalWordOccurances = {};
this.imageInvertedIndex = {};
this.textInvertedIndex = {};
this.images = [];
this.sites = [];
this.seen = {};
this.write = (url, html) => {
const documentId = this.sites.length;
const website = new Website(url, html);
const title = website.title();
const content = website.content(title);
const words = content.filter(item => typeof item !== "object");
const wordsLength = words.length;
const query = new Query(words);
const individualWords = query.individualize(words);
this.seen[url] = true;
this.sites.push({
url,
title,
description: website.description()
});
for (word of individualWords) {
const normalizedTf = query.count(word) / wordsLength;
const textInvertedIndexEntry = {
documentId,
normalizedTf
};
if (this.textInvertedIndex[word]) {
this.textInvertedIndex[word].push(textInvertedIndexEntry);
} else {
this.textInvertedIndex[word] = [textInvertedIndexEntry];
};
if (this.totalWordOccurances[word]) {
this.totalWordOccurances[word] += 1;
} else {
this.totalWordOccurances[word] = 1;
};
};
for (i = 0; i < content.length; i++) {
const item = content[i];
if (typeof item === "object") {
const imageId = this.images.length;
this.images.push(item);
for (word of individualWords) {
const imageScore = getImageScore(i, word, content);
const imageInvertedIndexEntry = {
imageId,
imageScore
};
if (this.imageInvertedIndex[word]) {
this.imageInvertedIndex[word].push(imageInvertedIndexEntry);
} else {
this.imageInvertedIndex[word] = [imageInvertedIndexEntry];
};
};
};
};
};
this.crawl = async () => {
while (this.sites.length !== this.maxIndexSize) {
let nextQueue = [];
const websitesUnfiltered = await Promise.all(this.queue.map((url) => {
const website = new Website(url);
return website.request();
}));
const websitesToAdd = this.maxIndexSize - this.sites.length;
let websites = websitesUnfiltered.filter(message => message !== "Failure")
.slice(0, websitesToAdd);
for (site of websites) {
const url = site.url;
const htmlCode = site.htmlCode;
const website = new Website(url, htmlCode);
this.write(url, htmlCode);
nextQueue = nextQueue.concat(website.urls());
};
nextQueue = new Query(nextQueue.filter(url => !this.seen[url]))
.individualize();
this.queue = nextQueue;
};
};
};
Called like this
const crawler = new Crawler(["https://stanford.edu/"], 25000000);
crawler.crawl();
this.textInvertedIndex = {}; is defining an Object of which push is not a valid function. you can change it to an array by defining it as this.textInvertedIndex = []; otherwise you can add key/value entries to the object as it is defined like this: this.textInvertedIndex[key] = value;
Turns out, my key was accessing this.textInvertedIndex[word]. And word was constructor. constructor is already a built in object property so it can never be rewritten as an array with .push defined. To solve this problem, make all object keys capital, so constructor will become CONSTRUCTOR, thus making sure that already existing object properties are never called.
I am writing code for a firebase function, the problem is that I need to use a class but when I call the class method the firebase function log shows this error:
ERROR:
ReferenceError: Proverbi is not defined
at exports.getProverbio.functions.https.onRequest (/srv/index.js:48:26)
at cloudFunction (/srv/node_modules/firebase-functions/lib/providers/https.js:49:16)
at /worker/worker.js:783:7
at /worker/worker.js:766:11
at _combinedTickCallback (internal/process/next_tick.js:132:7)
at process._tickDomainCallback (internal/process/next_tick.js:219:9)
Here's the "index.js" code:
//firebase deploy --only functions
// The Cloud Functions for Firebase SDK to create Cloud Functions and setup triggers.
const functions = require('firebase-functions');
// The Firebase Admin SDK to access the Firebase Realtime Database.
const admin = require('firebase-admin');
admin.initializeApp();
// Take the text parameter passed to this HTTP endpoint and insert it into the
// Realtime Database under the path /messages/:pushId/original
exports.addStanza = functions.https.onRequest(async (req, res) => {
// Grab the text parameter.
const nome = req.query.text;
// Push the new message into the Realtime Database using the Firebase Admin SDK.
const snapshot = await admin.database().ref('/stanze').push({giocatori: {giocatore:{nome:nome,punteggio:0}}});
// Redirect with 303 SEE OTHER to the URL of the pushed object in the Firebase console.
//res.redirect(200, nome.toString());
var link = snapshot.toString().split('/');
res.json({idStanza:link[4]});
});
// Listens for new messages added to /messages/:pushId/original and creates an
// uppercase version of the message to /messages/:pushId/uppercase
exports.addFirstPlayer = functions.database.ref('/stanze/{pushId}/giocatori/giocatore/nome')
.onCreate((snapshot, context) => {
// Grab the current value of what was written to the Realtime Database.
const nome = snapshot.val();
// const snapshot3 = snapshot.ref('/stanza/{pushId}/giocatori/giocatore').remove();
const snapshot2 = snapshot.ref.parent.parent.remove();
return snapshot.ref.parent.parent.push({nome:nome,punteggio:0});
});
exports.addPlayer = functions.https.onRequest(async (req, res) => {
// Grab the text parameter.
const nome = req.query.text;
const idStanza = req.query.id;
// Push the new message into the Realtime Database using the Firebase Admin SDK.
const snapshot = await admin.database().ref('/stanz/'+idStanza+"/giocatori").push({nome:nome,punteggio:0 });
// Redirect with 303 SEE OTHER to the URL of the pushed object in the Firebase console.
//res.redirect(200, nome.toString());
res.json({success:{id:idStanza}});
});
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({proverbio:{inizio:inizio,fine:fine,difficolta:difficolta}});
});
Here's the code that's causing the problem:
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({proverbio:{inizio:inizio,fine:fine,difficolta:difficolta}});
});
Here's the "Proverbi.class" code:
class Proverbi{
constructor(n) {
this.magicNumber = n;
}
getProverbio() {
var text = "";
switch (magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
How can I use the "Proverbi" class inside the "index.js"?
You need to add the definition of your Proverbi Class to the index.js file.
If you are just going to use this Class in the getProverbio Cloud Function, do as follows:
exports.getProverbio = functions.https.onRequest(async (req, res) => {
class Proverbi {
constructor(n) {
this.magicNumber = n;
}
getProverbio() {
var text = "";
switch (this.magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({ proverbio: { inizio: inizio, fine: fine, difficolta: difficolta } });
});
If you want to use the Class in other functions, just declare it as follows:
class Proverbi {
constructor(n) {
console.log(n);
}
getProverbio() {
console.log(this.magicNumber);
console.log(this.magicNumber);
var text = "";
switch (this.magicNumber) {
case 1:
text += ("Chip");
text += ("-Chop");
break;
}
return text;
}
}
exports.getProverbio = functions.https.onRequest(async (req, res) => {
const difficolta = req.query.difficolta;
var proverbioClass = new Proverbi(2);
var p = proverbioClass.getProverbio();
var proverbio = p.split('-');
var inizio = proverbio[0];
var fine = proverbio[1];
res.json({ proverbio: { inizio: inizio, fine: fine, difficolta: difficolta } });
});
i am working on lex chatbot and want to store user data in dynamodb.
here is my databaseManager.js file code
'use strict';
const { v1: uuidv1 } = require('uuid');
const AWS = require('aws-sdk');
const dynamo = new AWS.DynamoDB.DocumentClient();
module.exports.saveBookingToDatabase = async function(Arrival_city, Departure_city, Flight_type, Phone_number){
console.log('saveBookingToDatabase');
const item = {};
item.bookingId = uuidv1();
item.arrivalCity = Arrival_city;
item.departureCity = Departure_city;
item.classType = Flight_type;
item.phone = Phone_number;
const params = {
TableName: 'air_stallion',
Item: item
};
try {
let result = await dynamo.put(params)
console.log(`Saving ticket ${JSON.stringify(item)}`);
return item;
} catch(e) {
throw (e)
}
}
Table has been created but data is now showing in table
The values should not be empty, give some default values to prevent null or empty values.
For Example:
const item = {};
item.bookingId = uuidv1();
item.arrivalCity = Arrival_city || "Arr";
item.departureCity = Departure_city || "Dept";
item.classType = Flight_type || "Type";
item.phone = Phone_number || "Phone";
If the values are okay, then try with
let result = await dynamo.put(params).promise()