Related
I am new to Restful API development using NodeJS and SQL Server. I am trying to do a simple [post] operation where I am passing an array of objects to the API endpoint and then calling a SQL Server procedure with a table valued parameter. I am getting the below error
Cannot read properties of undefined (reading 'generateTypeInfo')
I was really shocked to see that there is not a single help topic found over Google regarding this error. I do not want to learn ASP.NET Core for this because JavaScript has an easy learning curve. Am I doing a mistake by developing a Rest API by using the combination of NodeJS and SQL Server? Below is my Related .JS file called in Post endpoint
const sql = require("mssql/msnodesqlv8");
const dataAccess = require("../DataAccess");
const fn_CreateProd = async function (product) {
let errmsg = "";
let connPool = null;
await sql
.connect(global.config)
.then((pool) => {
global.connPool = pool;
result = pool.request().query("select * from products where 1=2");
return result;
})
.then((retResult) => {
const srcTable = retResult.recordset.toTable("tvp_products");
let newsrcTable = Array.from(srcTable.columns);
console.log('Source table b4 mapping',srcTable)
newsrcTable = newsrcTable.map((i) => {
i.name = i.name.toUpperCase();
return i;
});
console.log('Source table after convert array with mapping',newsrcTable)
const prdTable = dataAccess.generateTable(
newsrcTable,
product,
"tvp_products"
);
console.log("Prepared TVp data", prdTable);
const newResult = dataAccess.execute(`sp3s_ins_products_tvp`, [
{ name: "tblprods", value: prdTable },
]);
console.log("Result of Execute Final procedure", newResult);
return newResult;
})
.then(result => {
console.log("Result of proc", result);
if (!result.errmsg) errmsg = "Products Inserted successfully";
else errmsg = result.errmsg;
})
.catch((err) => {
console.log("Enter catch of Posting prod", err.message);
errmsg = err.message;
})
.finally((resp) => {
sql.close();
});
return { retStatus: errmsg };
};
module.exports = fn_CreateProd;
and Content of Generatetable function are as below :
const generateTable = (columns, entities,tvpName) => {
const table = new mssql.Table(tvpName);
// const testobj = {type : [sql.numeric],name : 'Sanjay'}
// console.log('Columns testobj',testobj.type)
columns.forEach(column => {
// console.log('COlumn data for COlumn :',column)
if (column && typeof column === 'object' && column.name && column.type) {
let colOptions = {}
if (column.type==mssql.Numeric)
{
colOptions.scale=column.scale
colOptions.precision=column.precision
}
else
if (column.type==mssql.VarChar || column.type==mssql.Char )
{
colOptions.length = column.length
}
// console.log (`Column name type for column :${column.name} -${colType}-Actual :${column['type']}`)
if (column.hasOwnProperty('options')) {
table.columns.add(column.name.toUpperCase(), colType,column.options);
} else {
table.columns.add(column.name.toUpperCase(),colOptions)
}
}
});
console.log('Generated table',table)
const newEntities = entities.map(obj=>keystoUppercase(obj))
// console.log('New entities after uppercase',newEntities)
newEntities.forEach(entity => {
table.rows.add(...columns.map(i =>
entity[i.name]));
});
return table;
};
I have found the solution now. Actually, if you can see the code of generateTable function, I was adding the columns into the table but not mentioning the data type of the columns due to which this error was coming. I have added one more property [type] in the [colOptions] object being passed to columns.add command in the function [Generatetable]. Thanks a lot anyway to you for quick replies by Dale. K.
I am new to node js. I am trying to do scraping some value and name from the select tab from a particular website. My code as follows,
const rp = require('request-promise');
const crio = require('cheerio');
const url = 'https://myurl';
const getOptions = function(optionName,ignoreOption) {
return rp(url)
.then(function(html) {
crio('[name='+optionName+']', html).find('option').each((i,op) => {
if(crio(op).text() != ignoreOption && crio(op).text() != '')
return {
name: crio(op).text(),
value: crio(op).val(),
};
})
})
.catch(function(err) {
//handle error
});
};
getOptions('test','-- Select company name --')
.then( data =>
console.log(data)
);
When I try to run this code, I am getting result as 'undefined'. How do I get the return json response from the getOptions function here?
You are not returning a value inside your then function that is using crio.
Change
crio('[name='+optionName+']', html).find('option').each((i,op) => {
to
return crio('[name='+optionName+']', html).find('option').map((i,op) => {
Note the use of map instead of each.
I am trying to call one async function from inside a loop run by another async function. These functions call APIs and I am using request-promise using nodeJS.
functions.js file
const rp = require("request-promise");
// function (1)
async email_views: emailId => {
let data = {};
await rp({
url: 'myapiurl',
qs: { accessToken: 'xyz', emailID: emailId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above JSON looks like this:
...
data:{
records: [
{
...
contactID: 123456,
...
},
{
...
contactID: 456789,
...
}
]
}
...
I am running a loop to get individual record, where I am getting a contactID associated with each of them.
// function#2 (also in functions.js file)
async contact_detail: contactId => {
let data = {};
await rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above function takes one contactId as parameter and gets that contact's detail calling another API endpoint.
Both functions work fine when they are called separately. But I am trying to do it inside a loop like this:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId); // function#2
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
The problem is I am only getting the first contact back from the above code block, i.e. even I have 20 records from function#1, in the loop when I am calling contact_detail (function#2) for each contactID (cId), I get contact detail once, i.e. for the first cId only. For rest I get nothing!
What is the correct way to achieve this using nodeJs?
UPDATE:
const { App } = require("jovo-framework");
const { Alexa } = require("jovo-platform-alexa");
const { GoogleAssistant } = require("jovo-platform-googleassistant");
const { JovoDebugger } = require("jovo-plugin-debugger");
const { FileDb } = require("jovo-db-filedb");
const custom = require("./functions");
const menuop = require("./menu");
const stateus = require("./stateus");
const alexaSpeeches = require("./default_speech");
const app = new App();
app.use(new Alexa(), new GoogleAssistant(), new JovoDebugger(), new FileDb());
let sp = "";
async EmailViewsByContactIntent() {
try {
const viewEmailId =
this.$session.$data.viewEmailIdSessionKey != null
? this.$session.$data.viewEmailIdSessionKey
: this.$inputs.view_email_Id_Number.value;
let pageIndex =
this.$session.$data.viewEmailPageIndex != null
? this.$session.$data.viewEmailPageIndex
: 1;
const result = await custom.email_views_by_emailId(
viewEmailId,
pageIndex
);
const records = result.data.records;
if (records.length > 0) {
const totalRecords = result.data.paging.totalRecords;
this.$session.$data.viewEmailTotalPages = totalRecords;
sp = `i have found a total of ${totalRecords} following view records. `;
if (totalRecords > 5) {
sp += `i will tell you 5 records at a time. for next 5 records, please say, next. `;
this.$session.$data.viewEmailIdSessionKey = this.$inputs.view_email_Id_Number.value;
this.$session.$data.viewEmailPageIndex++;
}
for (let i = 0; i < records.length; i++) {
const r = records[i];
/* Here I want to pass r.contactID as contactId in the function contact_detail like this: */
const contact = await custom.contact_detail(r.contactID);
const contact_name = contact.data.firstName + " " + contact.data.lastName;
/* The above two lines of code fetch contact_name for the first r.contactID and for the rest I get an empty string only. */
const formatted_date = r.date.split(" ")[0];
sp += `contact ID ${spellOut_speech_builder(
r.contactID
)} had viewed on ${formatted_date} from IP address ${
r.ipAddress
}. name of contact is, ${contact_name}. `;
}
if (totalRecords > 5) {
sp += ` please say, next, for next 5 records. `;
}
} else {
sp = ``;
}
this.ask(sp);
} catch (e) {
this.tell(e);
}
}
I am building an alexa skill using JOVO framework and nodeJS.
UPDATE #2
As a test, I only returned the contactId which I am passing to the contact_detail function and I am getting the correct value back to the above code under my first UPDATE.
async contact_detail: contactId => {
return contactId;
}
It seems even after getting the value right, the function is somehow failing to execute. However, the same contact_detail function works perfectly OK, when I am calling it from another place. Only doesn't not work inside a loop.
What could be the reason?
I must be missing something but don't know what!
You are mixing async await and promises together which is causing you confusion. You typically would use one of the other(as async await effectivly provides syntax sugar so you can avoid dealing with the verbose promise code) in a given location.
Because you mixed the two you are in a weird area where the behavior is harder to nail down.
If you want to use async await your functions should look like
async contact_detail: contactId => {
try {
const body = await rp({
url: 'myapiurl2',
qs: { ... }
});
return JSON.parse(body);
} catch(e) {
console.log(e);
//This will return undefined in exception cases. You may want to catch at a higher level.
}
};
or with promises
async contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { ... }
})
.then( body => JSON.parse(body))
.catch( error => {
console.log(error);
//This will return undefined in exception cases. You probably dont want to catch here.
});
};
Keep in mind your current code executing the function will do each call in series. If you want to do them in parallel you will need to call the function a bit differently and use something like Promise.all to resolve the result.
Here you go:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
await Promise.all(records.map(async record => {
let cId = record.contactID;
let contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
});
console.log(names);
...
I'm posting this as an answer only because I need to show you some multi-line code as part of throubleshooting this. Not sure this solves your issue yet, but it is a problem.
Your contact_detail() function is not properly returning errors. Instead, it eats the error and resolves with an empty object. That could be what is causing your blank names. It should just return the promise directly and if you want to log the error, then it needs to rethrow. Also, there's no reason for it to be declared async or to use await. You can just return the promise directly. You can also let request-promise parts the JSON response for you too.
Also, I notice, there appears to be a syntax error in your .catch() which could also be part of the problem.
contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
json: true,
method: 'GET'
}).catch( error => {
// log error and rethrow so any error propagates
console.log(error);
throw error;
});
};
Then, you would call this like you originally were (note you still use await when calling it because it returns a promise):
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
I am developing a RESTful API using NodeJS and Express.
I noticed that incoming requests sometimes lack of some expected variables, which cause the program to crash, saying it couldn't set the value of a variable, to an 'undefined' value - as no value arrived with the request.
Example:
The application is expecting variableY, but instead variableX is being sent:
formData: { variableX: 'valueX' }
The program is expecting to receive variableY, with the following code:
const checkVariables = Joi.validate({
variableY: req.body.variableY,
}, schema);
The application crashes with the following error:
TypeError: Cannot read property 'variableY' of undefined
I thought about a few ways to handle that, including declaration of variables upon application initiation and using them along, using try-catch.
Another way will be to use if-else, if-chaining, or case-switch, but as you understood of course I am looking for the cleanest way to achieve that.
Any ideas?
Thank you.
** EDIT **
Progressed and managed to achieve the result using the object only. Once trying to reach any of it's inner fields the error will be thrown anyway, example:
if(req.body.variableY == undefined){console.log('The expected variable is undefined');} //true
When the validation addresses a field inside the 'undefined' object:
if(req.body.variableY.dataId == undefined){console.log('The expected variable is undefined');} //crashes
The following error is being thrown again:
TypeError: Cannot read property 'variableX' of undefined
After doing some more digging around, found this Stackoverflow thread:
How to check if object property exists with a variable holding the property name?
Tried using hasOwnProperty, but the same kind of error is being thrown:
TypeError: Cannot read property 'hasOwnProperty' of undefined
Tried wrapping variable declaration using try-catch, still didn't work:
try{
var variableX = req.body.variableX
var variableXDataId = req.body.variableX.dataId
}
catch(e){
res.status(400).send('Wrong request error: Please check your request variables and try again');
}
As this is a really basic validation that should be addressed by most of the RESTful APIs (validating that you get the expected incoming variables inside the request, so the program won't crash by having errors it can't handle - what is the common solution for such problems (expected / unexpected request validation)?
Thank you.
You can take another approach, check req.body before you reach checkVariables:
let body = req.body;
// data - your req.body
// requiredKeys - is an array of strings , [ key1, key2 ... keyN] | string[]
const setKeys = ( data, requiredKeys )=>{
if( !typeof requiredKeys.length ){
requiredKeys = [];
}
if(requiredKeys.length) requiredKeys.forEach( k =>{
k = k.replace(/\+/g,'/');
let keysList = [];
if( /\/+/g.test(k)){
keysList = k.split('/');
}else{
keysList = [k];
}
let [firstKey, ...rest] = keysList;
if( typeof data[firstKey] === 'undefined' ){
data[firstKey] = {};
}
if( rest.length ){
data[firstKey] = setKeys(data[firstKey], [rest.join('/')] );
}
})
return data;
}
let checkedData= setKeys(body, ['variableT','variableP/noname/emptyObj','custom/object/does/not/exist/but/it/will/be/created/here']);
const checkVariables = Joi.validate(checkedData, schema);
UPDATE
Below you will find an working example on how things should work during a /(let's say /usersStatus/:id ) request:
const express = require('express')
const app = express()
const port = 3000
const setKeys = (data, requiredKeys) => {
if (!typeof requiredKeys.length) {
requiredKeys = [];
}
if (requiredKeys.length) requiredKeys.forEach(k => {
k = k.replace(/\+/g, '/');
let keysList = [];
if (/\/+/g.test(k)) {
keysList = k.split('/');
} else {
keysList = [k];
}
let [firstKey, ...rest] = keysList;
if (typeof data[firstKey] === 'undefined') {
data[firstKey] = {};
}
if (rest.length) {
data[firstKey] = setKeys(data[firstKey], [rest.join('/')]);
}
})
return data;
}
/**
* Mock some data
*/
const getUserData = (req, res, next) => {
if (typeof req.body === 'undefined') {
req.body = {};
}
req.body = {
variableY: {
someName: 23
},
variableZ: {
name: 3,
type: {
id: 5,
typeName: 'something',
tags: ['a', 'b', 'c']
}
}
};
console.log('Middleware 1 getUserData');
next();
}
/**
* 1. Setup our middleware for checking keys
* "requiredKeys" is an array of strings
*/
const middlewareSetKeys = (requiredKeys, wrappedMiddleware) => {
return (req, res, next) => {
console.log('Middleware 2 middlewareSetKeys');
if (typeof req.body === "undefined") {
console.log('Leaving Middleware 2 since we don\'t have req.body');
next();
}
/**
* Update "req.body" with keys that we want to have available
* in our next middleware
*/
req.body = setKeys(req.body, requiredKeys);
if (typeof wrappedMiddleware === 'function') {
return wrappedMiddleware.call(this, req, res, next);
} else {
next();
}
}
}
/**
* 2. Let's assume a "user status" situation
* 2.1. We need userInfo from database
* 2.2. Some info won't be retrieved, unless the user accesed some parts of the website to trigger some mechanisms that allows those fields to be exposed, therefore the lack of keys
* 2.3. But we know those keys/objects, and we still want to be present so our code won't crash.
*/
// lets call our getUserData
app.get(
'/', // this path is for some userInfo
getUserData, // this returns userInfo and appends it to `req.data`
middlewareSetKeys([
'userActivity/daily/jobs', // these won't exist in getUserData because the user is lazy and he didn't apply for any JOBS
'userStatus/active/two-weeks-ago', // these won't exist in getUserData because the user joined two days ago. BUT WE STILL NEED IT coz reazons.
]), // We set our desired-later-to-use keys
(req, res, next) => {
/**
* 3. Now our req.body will have our keys
* even if they didn't exist in the getUserData middleware
*/
console.log('Middleware 3 Your middleware');
console.log(req.body);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(req.body, null, 2))
})
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
you can use express validator https://www.npmjs.com/package/express-validator
to validate incoming request.Then add this to your controller where a,b,c ,d are parameters you want to valaidate
const nonEmptyFields = ['a', 'b', 'c', 'd'];
nonEmptyFields.forEach(field => req.assert(field, `${field} cannot be blank`).notEmpty());
const errors = req.validationErrors();
if (errors) {
return res.status(400).send(errors);
}
for validating a field inside a field you can try doing this
typeof(req.body && req.body.name !== undefined)
A solution will be to set a default empty object to replace undefined at a parent level:
// checking for body.variableX.variableZ with object destructuring ES6
const {body = {}} = request;
const {variableX = {}, variableY} = body;
const {variableZ} = variableX.variableZ;
// or prior ES6
var body = request.body || {};
var variableX = body.variableX || {};
var variableY = variableX.variableY;
// or in a statement
var variableY = request.body && request.body.variableX ? request.body.variableX.variableY : undefined;
Based on that you can create your own function like getValue(request, 'body.variableX.variableY') to return null if any parent or the end value is undefined:
// asumes the value in the path is either object or undefined
function getValue(rootObj, path = '') {
const parts = key.split('.');
let value = rootObj || {};
let part;
while ((part = parts.shift()) && value !== null) {
value = value[part] || null;
}
return value;
};
I have a Node.js restful API built in express.js framework. It is usually hosted by pm2.
One of the services has very long process. When front end called the service, the process started up. Since there is an error in database, the process won't be done properly and the error would be caught. However, before the process reached the error, another exactly same process started with same parameters. So in the meantime, two processes were both running while one was ahead of the other. After a long time, the first process reached error point and returned error. Then the second one returned exactly the same thing.
I checked front end Network and noticed there was actually only one request sent. Where did the second request come from?
Edit 1:
The whole process is: first process sends query to db -> long time wait -> second process starts up -> second process sends query to db -> long time wait -> first process receives db response -> long time wait -> second process receives db response
Edit 2:
The code of the service is as follow:
import { Express, Request, Response } from "express";
import * as multer from "multer";
import * as fs from "fs";
import { Readable, Duplex } from "stream";
import * as uid from "uid";
import { Client } from "pg";
import * as gdal from "gdal";
import * as csv from "csv";
import { SuccessPayload, ErrorPayload } from "../helpers/response";
import { postgresQuery } from "../helpers/database";
import Config from "../config";
export default class ShapefileRoute {
constructor(app: Express) {
// Upload a shapefile
/**
* #swagger
* /shapefile:
* post:
* description: Returns the homepage
* responses:
* 200:
*/
app.post("/shapefile", (req: Request, res: Response, next: Function): void => {
// Create instance of multer
const multerInstance = multer().array("files");
multerInstance(req, res, (err: Error) => {
if (err) {
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Multer upload file error.",
errorDetail: err.message,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
return;
}
// Extract files
let files: any = req.files;
// Extract body
let body: any = JSON.parse(req.body.filesInfo);
// Other params
let writeFilePromises: Promise<any>[] = [];
let copyFilePromises: Promise<any>[] = [];
let rootDirectory: string = Config.uploadRoot;
let outputId: string = uid(4);
// Reset index of those files
let namesIndex: string[] = [];
files.forEach((item: Express.Multer.File, index: number) => {
if(item.originalname.split(".")[1] === "csv" || item.originalname.split(".")[1] === "txt" || item.originalname.split(".")[1] === "shp") {
namesIndex.push(item.originalname);
}
})
// Process and write all files to disk
files.forEach((item: Express.Multer.File, outterIndex: number) => {
if(item.originalname.split(".")[1] === "csv" || item.originalname.split(".")[1] === "txt") {
namesIndex.forEach((indexItem, index) => {
if(indexItem === item.originalname) {
ShapefileRoute.csv(item, index, writeFilePromises, body, rootDirectory, outputId,);
}
})
} else if (item.originalname.split(".")[1] === "shp") {
namesIndex.forEach((indexItem, index) => {
if(indexItem === item.originalname) {
ShapefileRoute.shp(item, index, writeFilePromises, body, rootDirectory, outputId,);
}
})
} else {
ShapefileRoute.shp(item, outterIndex, writeFilePromises, body, rootDirectory, outputId,);
}
})
// Copy files from disk to database
ShapefileRoute.copyFiles(req, res, next, writeFilePromises, copyFilePromises, req.reserveSuperPg, () => {
ShapefileRoute.loadFiles(req, res, next, copyFilePromises, body, outputId)
});
})
});
}
// Process csv file
static csv(file: Express.Multer.File, index: number, writeFilePromises: Promise<any>[], body: any, rootDirectory: string, outputId: string) {
// Streaming file to pivotcsv
writeFilePromises.push(new Promise((resolve, reject) => {
// Get specification from body
let delimiter: string;
let spec: any;
let lrsColumns: string[] = [null, null, null, null, null, null];
body.layers.forEach((jsonItem, i) => {
if (jsonItem.name === file.originalname.split(".")[0]) {
delimiter = jsonItem.file_spec.delimiter;
spec = jsonItem
jsonItem.lrs_cols.forEach((lrsCol) => {
switch(lrsCol.lrs_type){
case "rec_id":
lrsColumns[0] = lrsCol.name;
break;
case "route_id":
lrsColumns[1] = lrsCol.name;
break;
case "f_meas":
lrsColumns[2] = lrsCol.name;
break;
case "t_meas":
lrsColumns[3] = lrsCol.name;
break;
case "b_date":
lrsColumns[4] = lrsCol.name;
break;
case "e_date":
lrsColumns[5] = lrsCol.name;
break;
}
})
}
});
// Pivot csv file
ShapefileRoute.pivotCsv(file.buffer, `${rootDirectory}/${outputId}_${index}`, index, delimiter, outputId, lrsColumns, (path) => {
console.log("got pivotCsv result");
spec.order = index;
resolve({
path: path,
spec: spec
});
}, reject);
}));
}
// Process shapefile
static shp(file: Express.Multer.File, index: number, writeFilePromises: Promise<any>[], body: any, rootDirectory: string, outputId: string) {
// Write file to disk and then call shp2csv to gennerate csv
writeFilePromises.push(new Promise((resolve, reject) => {
// Write shpefile to disk
fs.writeFile(`${rootDirectory}/shps/${file.originalname}`, file.buffer, (err) => {
// If it is .shp file, resolve it's path and spec
if(file.originalname.split(".")[1] === "shp") {
// Find spec of the shapefile from body
body.layers.forEach((jsonItem, i) => {
if (jsonItem.name === file.originalname.split(".")[0]) {
let recordColumn: string = null;
let routeIdColumn: string = null;
jsonItem.lrs_cols.forEach((lrsLayer) => {
if (lrsLayer.lrs_type === "rec_id") {
recordColumn = lrsLayer.name;
}
if (lrsLayer.lrs_type === "route_id") {
routeIdColumn = lrsLayer.name;
}
})
// Transfer shp to csv
ShapefileRoute.shp2csv(`${rootDirectory}/shps/${file.originalname}`, `${rootDirectory}/${outputId}_${index}`, index, outputId, recordColumn, routeIdColumn, (path, srs) => {
// Add coordinate system, geom column and index of this file to spec
jsonItem.file_spec.proj4 = srs;
jsonItem.file_spec.geom_col = "geom";
jsonItem.order = index;
// Return path and spec
resolve({
path: path,
spec: jsonItem
})
}, (err) => {
reject;
})
}
});
} else {
resolve(null);
}
})
}));
}
// Copy files to database
static copyFiles(req: Request, res: Response, next: Function, writeFilePromises: Promise<any>[], copyFilePromises: Promise<any>[], client: Client, callback: () => void) {
// Take all files generated by writefile processes
Promise.all(writeFilePromises)
.then((results) => {
// Remove null results. They are from .dbf .shx etc of shapefile.
const files: any = results.filter(arr => arr);
// Create promise array. This will be triggered after all files are written to database.
files.forEach((file) => {
copyFilePromises.push(new Promise((copyResolve, copyReject) => {
let query: string = `copy lbo.lbo_temp from '${file.path}' WITH NULL AS 'null';`;
// Create super user call
postgresQuery(client, query, (data) => {
copyResolve(file.spec);
}, copyReject);
}));
});
// Trigger upload query
callback()
})
.catch((err) => {
// Response as error if any file generating is wrong
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Something wrong when processing csv and/or shapefile.",
errorDetail: err.message,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
})
}
// Load layers in database
static loadFiles(req: Request, res: Response, next: Function, copyFilePromises: Promise<any>[], body: any, outputId: string) {
Promise.all(copyFilePromises)
.then((results) => {
// Resort all results by the order assigned when creating files
results.sort((a, b) => {
return a.order - b.order;
});
results.forEach((result) => {
delete result.order;
});
// Create JSON for load layer database request
let taskJson = body;
taskJson.layers = results;
let query: string = `select lbo.load_layers2(p_session_id := '${outputId}', p_layers := '${JSON.stringify(taskJson)}'::json)`;
postgresQuery(req.reservePg, query, (data) => {
// Get result
let result = data.rows[0].load_layers2.result;
// Return 4003 error if no result
if (!result) {
let payload: ErrorPayload = {
code: 4003,
errorMessage: "Load layers error.",
errorDetail: data.rows[0].load_layers2.error ? data.rows[0].load_layers2.error.message : "Load layers returns no result.",
hints: "Check error detail"
};
req.reservePayload = payload;
next();
return;
}
let payload: SuccessPayload = {
type: "string",
content: "Upload files done."
};
req.reservePayload = payload;
next();
}, (err) => {
req.reservePayload = err;
next();
});
})
.catch((err) => {
// Response as error if any file generating is wrong
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Something wrong when copy files to database.",
errorDetail: err,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
})
}
// Pivot csv process. Write output csv to disk and return path of the file.
static pivotCsv(buffer: Buffer, outputPath: string, inputIndex: number, delimiter: string, outputId: string, lrsColumns: string[], callback: (path: string) => void, errCallback: (err: Error) => void) {
let inputStream: Duplex = new Duplex();
// Define output stream
let output = fs.createWriteStream(outputPath, {flags: "a"});
// Callback when output stream is done
output.on("finish", () => {
console.log("output stream finish");
callback(outputPath);
});
// Define parser stream
let parser = csv.parse({
delimiter: delimiter
});
// Close output stream when parser stream is end
parser.on("end", () => {
console.log("parser stream end");
output.end();
});
// Write data when a chunck is parsed
let header = [null, null, null, null, null, null];
let attributesHeader = [];
let i = 0;
let datumIndex: boolean = true;
parser.on("data", (chunk) => {
console.log("parser received on chunck: ", i);
if (datumIndex) {
chunk.forEach((datum, index) => {
if (lrsColumns.includes(datum)) {
header[lrsColumns.indexOf(datum)] = index;
} else {
attributesHeader.push({
name: datum,
index: index
})
}
});
datumIndex = false;
} else {
i ++;
// let layer_id = ;
let rec_id = header[0] ? chunk[header[0]] : i;
let route_id = header[1] ? chunk[header[1]] : null;
let f_meas = header[2] ? chunk[header[2]] : null;
let t_meas = header[3] ? chunk[header[3]] : null;
let b_date = header[4] ? chunk[header[4]] : null;
let e_date = header[5] ? chunk[header[5]] : null;
let attributes = {};
attributesHeader.forEach((attribute) => {
attributes[attribute.name] = chunk[attribute.index];
});
let attributesOrdered = {};
Object.keys(attributes).sort().forEach((key) => {
attributesOrdered[key] = attributes[key];
});
let outputData = `${outputId}\t${inputIndex}\t${rec_id}\t${route_id}\tnull\t${f_meas}\t${t_meas}\t${b_date}\t${e_date}\tnull\t${JSON.stringify(attributesOrdered)}\n`;
output.write(outputData);
}
});
inputStream.push(buffer);
inputStream.push(null);
inputStream.pipe(parser);
}
// Write shp and transfer to database format. Return file path and projection.
static shp2csv(inputPath: string, outputPath: string, i: number, ouputId: string, recordColumn: string, routeIdColumn: string, callback: (path: string, prj: string) => void, errCallback: (err: Error) => void) {
let dataset = gdal.open(inputPath);
let layercount = dataset.layers.count();
let layer = dataset.layers.get(0);
let output = fs.createWriteStream(outputPath, {flags: "a"});
output.on("finish", () => {
callback(outputPath, layer.srs.toProj4());
});
layer.features.forEach((feature, featureId) => {
let geom;
let recordId: number = null;
let routeId: string = null;
try {
let geomWKB = feature.getGeometry().toWKB();
let geomWKBString = geomWKB.toString("hex");
geom = geomWKBString;
if (recordColumn) {
recordId = feature.fields.get(recordColumn);
}
if (routeIdColumn) {
routeId = feature.fields.get(routeIdColumn);
}
}
catch (err) {
console.log(err);
}
let attributes = {};
let attributesOrdered = {};
feature.fields.forEach((value, field) => {
if (field != recordColumn && field != routeIdColumn) {
attributes[field] = value;
}
});
Object.keys(attributes).sort().forEach((key) => {
attributesOrdered[key] = attributes[key];
});
output.write(`${ouputId}\t${i.toString()}\t${recordId ? recordId : (featureId + 1).toString()}\t${routeId}\tnull\tnull\tnull\tnull\tnull\t${geom}\t${JSON.stringify(attributesOrdered)}\n`);
});
output.end();
}
}
The browser retries some requests if the server doesn't send a response and the browser hits its timeout value. Each browser may be configured with its own timeout, but 2 minutes sounds like it's probably the browser timeout.
You can't control the browser's timeout from your server. Two minutes is just too long to ask it to wait. You need a different design that responds sooner and then communicates back the eventual result later when it's ready. Either client polling or server push with webSocket/socket.io.
For client polling, you could have the server respond immediately from your first request and return back a token (some unique string). Then, the client can ask the server for the response for that token every minute until the server eventually has the response. If the server doesn't yet have the response, it just immediately returns back a code that means no response yet. If so, the client sets a timer and tries again in a minute, sending the token each time so the server knows which request it is asking about.
For server push, the client creates a persistent webSocket or socket.io connection to the server. When the client makes it's long running request, the server just immediately returns the same type of token described above. Then, when the server is done with the request, it sends the token and the final data over the socket.io connection. The client is listening for incoming messages on that socket.io connection and will receive the final response there.