How to add utm tags to a wix form? - webhooks

I am currently attempting to include UTM tags in a webhook that I am sending from a wix form. However, upon sending the webhook, I am either receiving an empty response or receiving "google.co.il" source. I am expecting specifically Google Ads or email sources.
I have added the code that I am using in an attempt to make this function properly. Any assistance or guidance would be greatly appreciated..
export function wixForms1_wixFormSubmitted(event) {
var obj = {};
var i = 0;
for (i=0;i<event.fields.length;i++){
obj[event.fields[i].id] = event.fields[i].fieldValue;
}
//console.log(obj);
fetch( "https://hook.eu1.make.com/62tsltjotop6iwboufggguuxhhqrjaen", {
"method": "post",
"headers": {
"Content-Type": "application/json"
},
"body": JSON.stringify(obj)
} )
.then( (httpResponse) => {
if (httpResponse.ok) {
return httpResponse.json();
} else {
return Promise.reject("Fetch did not succeed");
}
} )
.then( (json) => console.log(json) )
.catch(err => console.log(err));
}
$w.onReady(function () {
const query = wixLocation.query;
if (query) {
$w('#refbox').value = JSON.stringify(query);
}
let params = wixLocation.query
console.log(params)
$w('#campaign').value = params.campaign_name
$w('#source').value = params.utm_source
});
import wixWindow from 'wix-window';
import wixLocation from 'wix-location';
});
import wixWindow from 'wix-window';
import wixLocation from 'wix-location';
import {local, session, memory} from 'wix-storage';
let referrer = wixWindow.referrer;
let previousPageURL;
$w.onReady(function () {
previousPageURL = session.getItem("page");
session.setItem("page", wixLocation.url);
});
$w.onReady(function () {
// Write your JavaScript here
const prvurl = wixLocation.url;
if (prvurl) {
$w('#prvurl').value = JSON.stringify(prvurl);
}
// To select an element by ID use: $w("#elementID")
// Click "Preview" to run your code
const query = wixWindow.referrer;
if (query) {
$w('#refbox').value = JSON.stringify(query);
}
}
);

Related

How to speed up Fetching Google Place and Photos

I currently have the following code to fetch matching Google Places according to a received query as shown below:
async function searchGoogleBusiness(req, res) {
let { name } = req.query;
const apiKey = process.env.API_KEY;
const searchUrl = `https://maps.googleapis.com/maps/api/place/textsearch/json?query=`;
try {
let { data } = await axios.get(`${searchUrl}${name}&key=${apiKey}`)
let { status, error_message, results } = data;
if (status === 'OK') {
let businessResults = [];
if ((results ?? []).length > 0) {
for (let business of results) {
let businessDetails = {
....
}
if ((business.photos ?? []).length > 0) {
let { width = 1200, height = 1200, photo_reference } = business.photos[0];
let photoUrl = `https://maps.googleapis.com/maps/api/place/photo?photoreference=${photo_reference}&sensor=false&maxheight=${height}&maxwidth=${width}&key=${apiKey}`
try {
let businessPhotoResponse = await axios.get(photoUrl, { responseType: 'arraybuffer' });
let imageBuffer = businessPhotoResponse.data;
let base64Image = Buffer.from(imageBuffer, 'binary').toString('base64');
businessDetails.photo = `data:${businessPhotoResponse.headers['content-type']};base64,${base64Image}`;
} catch (e) {
businessDetails.photo = business.icon;
}
} else {
businessDetails.photo = business.icon;
}
businessResults.push(businessDetails);
}
}
...//Omitted
}
...//Omitted
} catch (e) {
...//Omitted
}
}
As you can immediately notice, the function takes forever to return when the results are more than 5 and the reason is because I'm looping through each business to make another api call to fetch each photo.
I don't like this approach at all.
This idea of making another network call using photoReferences is really affecting my site speed and basically just makes my users angry.
Is there no way to automatically fetch the photo urls along just in the first request?

Creating and Capturing payment with paypal v2 node.js

I am trying to integrate PayPal server-side payment on a website with a new v2 since v1 is deprecated. Since I had to include quite a lot of code from their git, i will post here everything for a working example
In v1 i could do it really easily:
app.get('/create', function (req, res) {
//build PayPal payment request
let payReq = JSON.stringify({
'intent': 'sale',
'redirect_urls': {
'return_url': 'http://localhost:8081/process',
'cancel_url': 'http://localhost:8081/cancel'
},
'payer': {
'payment_method': 'paypal'
},
'transactions': [{
'amount': {
'total': '7.47',
'currency': 'USD'
},
'description': 'This is the payment transaction description.'
}]
});
paypal.payment.create(payReq, function (error, payment) {
if (error) {
console.error(error);
} else {
//capture HATEOAS links
let links = {};
payment.links.forEach(function (linkObj) {
links[linkObj.rel] = {
'href': linkObj.href,
'method': linkObj.method
};
})
//if redirect url present, redirect user
if (links.hasOwnProperty('approval_url')) {
res.redirect(links['approval_url'].href);
} else {
console.error('no redirect URI present');
}
}
});
});
app.get('/process', function (req, res) {
let paymentId = req.query.paymentId;
let payerId = {'payer_id': req.query.PayerID};
paypal.payment.execute(paymentId, payerId, function (error, payment) {
if (error) {
console.error(error);
} else {
if (payment.state == 'approved') {
const payerCountry = payment.payer.payer_info.country_code;
const total = payment.transactions[0].amount.total;
const currency = payment.transactions[0].amount.currency;
savePayment(payerCountry, total, currency);
res.send('payment completed successfully ' + cnt++);
} else {
res.send('payment not successful');
}
}
});
});
The create endpoint basically creates the order, the paypal API returns hateos links and the controller says the browser should redirect to that link. Once redirected , if user approves payment on paypal site he is redirected to on of
'redirect_urls': {
'return_url': 'http://localhost:8081/process',
'cancel_url': 'http://localhost:8081/cancel'
},
process endpoints retrieves PAYMENT ID and PAYER ID from query and captures the order - allowing me to do whatever i want to do ( e.g save payment in db ) in callback.
Now v2 seems like huge mess:
Following devs guide i have created
Paypal button (copy paste ):
import React, {useEffect} from "react";
import {PayPalButtons, PayPalScriptProvider, usePayPalScriptReducer} from "#paypal/react-paypal-js";
// This values are the props in the UI
const amount = "2";
const currency = "USD";
const style = {"layout": "vertical"};
const ButtonWrapper = ({currency, showSpinner}) => {
// usePayPalScriptReducer can be use only inside children of PayPalScriptProviders
// This is the main reason to wrap the PayPalButtons in a new component
const [{options, isPending}, dispatch] = usePayPalScriptReducer();
useEffect(() => {
dispatch({
type: "resetOptions",
value: {
...options,
currency: currency,
},
});
}, [currency, showSpinner]);
const createOrder = (data, actions) => {
console.log("create")
return fetch('http://localhost:8081/create', {
method: 'post'
}).then(function (res) {
return res.json();
}).then(function (orderData) {
console.log(orderData);
window.location = orderData.redirect;
});
}
// Call your server to finalize the transaction
const onApprove = (data, actions) => {
console.log("approve")
return fetch('/process', {
method: 'post'
}).then(function (res) {
return res.json();
}).then(function (orderData) {
// Three cases to handle:
// (1) Recoverable INSTRUMENT_DECLINED -> call actions.restart()
// (2) Other non-recoverable errors -> Show a failure message
// (3) Successful transaction -> Show confirmation or thank you
// This example reads a v2/checkout/orders capture response, propagated from the server
// You could use a different API or structure for your 'orderData'
var errorDetail = Array.isArray(orderData.details) && orderData.details[0];
if (errorDetail && errorDetail.issue === 'INSTRUMENT_DECLINED') {
return actions.restart(); // Recoverable state, per:
// https://developer.paypal.com/docs/checkout/integration-features/funding-failure/
}
if (errorDetail) {
var msg = 'Sorry, your transaction could not be processed.';
if (errorDetail.description) msg += '\n\n' + errorDetail.description;
if (orderData.debug_id) msg += ' (' + orderData.debug_id + ')';
return alert(msg); // Show a failure message (try to avoid alerts in production environments)
}
// Successful capture! For demo purposes:
console.log('Capture result', orderData, JSON.stringify(orderData, null, 2));
var transaction = orderData.purchase_units[0].payments.captures[0];
alert('Transaction ' + transaction.status + ': ' + transaction.id + '\n\nSee console for all available details');
});
}
return (<>
{(showSpinner && isPending) && <div className="spinner"/>}
<PayPalButtons
style={style}
disabled={false}
forceReRender={[amount, currency, style]}
fundingSource={undefined}
createOrder={(data, actions) => createOrder(data, actions)}
onApprove={(data, actions) => onApprove(data, actions)}
/>
</>
);
}
export default function PayPalButton() {
return (
<div style={{ maxWidth: "750px", minHeight: "200px" }}>
<PayPalScriptProvider
options={{
"client-id": "test",
components: "buttons",
currency: "USD"
}}
>
<ButtonWrapper
currency={currency}
showSpinner={false}
/>
</PayPalScriptProvider>
</div>
);
}
And then followed the flow from paypal's github example
created their HttpClient
const checkoutNodeJssdk = require('#paypal/checkout-server-sdk');
/**
* Returns PayPal HTTP client instance with environment which has access
* credentials context. This can be used invoke PayPal API's provided the
* credentials have the access to do so.
*/
function client() {
return new checkoutNodeJssdk.core.PayPalHttpClient(environment());
}
/**
* Setting up and Returns PayPal SDK environment with PayPal Access credentials.
* For demo purpose, we are using SandboxEnvironment. In production this will be
* LiveEnvironment.
*/
function environment() {
let clientId = process.env.PAYPAL_CLIENT_ID || '<clientId>';
let clientSecret = process.env.PAYPAL_CLIENT_SECRET || '<secret>';
if (process.env.NODE_ENV === 'production') {
return new checkoutNodeJssdk.core.LiveEnvironment(clientId, clientSecret);
}
return new checkoutNodeJssdk.core.SandboxEnvironment(clientId, clientSecret);
}
async function prettyPrint(jsonData, pre=""){
let pretty = "";
function capitalize(string) {
return string.charAt(0).toUpperCase() + string.slice(1).toLowerCase();
}
for (let key in jsonData){
if (jsonData.hasOwnProperty(key)){
if (isNaN(key))
pretty += pre + capitalize(key) + ": ";
else
pretty += pre + (parseInt(key) + 1) + ": ";
if (typeof jsonData[key] === "object"){
pretty += "\n";
pretty += await prettyPrint(jsonData[key], pre + "\t");
}
else {
pretty += jsonData[key] + "\n";
}
}
}
return pretty;
}
module.exports = {client: client, prettyPrint:prettyPrint};
created their CreateOrder:
/**
* PayPal SDK dependency
*/
const checkoutNodeJssdk = require('#paypal/checkout-server-sdk');
/**
* PayPal HTTP client dependency
*/
const payPalClient = require('./PayPalClient');
/**
* Setting up the JSON request body for creating the Order. The Intent in the
* request body should be set as "CAPTURE" for capture intent flow.
*
*/
function buildRequestBody() {
return {
"intent": "CAPTURE",
"application_context": {
"return_url": "http://localhost:8081/process",
"cancel_url": "https://www.example.com",
"locale": "en-US",
"landing_page": "BILLING",
"user_action": "CONTINUE"
},
"purchase_units": [
{
"soft_descriptor": "Donation",
"amount": {
"currency_code": "USD",
"value": "220.00"
}
}
]
};
}
/**
* This is the sample function which can be sued to create an order. It uses the
* JSON body returned by buildRequestBody() to create an new Order.
*/
async function createOrder(debug=false) {
try {
const request = new checkoutNodeJssdk.orders.OrdersCreateRequest();
request.headers["prefer"] = "return=representation";
request.requestBody(buildRequestBody());
const response = await payPalClient.client().execute(request);
if (debug){
console.log("Status Code: " + response.statusCode);
console.log("Status: " + response.result.status);
console.log("Order ID: " + response.result.id);
console.log("Intent: " + response.result.intent);
console.log("Links: ");
response.result.links.forEach((item, index) => {
let rel = item.rel;
let href = item.href;
let method = item.method;
let message = `\t${rel}: ${href}\tCall Type: ${method}`;
console.log(message);
});
console.log(`Gross Amount: ${response.result.purchase_units[0].amount.currency_code} ${response.result.purchase_units[0].amount.value}`);
// To toggle print the whole body comment/uncomment the below line
console.log(JSON.stringify(response.result, null, 4));
}
return response;
}
catch (e) {
console.log(e)
}
}
/**
* This is the driver function which invokes the createOrder function to create
* an sample order.
*/
if (require.main === module){
(async() => await createOrder(true))();
}
/**
* Exports the Create Order function. If needed this can be invoked from the
* order modules to execute the end to flow like create order, retrieve, capture
* and refund(Optional)
*/
module.exports = {createOrder:createOrder};
And endpoints:
const createUsersOrder = async (res) => {
try {
let response = await createOrder();
console.log("Creating Order...");
let orderId = "";
if (response.statusCode === 201){
console.log("Created Successfully");
orderId = response.result.id;
console.log("Links:");
response.result.links.forEach((item, index) => {
let rel = item.rel;
let href = item.href;
let method = item.method;
let message = `\t${rel}: ${href}\tCall Type: ${method}`;
console.log(message);
});
let links = {};
response.result.links.forEach(function (linkObj) {
links[linkObj.rel] = {
'href': linkObj.href,
'method': linkObj.method
};
})
//if redirect url present, redirect user
if (links.hasOwnProperty('approve')) {
var returnObj = {redirect : links['approve'].href}
console.log("Returning " + returnObj)
res.send(returnObj);
} else {
console.error('no redirect URI present');
}
}
console.log("Copy approve link and paste it in browser. Login with buyer account and follow the instructions.\nOnce approved hit enter...");
return
} catch (error) {
console.log('There was an error: ', error);
}
};
app.post("/create", function(req,res) {
createUsersOrder(res);
})
Here, this is called when button is clicked, as "createOrder" method is called. I create order just like in the v1 code, and then redirect browser to the url. However when the user approves transaction on paypal, and thus is being redirected to one of
"application_context": {
"return_url": "http://localhost:8081/process",
"cancel_url": "http://localhost:8081/cancel",
"locale": "en-US",
"landing_page": "BILLING",
"user_action": "CONTINUE"
},
return url ( /process route for success ), the request DOES NOT contain payment_id, only PAYER_ID. But the payment_id ( or order_id in v2 ) is needed to capture the order.
Since i have found literally zero blogs, tutorials, guide for v2 ( only millions for v1) i am confused where to get the order id. Do i really need to save it in DB after i create the order? OR is there any other trick?
Also, the button contains onApprove method, but after creating order, the browser is redirected to paypal, and the paypal redirects user to http://localhost:8081/process endpoint - thus the onApprove method is never invoked and useless (?).
This whole flow of v2 is really confusing, is there something i am missing?
Thanks for help
With your v2 code, do not use any redirects. At all. Your button should call 2 endpoints on your server. These two endpoints should (respectively) do the API operations of creating and capturing the order, and return the JSON result in each case (the capture route can do any server-side operations like storing the transaction result in the database before forwarding the JSON result to the client caller, since the client needs to handle any capture error situations as well as showing a success message). You can find a full stack node.js example in the PayPal integration guide, but it's fine to keep your #paypal/react-paypal-js code pretty much as-is for the front end.

NodeJS GC function cannot be initialized

Trying out my first NodeJS cloud function so far unsuccessfully despite working fine VS code. Getting following error
Function cannot be initialized. Error: function terminated.
Looking through the logs I see some potential issues
Detailed stack trace: ReferenceError: supabase_public_url is not defined
Provided module can't be loaded (doesn't specify)
Thoughts: Am I doing it wrong with the secret manager and using the pub/sub incorrect?
My Code index.js
import { createClient } from '#supabase/supabase-js'
import sgMail from "#sendgrid/mail"
import { SecretManagerServiceClient } from '#google-cloud/secret-manager'
//activate cloud secret manager
const client = new SecretManagerServiceClient()
const supabaseUrl = client.accessSecretVersion(supabase_public_url)
const supabaseKey = client.accessSecretVersion(supabase_service_key)
const sendgridKey = client.accessSecretVersion(sendgrid_service_key)
sgMail.setApiKey(sendgridKey)
const supabase = createClient(supabaseUrl, supabaseKey)
// get data for supabase where notifications coins are true
const supabaseNotifications = async() => {
let { data, error } = await supabase
.from('xxx')
.select('*, xxx!inner(coin, xx, combo_change, combo_signal, combo_prev_signal), xxx!inner(email)')
.eq('crypto_signals.combo_change', true)
if(error) {
console.error(error)
return
}
return data
}
//create an array of user emails from supabase data
const userEmail = (data) => {
try {
const emailList = []
for (let i of data) {
if (emailList.includes(i.profiles.email) != true) {
emailList.push(i.profiles.email)
} else {}
}
return emailList
}
catch(e) {
console.log(e)
}
}
// function to take email list and supabase data to generate emails to users
const sendEmail = (e, data ) => {
try {
for (let i of e) {
const signalList = []
for (let x of data) {
if(i == x.profiles.email) {
signalList.push(x)
} else {}
}
// create msg and send from my email to the user
const msg = {
to: i,
from:"xxxx",
subject: "Coin notification alert from CryptoOwl",
text: "One or more of you coins have a new signal",
html: signalList.toString()
}
sgMail.send(msg)
console.log(i)
}
}
catch(e) {
console.log(e)
}
}
// main function combines all 3 functions (supabase is await)
async function main(){
let supabaseData = await supabaseNotifications();
let supabaseEmails = userEmail(supabaseData);
let sendgridEmails = sendEmail(supabaseEmails, supabaseData);
}
exports.sendgridNotifications = (event, context) => {
main()
};
my package.json with type module to use import above
{
"type":"module",
"dependencies":{
"#sendgrid/mail":"^7.6.1",
"#supabase/supabase-js":"1.30.0",
"#google-cloud/secret-manager": "^3.11.0"
}
}
I'm not at all versed in Google Secret Manager but a rapid look at the Node.js library documentation shows (if I'm not mistaking) that accessSecretVersion() is an asynchronous method.
As a matter of facts, we find in the doc examples like the following one:
async function accessSecretVersion() {
const [version] = await client.accessSecretVersion({
name: name,
});
// Extract the payload as a string.
const payload = version.payload.data.toString();
// WARNING: Do not print the secret in a production environment - this
// snippet is showing how to access the secret material.
console.info(`Payload: ${payload}`);
}
See https://cloud.google.com/secret-manager/docs/samples/secretmanager-access-secret-version#secretmanager_access_secret_version-nodejs

Bot Framework v4 Node.js Location

I'm in the process of designing a chat bot and trying to find some Node.js sample code and/or documentation on how to implement the Azure Maps service as part of Bot Framework V4. There are many examples of how this is accomplished in V3, but there seems to be no examples of a V4 solution for Node.js. I'm looking to create a step in my botbuilder-dialog flow that would launch a simple "where do we ship it too" location dialog that would guide the user through the dialog and store the address results as part of that users profile. Any help or advice on this would be appreciated.
Yes, this is doable. I created a class (probably overkill, but oh well) in which I make my API call, with my supplied parameters, to get the map. I decided to use Azure Maps (vs Bing Maps) only because I was curious in how it differed. There isn't any reason you couldn't do this with Bing Maps, as well.
In the bot, I am using a component dialog because of how I have the rest of my bot designed. When the dialog ends, it will fall off the stack and return to the parent dialog.
In my scenario, the bot presents the user with a couple choices. "Send me a map" generates a map and sends it in an activity to the client/user. Anything else sends the user onward ending the dialog.
You will need to decide how you are getting the user's location. I developed this with Web Chat in mind, so I am getting the geolocation from the browser and returning it to the bot to be used when getMap() is called.
const { ActivityTypes, InputHints } = require('botbuilder');
const fetch = require('node-fetch');
class MapHelper {
async getMap(context, latitude, longitude) {
var requestOptions = {
method: 'GET',
headers: {
'Content-Type': 'application/json'
},
redirect: 'follow'
};
const result = await fetch(`https://atlas.microsoft.com/map/static/png?subscription-key=${ process.env.AZURE_MAPS_KEY }&api-version=1.0&layer=basic&zoom=13&center=${ longitude },${ latitude }&language=en-US&pins=default|al.67|la12 3|lc000000||'You!'${ longitude } ${ latitude }&format=png`, requestOptions)
.then(response => response.arrayBuffer())
.then(async result => {
const bufferedData = Buffer.from(result, 'binary');
const base64 = bufferedData.toString('base64');
const reply = { type: ActivityTypes.Message };
const attachment = {
contentType: 'image/png',
contentUrl: `data:image/png;base64,${ base64 }`
};
reply.attachments = [attachment];
await context.sendActivity(reply, null, InputHints.IgnoringInput);
})
.catch(error => {
if (error) throw new Error(error);
});
return result;
};
};
module.exports.MapHelper = MapHelper;
const { ChoicePrompt, ChoiceFactory, ComponentDialog, ListStyle, WaterfallDialog } = require('botbuilder-dialogs');
const { MapHelper } = require('./mapHelper');
const CONFIRM_LOCALE_DIALOG = 'confirmLocaleDialog';
const CHOICE_PROMPT = 'confirmPrompt';
class ConfirmLocaleDialog extends ComponentDialog {
constructor() {
super(CONFIRM_LOCALE_DIALOG);
this.addDialog(new ChoicePrompt(CHOICE_PROMPT))
.addDialog(new WaterfallDialog(CONFIRM_LOCALE_DIALOG, [
this.askLocationStep.bind(this),
this.getMapStep.bind(this)
]));
this.initialDialogId = CONFIRM_LOCALE_DIALOG;
}
async askLocationStep(stepContext) {
const choices = ['Send me a map', "I'll have none of this nonsense!"];
return await stepContext.prompt(CHOICE_PROMPT, {
prompt: 'Good sir, may I pinpoint you on a map?',
choices: ChoiceFactory.toChoices(choices),
style: ListStyle.suggestedAction
});
}
async getMapStep(stepContext) {
const { context, context: { activity } } = stepContext;
const text = activity.text.toLowerCase();
if (text === 'send me a map') {
const { latitude, longitude } = activity.channelData;
const mapHelper = new MapHelper();
await mapHelper.getMap(context, latitude, longitude);
const message = 'Thanks for sharing!';
await stepContext.context.sendActivity(message);
return await stepContext.endDialog();
} else {
await stepContext.context.sendActivity('No map for you!');
return await stepContext.endDialog();
}
}
}
module.exports.ConfirmLocaleDialog = ConfirmLocaleDialog;
module.exports.CONFIRM_LOCALE_DIALOG = CONFIRM_LOCALE_DIALOG;
Hope of help!
---- EDIT ----
Per request, location data can be obtained from the browser using the below method. It is, of course, dependent on the user granting access to location data.
navigator.geolocation.getCurrentPosition( async (position) => {
const { latitude, longitude } = position.coords;
// Do something with the data;
console.log(latitude, longitude)
})

Node.js long process ran twice

I have a Node.js restful API built in express.js framework. It is usually hosted by pm2.
One of the services has very long process. When front end called the service, the process started up. Since there is an error in database, the process won't be done properly and the error would be caught. However, before the process reached the error, another exactly same process started with same parameters. So in the meantime, two processes were both running while one was ahead of the other. After a long time, the first process reached error point and returned error. Then the second one returned exactly the same thing.
I checked front end Network and noticed there was actually only one request sent. Where did the second request come from?
Edit 1:
The whole process is: first process sends query to db -> long time wait -> second process starts up -> second process sends query to db -> long time wait -> first process receives db response -> long time wait -> second process receives db response
Edit 2:
The code of the service is as follow:
import { Express, Request, Response } from "express";
import * as multer from "multer";
import * as fs from "fs";
import { Readable, Duplex } from "stream";
import * as uid from "uid";
import { Client } from "pg";
import * as gdal from "gdal";
import * as csv from "csv";
import { SuccessPayload, ErrorPayload } from "../helpers/response";
import { postgresQuery } from "../helpers/database";
import Config from "../config";
export default class ShapefileRoute {
constructor(app: Express) {
// Upload a shapefile
/**
* #swagger
* /shapefile:
* post:
* description: Returns the homepage
* responses:
* 200:
*/
app.post("/shapefile", (req: Request, res: Response, next: Function): void => {
// Create instance of multer
const multerInstance = multer().array("files");
multerInstance(req, res, (err: Error) => {
if (err) {
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Multer upload file error.",
errorDetail: err.message,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
return;
}
// Extract files
let files: any = req.files;
// Extract body
let body: any = JSON.parse(req.body.filesInfo);
// Other params
let writeFilePromises: Promise<any>[] = [];
let copyFilePromises: Promise<any>[] = [];
let rootDirectory: string = Config.uploadRoot;
let outputId: string = uid(4);
// Reset index of those files
let namesIndex: string[] = [];
files.forEach((item: Express.Multer.File, index: number) => {
if(item.originalname.split(".")[1] === "csv" || item.originalname.split(".")[1] === "txt" || item.originalname.split(".")[1] === "shp") {
namesIndex.push(item.originalname);
}
})
// Process and write all files to disk
files.forEach((item: Express.Multer.File, outterIndex: number) => {
if(item.originalname.split(".")[1] === "csv" || item.originalname.split(".")[1] === "txt") {
namesIndex.forEach((indexItem, index) => {
if(indexItem === item.originalname) {
ShapefileRoute.csv(item, index, writeFilePromises, body, rootDirectory, outputId,);
}
})
} else if (item.originalname.split(".")[1] === "shp") {
namesIndex.forEach((indexItem, index) => {
if(indexItem === item.originalname) {
ShapefileRoute.shp(item, index, writeFilePromises, body, rootDirectory, outputId,);
}
})
} else {
ShapefileRoute.shp(item, outterIndex, writeFilePromises, body, rootDirectory, outputId,);
}
})
// Copy files from disk to database
ShapefileRoute.copyFiles(req, res, next, writeFilePromises, copyFilePromises, req.reserveSuperPg, () => {
ShapefileRoute.loadFiles(req, res, next, copyFilePromises, body, outputId)
});
})
});
}
// Process csv file
static csv(file: Express.Multer.File, index: number, writeFilePromises: Promise<any>[], body: any, rootDirectory: string, outputId: string) {
// Streaming file to pivotcsv
writeFilePromises.push(new Promise((resolve, reject) => {
// Get specification from body
let delimiter: string;
let spec: any;
let lrsColumns: string[] = [null, null, null, null, null, null];
body.layers.forEach((jsonItem, i) => {
if (jsonItem.name === file.originalname.split(".")[0]) {
delimiter = jsonItem.file_spec.delimiter;
spec = jsonItem
jsonItem.lrs_cols.forEach((lrsCol) => {
switch(lrsCol.lrs_type){
case "rec_id":
lrsColumns[0] = lrsCol.name;
break;
case "route_id":
lrsColumns[1] = lrsCol.name;
break;
case "f_meas":
lrsColumns[2] = lrsCol.name;
break;
case "t_meas":
lrsColumns[3] = lrsCol.name;
break;
case "b_date":
lrsColumns[4] = lrsCol.name;
break;
case "e_date":
lrsColumns[5] = lrsCol.name;
break;
}
})
}
});
// Pivot csv file
ShapefileRoute.pivotCsv(file.buffer, `${rootDirectory}/${outputId}_${index}`, index, delimiter, outputId, lrsColumns, (path) => {
console.log("got pivotCsv result");
spec.order = index;
resolve({
path: path,
spec: spec
});
}, reject);
}));
}
// Process shapefile
static shp(file: Express.Multer.File, index: number, writeFilePromises: Promise<any>[], body: any, rootDirectory: string, outputId: string) {
// Write file to disk and then call shp2csv to gennerate csv
writeFilePromises.push(new Promise((resolve, reject) => {
// Write shpefile to disk
fs.writeFile(`${rootDirectory}/shps/${file.originalname}`, file.buffer, (err) => {
// If it is .shp file, resolve it's path and spec
if(file.originalname.split(".")[1] === "shp") {
// Find spec of the shapefile from body
body.layers.forEach((jsonItem, i) => {
if (jsonItem.name === file.originalname.split(".")[0]) {
let recordColumn: string = null;
let routeIdColumn: string = null;
jsonItem.lrs_cols.forEach((lrsLayer) => {
if (lrsLayer.lrs_type === "rec_id") {
recordColumn = lrsLayer.name;
}
if (lrsLayer.lrs_type === "route_id") {
routeIdColumn = lrsLayer.name;
}
})
// Transfer shp to csv
ShapefileRoute.shp2csv(`${rootDirectory}/shps/${file.originalname}`, `${rootDirectory}/${outputId}_${index}`, index, outputId, recordColumn, routeIdColumn, (path, srs) => {
// Add coordinate system, geom column and index of this file to spec
jsonItem.file_spec.proj4 = srs;
jsonItem.file_spec.geom_col = "geom";
jsonItem.order = index;
// Return path and spec
resolve({
path: path,
spec: jsonItem
})
}, (err) => {
reject;
})
}
});
} else {
resolve(null);
}
})
}));
}
// Copy files to database
static copyFiles(req: Request, res: Response, next: Function, writeFilePromises: Promise<any>[], copyFilePromises: Promise<any>[], client: Client, callback: () => void) {
// Take all files generated by writefile processes
Promise.all(writeFilePromises)
.then((results) => {
// Remove null results. They are from .dbf .shx etc of shapefile.
const files: any = results.filter(arr => arr);
// Create promise array. This will be triggered after all files are written to database.
files.forEach((file) => {
copyFilePromises.push(new Promise((copyResolve, copyReject) => {
let query: string = `copy lbo.lbo_temp from '${file.path}' WITH NULL AS 'null';`;
// Create super user call
postgresQuery(client, query, (data) => {
copyResolve(file.spec);
}, copyReject);
}));
});
// Trigger upload query
callback()
})
.catch((err) => {
// Response as error if any file generating is wrong
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Something wrong when processing csv and/or shapefile.",
errorDetail: err.message,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
})
}
// Load layers in database
static loadFiles(req: Request, res: Response, next: Function, copyFilePromises: Promise<any>[], body: any, outputId: string) {
Promise.all(copyFilePromises)
.then((results) => {
// Resort all results by the order assigned when creating files
results.sort((a, b) => {
return a.order - b.order;
});
results.forEach((result) => {
delete result.order;
});
// Create JSON for load layer database request
let taskJson = body;
taskJson.layers = results;
let query: string = `select lbo.load_layers2(p_session_id := '${outputId}', p_layers := '${JSON.stringify(taskJson)}'::json)`;
postgresQuery(req.reservePg, query, (data) => {
// Get result
let result = data.rows[0].load_layers2.result;
// Return 4003 error if no result
if (!result) {
let payload: ErrorPayload = {
code: 4003,
errorMessage: "Load layers error.",
errorDetail: data.rows[0].load_layers2.error ? data.rows[0].load_layers2.error.message : "Load layers returns no result.",
hints: "Check error detail"
};
req.reservePayload = payload;
next();
return;
}
let payload: SuccessPayload = {
type: "string",
content: "Upload files done."
};
req.reservePayload = payload;
next();
}, (err) => {
req.reservePayload = err;
next();
});
})
.catch((err) => {
// Response as error if any file generating is wrong
let payload: ErrorPayload = {
code: 4004,
errorMessage: "Something wrong when copy files to database.",
errorDetail: err,
hints: "Check error detail"
};
req.reservePayload = payload;
next();
})
}
// Pivot csv process. Write output csv to disk and return path of the file.
static pivotCsv(buffer: Buffer, outputPath: string, inputIndex: number, delimiter: string, outputId: string, lrsColumns: string[], callback: (path: string) => void, errCallback: (err: Error) => void) {
let inputStream: Duplex = new Duplex();
// Define output stream
let output = fs.createWriteStream(outputPath, {flags: "a"});
// Callback when output stream is done
output.on("finish", () => {
console.log("output stream finish");
callback(outputPath);
});
// Define parser stream
let parser = csv.parse({
delimiter: delimiter
});
// Close output stream when parser stream is end
parser.on("end", () => {
console.log("parser stream end");
output.end();
});
// Write data when a chunck is parsed
let header = [null, null, null, null, null, null];
let attributesHeader = [];
let i = 0;
let datumIndex: boolean = true;
parser.on("data", (chunk) => {
console.log("parser received on chunck: ", i);
if (datumIndex) {
chunk.forEach((datum, index) => {
if (lrsColumns.includes(datum)) {
header[lrsColumns.indexOf(datum)] = index;
} else {
attributesHeader.push({
name: datum,
index: index
})
}
});
datumIndex = false;
} else {
i ++;
// let layer_id = ;
let rec_id = header[0] ? chunk[header[0]] : i;
let route_id = header[1] ? chunk[header[1]] : null;
let f_meas = header[2] ? chunk[header[2]] : null;
let t_meas = header[3] ? chunk[header[3]] : null;
let b_date = header[4] ? chunk[header[4]] : null;
let e_date = header[5] ? chunk[header[5]] : null;
let attributes = {};
attributesHeader.forEach((attribute) => {
attributes[attribute.name] = chunk[attribute.index];
});
let attributesOrdered = {};
Object.keys(attributes).sort().forEach((key) => {
attributesOrdered[key] = attributes[key];
});
let outputData = `${outputId}\t${inputIndex}\t${rec_id}\t${route_id}\tnull\t${f_meas}\t${t_meas}\t${b_date}\t${e_date}\tnull\t${JSON.stringify(attributesOrdered)}\n`;
output.write(outputData);
}
});
inputStream.push(buffer);
inputStream.push(null);
inputStream.pipe(parser);
}
// Write shp and transfer to database format. Return file path and projection.
static shp2csv(inputPath: string, outputPath: string, i: number, ouputId: string, recordColumn: string, routeIdColumn: string, callback: (path: string, prj: string) => void, errCallback: (err: Error) => void) {
let dataset = gdal.open(inputPath);
let layercount = dataset.layers.count();
let layer = dataset.layers.get(0);
let output = fs.createWriteStream(outputPath, {flags: "a"});
output.on("finish", () => {
callback(outputPath, layer.srs.toProj4());
});
layer.features.forEach((feature, featureId) => {
let geom;
let recordId: number = null;
let routeId: string = null;
try {
let geomWKB = feature.getGeometry().toWKB();
let geomWKBString = geomWKB.toString("hex");
geom = geomWKBString;
if (recordColumn) {
recordId = feature.fields.get(recordColumn);
}
if (routeIdColumn) {
routeId = feature.fields.get(routeIdColumn);
}
}
catch (err) {
console.log(err);
}
let attributes = {};
let attributesOrdered = {};
feature.fields.forEach((value, field) => {
if (field != recordColumn && field != routeIdColumn) {
attributes[field] = value;
}
});
Object.keys(attributes).sort().forEach((key) => {
attributesOrdered[key] = attributes[key];
});
output.write(`${ouputId}\t${i.toString()}\t${recordId ? recordId : (featureId + 1).toString()}\t${routeId}\tnull\tnull\tnull\tnull\tnull\t${geom}\t${JSON.stringify(attributesOrdered)}\n`);
});
output.end();
}
}
The browser retries some requests if the server doesn't send a response and the browser hits its timeout value. Each browser may be configured with its own timeout, but 2 minutes sounds like it's probably the browser timeout.
You can't control the browser's timeout from your server. Two minutes is just too long to ask it to wait. You need a different design that responds sooner and then communicates back the eventual result later when it's ready. Either client polling or server push with webSocket/socket.io.
For client polling, you could have the server respond immediately from your first request and return back a token (some unique string). Then, the client can ask the server for the response for that token every minute until the server eventually has the response. If the server doesn't yet have the response, it just immediately returns back a code that means no response yet. If so, the client sets a timer and tries again in a minute, sending the token each time so the server knows which request it is asking about.
For server push, the client creates a persistent webSocket or socket.io connection to the server. When the client makes it's long running request, the server just immediately returns the same type of token described above. Then, when the server is done with the request, it sends the token and the final data over the socket.io connection. The client is listening for incoming messages on that socket.io connection and will receive the final response there.

Resources