Parse large datasets and notify users efficiently - node.js

I have a huge dataset that increases by hundreds/thousands items for sale every few minutes. I will be parsing this data as it comes in and I'm looking for suggestions on how to efficiently match new incoming items with users who want the item.
Examples of how other have done this would mean the world to me. Even just clarification on what to search for online would be helpful. I'm having difficulty asking the right questions to search engines so I can learn how to solve this (usually I'm very good at this sort of thing).
Thank you fellow geniuses!
p.s. Hoping so solve this problem with nodejs
Inventory Dataset Example:
{
"anchor": 982179125,
"next_page": 1,
"next_tier": 0,
"num_matches": 47037,
"postings": [
{
"category": "SBIK",
"category_group": "SSSS",
"external_id": "4488716340",
"heading": "60cm Aluminum/Carbon Road Bike, Ultegra/105 equipped, 20-Spd",
"location": {
"accuracy": 8,
"city": "USA-SFO-SNF",
"country": "USA",
"county": "USA-CA-SAF",
"geolocation_status": 3,
"lat": "37.801041",
"locality": "USA-SFO-MRS",
"long": "-122.434151",
"metro": "USA-SFO",
"region": "USA-SFO-SAF",
"state": "USA-CA",
"zipcode": "USA-94123"
},
"price": 800.0
},
{
"category": "SBOT",
"category_group": "SSSS",
"external_id": "4488164318",
"heading": "Burley Bee Boat Trailer",
"location": {
"accuracy": 8,
"city": "USA-SFO-SNF",
"country": "USA",
"county": "USA-CA-SAF",
"geolocation_status": 3,
"lat": "37.7999",
"locality": "USA-SFO-MRS",
"long": "-122.4342",
"metro": "USA-SFO",
"region": "USA-SFO-SAF",
"state": "USA-CA",
"zipcode": "USA-94123"
},
"price": 150.0
},
{
"category": "SAUT",
"category_group": "SSSS",
"external_id": "4459983854",
"heading": "Car frame extra light castom made",
"location": {
"accuracy": 8,
"city": "USA-SFO-SNF",
"country": "USA",
"county": "USA-CA-SAF",
"geolocation_status": 3,
"lat": "37.7999",
"locality": "USA-SFO-MRS",
"long": "-122.4342",
"metro": "USA-SFO",
"region": "USA-SFO-SAF",
"state": "USA-CA",
"zipcode": "USA-94123"
},
"price": 300.0
},
"success": true,
"time_fetch": 9.4971656799316406,
"time_search": 42.315006256103516,
"time_taken": 183.84885787963867
}
My users are looking for:
{name:"tom",wants:"new car"}
{name:"john",wants:"Road Bike",price:[low:100,high:1000]}
{name:"becky",wants:"dolls"}
{name:"sam",wants:"Drobo",price:[low:0,high:500]}]
...
...
...

Related

How to get latest one record of each unique id in cosmosDB SQL API

Please comment if more explanation or Details required
I have a CosmosDb SQL API container with around 5 million+ document, where each document in container looks similar to below example.
{
"studentId": "ST009",
"name": "Satyam",
"Class": 8,
"Subject": "Mathematics",
"Address": "XYZ"
"contact": "9989126789"
"_ts": 1650372161
}
Each Student will have unique studentId, lets assume that the complete documents contains data of 300 students only(i.e. the 5 million+ document is having data of only those 300 students), I need to get only latest document of each student, I need a query that can give me only 300 records based on unique studentId.
Below is the example of the container documents
{
"studentId": "ST009",
"name": "Satyam",
"Class": 8,
"Subject": "Mathematics",
"Address": "XYZ"
"contact": "9989126789"
"_ts": 1650372161
}
{
"studentId": "ST008",
"name": "Siddharth",
"Class": 9,
"Subject": "Mathematics",
"Address": "XYZ"
"contact": "9989126789"
"_ts": 1650372152
}
{
"studentId": "ST010",
"name": "Debashis",
"Class": 8,
"Subject": "Science",
"Address": "XYZ"
"contact": "9989126789"
"_ts": 1650372167
}
{
"studentId": "ST009",
"name": "Satyam",
"Class": 8,
"Subject": "Science",
"Address": "XYZ"
"contact": "9989126789"
"_ts": 1650372169
}
...

Orders are not displayed at merchant console

I have developed an app in nodejs where I have integrated the Mastercard payment method and it was working fine till Friday. But now when I send money from MasterCard to the merchant account it gives the success response but there is nothing on the merchant console. here is my response
"paymentInfoList": [
{
"url": "https://test-gateway.mastercard.com/api/rest/version/57/merchant/merchant/order/order-1598868168410/transaction/transaction-1598868195864",
"mthd": "PUT",
"payload": {
"apiOperation": "CAPTURE",
"transaction": {
"amount": "1144",
"currency": "USD",
"reference": 1598868195864
}
},
"resbody": {
"authorizationResponse": {
"cardSecurityCodeError": "M",
"commercialCardIndicator": "1",
"date": "0831",
"financialNetworkCode": "MCC",
"posData": "1025104006600",
"posEntryMode": "812",
"processingCode": "000000",
"responseCode": "00",
"stan": "204684",
"time": "100313",
"transactionIdentifier": "0114GR"
},
"gatewayEntryPoint": "WEB_SERVICES_API",
"merchant": "GTB123456D00",
"order": {
"amount": 1144,
"certainty": "FINAL",
"chargeback": {
"amount": 0,
"currency": "USD"
},
"creationTime": "2020-08-31T10:03:13.631Z",
"currency": "USD",
"id": "order-1598868168410",
"lastUpdatedTime": "2020-08-31T10:03:33.156Z",
"merchantAmount": 1144,
"merchantCategoryCode": "5399",
"merchantCurrency": "USD",
"reference": "1598868168410",
"status": "CAPTURED",
"totalAuthorizedAmount": 1144,
"totalCapturedAmount": 1144,
"totalRefundedAmount": 0
},
"response": {
"acquirerCode": "00",
"acquirerMessage": "Approved",
"cardSecurityCode": {
"acquirerCode": "M",
"gatewayCode": "MATCH"
},
"gatewayCode": "APPROVED"
},
"result": "SUCCESS",
"sourceOfFunds": {
"provided": {
"card": {
"brand": "MASTERCARD",
"expiry": {
"month": "9",
"year": "21"
},
"fundingMethod": "CREDIT",
"issuer": "AFRILAND FIRST BANK",
"number": "512345xxxxxx0008",
"scheme": "MASTERCARD",
"storedOnFile": "NOT_STORED"
}
},
"type": "CARD"
},
"timeOfLastUpdate": "2020-08-31T10:03:33.156Z",
"timeOfRecord": "2020-08-31T10:03:33.064Z",
"transaction": {
"acquirer": {
"batch": 20200831,
"date": "0831",
"id": "GTB_S2I",
"merchantId": "mymerchant",
"settlementDate": "2020-08-31",
"timeZone": "+0100",
"transactionId": "0114GR"
},
"amount": 1144,
"authorizationCode": "010602",
"currency": "USD",
"id": "transaction-1598868195864",
"receipt": "024410204684",
"reference": "1598868195864",
"source": "INTERNET",
"stan": "205744",
"terminal": "GTBS2I04",
"type": "CAPTURE"
},
"version": "57"
}
}
]
now you can see the response is success but it doesnot display on merchant panel/console
It was the issue of the merchant panel/console. It didn't display the orders the same day but from the next day it was working fine and all the previous orders were there, and new orders were also showing on the console/panel So it was their server problem.

How to get Cities from Geo Data

I have some data about automobiles(particularly Transport Trucks). The Data includes Time , Latitude and Longitude of the vehicles among other things.
En Example Would Be:
Latitude Longitude UTC
19.250463 73.042856 2018-12-01 00:02:19
19.249853 73.043862 2018-12-01 00:04:12
19.24953 73.043923 2018-12-01 00:04:41
19.249527 73.043938 2018-12-01 00:09:12
19.249457 73.044172 2018-12-01 00:14:12
19.249484 73.044128 2018-12-01 00:19:12
There is a lot of data. The problem i am facing right now is to find the various 'Cities visited by this certain vehicle.
Here is a plot of Latitude and Longitude:
Here I have added the number of points within a 20KM radius of each point as a feature in the data. Then I have applied DBSCAN on these three features and represented the classes as colors.
As you can see, It fails to recognize the 'Intersection' points. Ideally i need a way to find corners, endpoints in this data.
Having a predefined list of cities along with their locations is not an option, because the general are of these trucks can vary greatly and then can visit small towns as well.
There are a couple of ways that you can solve this problem. The first, which would probably be the most performant, would be to build the inverse geocoding database yourself using information provided here: http://download.geonames.org/export/dump/
Of course this information would then have to be kept up to date which could be a pain.
If you're okay with a bit of latency, there are also third party services that do this for you. I run one of these third party services called https://geo-info.co/
You'd be able to solve this problem very easily with this API. For example
https://geo-info.co/19.250463,73.042856
Which would return the city along with some other info such as nearby locations.
{
"city": "Vishnunagar",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.267",
"longitude": "73.0715",
"nearby": [
{
"city": "Dombivali I.A.",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.267",
"longitude": "73.0715",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421203"
},
{
"city": "Dombivali",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.2167",
"longitude": "73.0833",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421201"
},
{
"city": "Ramnagar (Thane)",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.2167",
"longitude": "73.0833",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421201"
},
{
"city": "Thakurli",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.2167",
"longitude": "73.0833",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421201"
},
{
"city": "Tilaknagar (Thane)",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.2167",
"longitude": "73.0833",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421201"
},
{
"city": "Bhiwandi",
"community": "Bhiwandi",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.3002",
"longitude": "73.0588",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421308"
},
{
"city": "Khoni",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.3173",
"longitude": "73.0597",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421204"
},
{
"city": "Manpada",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.3173",
"longitude": "73.0597",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421204"
},
{
"city": "Nilje",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.3173",
"longitude": "73.0597",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421204"
},
{
"city": "Padle",
"community": "Kalyan",
"community2": "",
"country": "IN",
"county": "Thane",
"county2": "517",
"latitude": "19.3173",
"longitude": "73.0597",
"state": "Maharashtra",
"state2": "16",
"zipcode": "421204"
}
],
"state": "Maharashtra",
"state2": "16",
"zipcode": "421202"
}

Bot framework V4 how to add facebook receipt with discount

Is it possible to use the Discount option from Facebook Messenger's Receipt Template with Bot Framework V4?
Because I tried searching the samples, but they require string values, while the Facebook Template requires arrays.
Example:
"adjustments":[
{
"name":"New Customer Discount",
"amount":20
},
{
"name":"$10 Off Coupon",
"amount":10
}
],
Facebook Templates can be sent using ChannelData. Here is an example of a receipt template with adjustments:
await context.sendActivity({
text: 'Receipt',
channelData: {
"attachment":{
"type":"template",
"payload": {
"template_type": "receipt",
"recipient_name": "Stephane Crozatier",
"order_number": "12345678902",
"currency": "USD",
"payment_method": "Visa 2345",
"order_url": "http://petersapparel.parseapp.com/order?order_id=123456",
"timestamp": "1428444852",
"address": {
"street_1": "1 Hacker Way",
"street_2": "",
"city": "Menlo Park",
"postal_code": "94025",
"state": "CA",
"country": "US"
},
"summary": {
"subtotal": 75.00,
"shipping_cost": 4.95,
"total_tax": 6.19,
"total_cost": 56.14
},
"adjustments": [{
"name": "New Customer Discount",
"amount": 20
},
{
"name": "$10 Off Coupon",
"amount": 10
}
],
"elements": [{
"title": "Classic White T-Shirt",
"subtitle": "100% Soft and Luxurious Cotton",
"quantity": 2,
"price": 50,
"currency": "USD",
"image_url": "http://petersapparel.parseapp.com/img/whiteshirt.png"
},
{
"title": "Classic Gray T-Shirt",
"subtitle": "100% Soft and Luxurious Cotton",
"quantity": 1,
"price": 25,
"currency": "USD",
"image_url": "http://petersapparel.parseapp.com/img/grayshirt.png"
}
]
}
}
}
});

Core data relations between objects

Please suggest about how to create relation in Core Data in the following situation: I created a user object which contained 2 relations pointing to the same object (address) but relation names are homeAddress and workAddress. But on Xcode5 it gives some warning that the inverse relation is not reciprocal.
JSON:
{
"user": {
"user_id": "123123",
"email": "user#email.com",
"first_name": "Jean Luc",
"last_name": "Picard",
"home_address": {
"street": "DowningStreet",
"number": "11",
"city": "London",
"state": "GreaterLondon",
"country": "UnitedKingdom",
"zip": "SW1A2QAB",
"name": "Homesweethome",
"created_at": "1365526421456",
"location": {
"latitude": 3.1,
"longitude": 3.2
}
},
"work_address": {
"street": "AmphitheatreParkway",
"number": "1600",
"city": "MountainView",
"state": "California",
"country": "UnitedStates",
"zip": "94043",
"name": "WorkHarder",
"created_at": "1365526642123",
"location": {
"latitude": 3.1,
"longitude": 3.2
}
}
}
}
I want to avoid creating 2 separate address objects.

Resources