How to check if condition in foreach using logic app in efficient way? - azure

Question

I have two array variables in for each i compare values one by one abd process further but issue is it is taking to much time for executing when I have 1000 of records.
Variable_1 having around 10 records
Variable_2 which is output if HTTP will have around 1000+ records
Variable_1 =
[
{
"Name": "AAA",
"OwnerId": "121a",
},
{
"Name": "BBB",
"OwnerId": "123b",
},
...
]
and
Variable_2 =
[
{
"Demo_Name": "AAA"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
}
....
]
I have used 2 for each one is forVariable_1and another is forVariable_2``.
and comparing Variable_1.Name == Variable_2.Demo_Name one by one.
so when Variable_2 is having more than 1000+ records it took around a hour to check condition one by one
any solution for this to do it in a fast way?
outer loop is taking around 2h
If I try concurrent execution then increment variable set wrong
values

Answer 1

I tried doing this with Azure Storage Tables as a source for the result, just to see how fast I could iterate through Var2 with 1465 records and Var1 with 9 records. I do it twice here to see just to figure out which way was easiest to do it in. The result here was XXXX matches, the values stored in an Azure Storage Table which I would fetch after the loops were done. I have added a Terminate Action which you could move them before the action For each just to see use one of them for each loop instead of both.
For the record, not sure if this is exactly what you want. Also with this way, if you switch the For each array to loop over Var1 instead of Var2 It was done in less then 10 seconds with all objects. If you make it into different arrays you can filter the two arrays, you can probably use an intersection also. All you need to do is to fetch the records from the Azure Storage Table which is in JSON notation, now with GUIDs as ids.
4 Minutes
This is the flow for the first one of the two loops, the one I like the most, feels cleaner.
Storage Table result which you would need to truncate or something when you are done or something:
You need an Azure Storage Table to offload the results to.
When posting this I reduced Var2 in size as it made no sense in posting 1600 objects.
Code:
{
"$connections": {
"value": {
"azuretables": {
"connectionId": "/subscriptions/*YOUR SUBSCRIPTION ID*/resourceGroups/*RESOURCEGROUP NAME*/providers/Microsoft.Web/connections/azuretables",
"connectionName": "azuretables",
"id": "/subscriptions/*YOUR SUBSCRIPTION ID*/providers/Microsoft.Web/locations/westeurope/managedApis/azuretables"
}
}
},
"definition": {
"$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
"actions": {
"For_each": {
"actions": {
"For_each_2": {
"actions": {
"Condition": {
"actions": {
"Insert_Entity": {
"inputs": {
"body": {
"PartitionKey": "#{items('For_each_2')['Name']}",
"RowKey": "#{guid()}"
},
"host": {
"connection": {
"name": "#parameters('$connections')['azuretables']['connectionId']"
}
},
"method": "post",
"path": "/Tables/#{encodeURIComponent('stackoverflowforeachtmp')}/entities"
},
"runAfter": {},
"type": "ApiConnection"
}
},
"expression": {
"and": [
{
"equals": [
"#items('For_each_2')['Name']",
"#items('For_each')['Demo_Name']"
]
}
]
},
"runAfter": {},
"type": "If"
}
},
"foreach": "#variables('TestValueVariable1')",
"runAfter": {
"Set_TestVariable2": [
"Succeeded"
]
},
"runtimeConfiguration": {
"concurrency": {
"repetitions": 50
}
},
"type": "Foreach"
},
"Set_TestVariable1": {
"inputs": {
"name": "TestValueVariable2",
"value": "#{items('For_each')['Demo_Name']}"
},
"runAfter": {},
"type": "SetVariable"
},
"Set_TestVariable2": {
"inputs": {
"name": "TestValueVariable1",
"value": "#variables('Variable 1')"
},
"runAfter": {
"Set_TestVariable1": [
"Succeeded"
]
},
"type": "SetVariable"
}
},
"foreach": "#array(variables('Variable 2'))",
"runAfter": {
"For_each_3": [
"Succeeded"
]
},
"runtimeConfiguration": {
"concurrency": {
"repetitions": 50
}
},
"type": "Foreach"
},
"For_each_3": {
"actions": {
"Filter_array": {
"inputs": {
"from": "#array(variables('Variable 1'))",
"where": "#equals(items('For_each_3')['Demo_Name'], item()['Name'])"
},
"runAfter": {},
"type": "Query"
},
"For_each_4": {
"actions": {
"Insert_Entity_2": {
"inputs": {
"body": {
"EvaluationKey": "#{items('For_each_3')['Demo_Name']}-#{items('For_each_4')['Name']}",
"Owner": "#{items('For_each_4')['OwnerId']}",
"PartitionKey": "#{items('For_each_4')['Name']}",
"RowKey": "#{guid()}"
},
"host": {
"connection": {
"name": "#parameters('$connections')['azuretables']['connectionId']"
}
},
"method": "post",
"path": "/Tables/#{encodeURIComponent('stackoverflowforeachtmp')}/entities"
},
"runAfter": {},
"type": "ApiConnection"
}
},
"foreach": "#body('Filter_array')",
"runAfter": {
"Filter_array": [
"Succeeded"
]
},
"type": "Foreach"
}
},
"foreach": "#array(variables('Variable 2'))",
"runAfter": {
"TestValueVariable2": [
"Succeeded"
]
},
"type": "Foreach"
},
"Get_entities": {
"inputs": {
"host": {
"connection": {
"name": "#parameters('$connections')['azuretables']['connectionId']"
}
},
"method": "get",
"path": "/Tables/#{encodeURIComponent('stackoverflowforeachtmp')}/entities"
},
"runAfter": {
"For_each": [
"Succeeded"
]
},
"runtimeConfiguration": {
"paginationPolicy": {
"minimumItemCount": 30000
}
},
"type": "ApiConnection"
},
"Initialize_variable": {
"inputs": {
"variables": [
{
"name": "Variable 1",
"type": "Array",
"value": [
{
"Name": "AAA",
"OwnerId": "121a"
},
{
"Name": "BBB",
"OwnerId": "123b"
},
{
"Name": "AAA",
"OwnerId": "123b"
},
{
"Name": "BBB",
"OwnerId": "123b"
},
{
"Name": "FFF",
"OwnerId": "123b"
},
{
"Name": "BBB",
"OwnerId": "123b"
},
{
"Name": "DDD",
"OwnerId": "123b"
},
{
"Name": "CCC",
"OwnerId": "123b"
},
{
"Name": "BBB",
"OwnerId": "123b"
}
]
}
]
},
"runAfter": {},
"type": "InitializeVariable"
},
"Initialize_variable_2": {
"inputs": {
"variables": [
{
"name": "Variable 2",
"type": "Array",
"value": [
{
"Demo_Name": "AAA"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "CCC"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "FFF"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "BBB"
},
{
"Demo_Name": "FFF"
},
{
"Demo_Name": "AAA"
}
]
}
]
},
"runAfter": {
"Initialize_variable": [
"Succeeded"
]
},
"type": "InitializeVariable"
},
"Set_variable": {
"inputs": {
"name": "TestValueVariable2",
"value": "#{length(body('Get_entities')?['value'])}"
},
"runAfter": {
"Terminate": [
"Succeeded"
]
},
"type": "SetVariable"
},
"Terminate": {
"inputs": {
"runStatus": "Succeeded"
},
"runAfter": {
"Get_entities": [
"Succeeded"
]
},
"type": "Terminate"
},
"TestValueVariable1": {
"inputs": {
"variables": [
{
"name": "TestValueVariable1",
"type": "Array"
}
]
},
"runAfter": {
"Initialize_variable_2": [
"Succeeded"
]
},
"type": "InitializeVariable"
},
"TestValueVariable2": {
"inputs": {
"variables": [
{
"name": "TestValueVariable2",
"type": "String"
}
]
},
"runAfter": {
"TestValueVariable1": [
"Succeeded"
]
},
"type": "InitializeVariable"
}
},
"contentVersion": "1.0.0.0",
"outputs": {},
"parameters": {
"$connections": {
"defaultValue": {},
"type": "Object"
}
},
"triggers": {
"Recurrence": {
"recurrence": {
"frequency": "Month",
"interval": 3
},
"type": "Recurrence"
}
}
}
}