502 Bad Gateway from Azure Application Gateway Connecting to Azure Container Instance - azure

Question

I am working on learning Terraform and Azure Web Services. After following a series of tutorials, I've been working on getting an Azure Container Instance setup that talks to a CosmosDB instance within a virtual network, and I want an Application Gateway setup that will allow HTTP connections to the Azure Container Instance.
Currently, when I call the IP address assigned to the Application Gateway, I receive a 502 Bad Gateway. I've verified that the image I'm running in the Azure Container Instance works locally. I have a feeling that the issues I'm facing are in relation to the back-end address pool I've configured, and possibly an issue with the rules I've setup in my network security group (nsg-myapp).
I was wondering if someone could look at my Terraform and identify what I've not configured correctly? The closest question I found similar to my scenario on StackOverflow as this unresolved question from last year.
network.tf
resource "azurerm_virtual_network" "myappdb" {
name = "myappdb-vnet"
address_space = ["10.7.0.0/16"]
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
}
resource "azurerm_subnet" "internal" {
name = "internal"
resource_group_name = azurerm_resource_group.rg.name
virtual_network_name = azurerm_virtual_network.myappdb.name
address_prefixes = ["10.7.2.0/24"]
service_endpoints = ["Microsoft.AzureCosmosDB"]
delegation {
name = "acidelegationservice"
service_delegation {
name = "Microsoft.ContainerInstance/containerGroups"
actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
}
}
enforce_private_link_endpoint_network_policies = true
}
resource "azurerm_subnet" "frontend" {
name = "myapp-frontend"
resource_group_name = azurerm_resource_group.rg.name
virtual_network_name = azurerm_virtual_network.myappdb.name
address_prefixes = ["10.7.0.0/24"]
}
resource "azurerm_network_security_group" "nsg-myapp" {
name = "nsg-aci"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
security_rule {
name = "from-gateway-subnet"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_ranges = [22, 80, 443, 445, 8000]
source_address_prefixes = azurerm_subnet.internal.address_prefixes
destination_address_prefix = azurerm_subnet.internal.address_prefixes[0]
}
security_rule {
name = "DenyAllInBound-Override"
priority = 900
direction = "Inbound"
access = "Deny"
protocol = "*"
source_port_range = "*"
destination_port_range = "*"
source_address_prefix = "*"
destination_address_prefix = "*"
}
security_rule {
name = "to-internet"
priority = 100
direction = "Outbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_ranges = [80, 443, 445]
source_address_prefix = "*"
destination_address_prefix = "*"
}
security_rule {
name = "DenyAllOutBound-Override"
priority = 900
direction = "Outbound"
access = "Deny"
protocol = "*"
source_port_range = "*"
destination_port_range = "*"
source_address_prefix = "*"
destination_address_prefix = "*"
}
}
resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
subnet_id = azurerm_subnet.internal.id
network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}
resource "azurerm_network_profile" "containergroup_profile" {
name = "acg-profile"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
container_network_interface {
name = "acg-nic"
ip_configuration {
name = "aciipconfig"
subnet_id = azurerm_subnet.internal.id
}
}
}
resource "azurerm_public_ip" "myappip" {
name = "myappip"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
allocation_method = "Static"
sku = "Standard"
}
locals {
backend_address_pool_name = "${azurerm_virtual_network.myappdb.name}-beap"
frontend_port_name = "${azurerm_virtual_network.myappdb.name}-feport"
frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
http_setting_name = "${azurerm_virtual_network.myappdb.name}-be-htst"
listener_name = "${azurerm_virtual_network.myappdb.name}-httplstn"
request_routing_rule_name = "${azurerm_virtual_network.myappdb.name}-rqrt"
redirect_configuration_name = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}
resource "azurerm_application_gateway" "network" {
name = "myapp-appgateway"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
sku {
name = "Standard_v2"
tier = "Standard_v2"
capacity = 2
}
gateway_ip_configuration {
name = "my-gateway-ip-configuration"
subnet_id = azurerm_subnet.frontend.id
}
frontend_port {
name = local.frontend_port_name
port = 80
}
frontend_ip_configuration {
name = local.frontend_ip_configuration_name
public_ip_address_id = azurerm_public_ip.myappip.id
}
backend_address_pool {
name = local.backend_address_pool_name
ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
}
backend_http_settings {
name = local.http_setting_name
cookie_based_affinity = "Disabled"
path = "/path1/"
port = 80
protocol = "Http"
request_timeout = 60
}
http_listener {
name = local.listener_name
frontend_ip_configuration_name = local.frontend_ip_configuration_name
frontend_port_name = local.frontend_port_name
protocol = "Http"
}
request_routing_rule {
name = local.request_routing_rule_name
rule_type = "Basic"
http_listener_name = local.listener_name
backend_address_pool_name = local.backend_address_pool_name
backend_http_settings_name = local.http_setting_name
}
}
container.tf
resource "azurerm_container_group" "tf_cg_sampleapi" {
depends_on = [azurerm_cosmosdb_account.db]
name = "cg_myapp"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
network_profile_id = azurerm_network_profile.containergroup_profile.id
ip_address_type = "Private"
# dns_name_label = "sampleapitf"
os_type = "Linux"
identity {
type = "SystemAssigned"
}
container {
name = "myapp"
image = "sample/myapp"
cpu = 1
memory = 1
ports {
port = 80
protocol = "TCP"
}
ports {
port = 443
protocol = "TCP"
}
secure_environment_variables = {
"MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
}
}
}

Answer 1

I met the similar issue and in my case(containers on top of Azure App Service) I needed to put the depends_on block inside the application gateway resource creation with regards to app services being created in the first place. So in your case should be:
resource "azurerm_application_gateway" "network" {
name = "myapp-appgateway"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
sku {
name = "Standard_v2"
tier = "Standard_v2"
capacity = 2
}
gateway_ip_configuration {
name = "my-gateway-ip-configuration"
subnet_id = azurerm_subnet.frontend.id
}
frontend_port {
name = local.frontend_port_name
port = 80
}
frontend_ip_configuration {
name = local.frontend_ip_configuration_name
public_ip_address_id = azurerm_public_ip.myappip.id
}
backend_address_pool {
name = local.backend_address_pool_name
ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
}
backend_http_settings {
name = local.http_setting_name
cookie_based_affinity = "Disabled"
path = "/path1/"
port = 80
protocol = "Http"
request_timeout = 60
}
http_listener {
name = local.listener_name
frontend_ip_configuration_name = local.frontend_ip_configuration_name
frontend_port_name = local.frontend_port_name
protocol = "Http"
}
request_routing_rule {
name = local.request_routing_rule_name
rule_type = "Basic"
http_listener_name = local.listener_name
backend_address_pool_name = local.backend_address_pool_name
backend_http_settings_name = local.http_setting_name
}
depends_on = [ azurerm_container_group.tf_cg_sampleapi, ]
}

Answer 2

I figured out the root cause of my 502 Gateway error was due to health checks not being setup / not working. Consequently, I setup custom probes that would go to an API endpoint to return a 200 OK response. Of course, I will configure this endpoint to actually check to see if I can connect to my services, but this was just a test to verify this was the issue.
I also removed the DenyAllInBound-Override and DenyAllOutBound-Override rules within my nsg-aci security group, as this was causing issues with my ACI to connect to my Cosmos DB.
This was my resulting network.tf and container.tf files:
network.tf
resource "azurerm_virtual_network" "myappdb" {
name = "myappdb-vnet"
address_space = ["10.7.0.0/16"]
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
}
resource "azurerm_subnet" "internal" {
name = "internal"
resource_group_name = azurerm_resource_group.rg.name
virtual_network_name = azurerm_virtual_network.myappdb.name
address_prefixes = ["10.7.2.0/24"]
service_endpoints = ["Microsoft.AzureCosmosDB"]
delegation {
name = "acidelegationservice"
service_delegation {
name = "Microsoft.ContainerInstance/containerGroups"
actions = ["Microsoft.Network/virtualNetworks/subnets/join/action", "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"]
}
}
enforce_private_link_endpoint_network_policies = true
}
resource "azurerm_subnet" "frontend" {
name = "myapp-frontend"
resource_group_name = azurerm_resource_group.rg.name
virtual_network_name = azurerm_virtual_network.myappdb.name
address_prefixes = ["10.7.0.0/24"]
}
resource "azurerm_network_security_group" "nsg-myapp" {
name = "nsg-aci"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
security_rule {
name = "from-gateway-subnet"
priority = 100
direction = "Inbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_ranges = [22, 80, 443, 445, 8000]
source_address_prefixes = azurerm_subnet.internal.address_prefixes
destination_address_prefixes = azurerm_subnet.internal.address_prefixes
}
security_rule {
name = "to-internet"
priority = 100
direction = "Outbound"
access = "Allow"
protocol = "Tcp"
source_port_range = "*"
destination_port_ranges = [80, 443, 445]
source_address_prefix = "*"
destination_address_prefix = "*"
}
}
resource "azurerm_subnet_network_security_group_association" "sn-nsg-aci" {
subnet_id = azurerm_subnet.internal.id
network_security_group_id = azurerm_network_security_group.nsg-myapp.id
}
resource "azurerm_network_profile" "containergroup_profile" {
name = "acg-profile"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
container_network_interface {
name = "acg-nic"
ip_configuration {
name = "aciipconfig"
subnet_id = azurerm_subnet.internal.id
}
}
}
resource "azurerm_public_ip" "myappip" {
name = "myappip"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
allocation_method = "Static"
sku = "Standard"
}
locals {
backend_address_pool_name = "${azurerm_virtual_network.myappdb.name}-beap"
frontend_port_name = "${azurerm_virtual_network.myappdb.name}-feport"
frontend_ip_configuration_name = "${azurerm_virtual_network.myappdb.name}-feip"
http_setting_name = "${azurerm_virtual_network.myappdb.name}-be-htst"
listener_name = "${azurerm_virtual_network.myappdb.name}-httplstn"
request_routing_rule_name = "${azurerm_virtual_network.myappdb.name}-rqrt"
redirect_configuration_name = "${azurerm_virtual_network.myappdb.name}-rdrcfg"
}
resource "azurerm_application_gateway" "network" {
name = "myapp-appgateway"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
sku {
name = "Standard_v2"
tier = "Standard_v2"
capacity = 2
}
gateway_ip_configuration {
name = "my-gateway-ip-configuration"
subnet_id = azurerm_subnet.frontend.id
}
frontend_port {
name = local.frontend_port_name
port = 80
}
frontend_ip_configuration {
name = local.frontend_ip_configuration_name
public_ip_address_id = azurerm_public_ip.myappip.id
}
backend_address_pool {
name = local.backend_address_pool_name
ip_addresses = [azurerm_container_group.tf_cg_sampleapi.ip_address]
}
probe {
interval = 60
timeout = 60
name = "status"
protocol = "Http"
path = "/api/status/"
unhealthy_threshold = 3
host = "127.0.0.1"
}
backend_http_settings {
name = local.http_setting_name
cookie_based_affinity = "Disabled"
path = "/"
port = 80
protocol = "Http"
request_timeout = 60
probe_name = "status"
}
http_listener {
name = local.listener_name
frontend_ip_configuration_name = local.frontend_ip_configuration_name
frontend_port_name = local.frontend_port_name
protocol = "Http"
}
request_routing_rule {
name = local.request_routing_rule_name
rule_type = "Basic"
http_listener_name = local.listener_name
backend_address_pool_name = local.backend_address_pool_name
backend_http_settings_name = local.http_setting_name
}
depends_on = [azurerm_container_group.tf_cg_sampleapi, ]
}
container.tf
resource "azurerm_container_group" "tf_cg_sampleapi" {
depends_on = [azurerm_cosmosdb_account.db]
name = "cg_myapp"
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
network_profile_id = azurerm_network_profile.containergroup_profile.id
ip_address_type = "Private"
# dns_name_label = "sampleapitf"
os_type = "Linux"
container {
name = "myapp"
image = "sample/myapp"
cpu = 1
memory = 1
ports {
port = 80
protocol = "TCP"
}
ports {
port = 443
protocol = "TCP"
}
secure_environment_variables = {
"MYAPP_CONNECTION_STRING" = azurerm_cosmosdb_account.db.connection_strings[0]
}
}
}