Terraform - Long running Azure deployment errors out - azure

We are using Terraform executed via a shell script from a Bash window to deploy an App Service Environment. Deploying an App Service Environment takes anywhere from 1 to 2 hours to complete.
The Terraform deployment times out after 1 hour with an error that says:
azurerm_template_deployment.ase: Error creating deployment: azure#WaitForCompletion: context has been cancelled: StatusCode=200 -- Original Error: context deadline exceeded
The deployment does not actually stop and eventually succeeds. If after it completes (as observed in the Azure portal) we re-run the Terraform deployment, the deployment completes successfully.
Terraform log file:
https://gist.github.com/Phydeauxman/0f9aa3d1c1379c36e2f8f420d0ae345e
Terraform config file:
provider "azurerm" {
subscription_id = "${var.sub_id}"
}
data "terraform_remote_state" "rg" {
backend = "azurerm"
config {
storage_account_name = "${var.tfstate_storage_account}"
container_name = "${var.tfstate_container}"
key = "${var.tfstate_rgstate_file}"
access_key = "${var.tfstate_access_key}"
}
}
resource "azurerm_resource_group" "ase_rg" {
name = "${var.ilbase_rg_name}"
location = "${data.terraform_remote_state.rg.rglocation}"
}
resource "azurerm_template_deployment" "ase" {
name = "ILBASE_ARM_template"
resource_group_name = "${azurerm_resource_group.ase_rg.name}"
template_body = <<DEPLOY
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"ilbase_name": {
"type": "string"
},
"ilbase_domain_name": {
"type": "string"
},
"ilbase_subnet_name": {
"type": "string"
},
"ilbase_rglocation": {
"defaultValue": "East US",
"type": "string"
},
"vnet_id": {
"type": "string"
}
},
"variables": {
},
"resources": [
{
"apiVersion": "2016-09-01",
"type": "Microsoft.Web/hostingEnvironments",
"name": "[parameters('ilbase_name')]",
"kind": "ASEV2",
"location": "[parameters('ilbase_rglocation')]",
"properties": {
"name": "[parameters('ilbase_name')]",
"location": "[parameters('ilbase_rglocation')]",
"virtualNetwork": {
"Id": "[parameters('vnet_id')]",
"Subnet": "[parameters('ilbase_subnet_name')]"
},
"internalLoadBalancingMode": "Web, Publishing",
"multiSize": "Standard_D1_V2",
"multiRoleCount": 2,
"workerPools": null,
"ipsslAddressCount": 0,
"dnsSuffix": "[parameters('ilbase_domain_name')]",
"networkAccessControlList": [],
"frontEndScaleFactor": 15,
"apiManagementAccountId": null,
"suspended": false,
"dynamicCacheEnabled": null,
"clusterSettings": null
}
}
],
"outputs": {
}
}
DEPLOY
# these key-value pairs are passed into the ARM Template's `parameters` block
parameters {
"vnet_id" = "${data.terraform_remote_state.rg.vnetid}"
"ilbase_subnet_name" = "${data.terraform_remote_state.rg.sn2name}"
"ilbase_name" = "${var.ilbase_name}"
"ilbase_domain_name" = "${var.ilbase_domain_name}"
}
deployment_mode = "Incremental"
}
#output "storageAccountName" {
# value = "${azurerm_template_deployment.test.outputs["storageAccountName"]}"
#}

There doesn't seem to be a specific question, but as of the AzureRM 2.0 provider it is now possible to add custom timeouts. Secondly it is considered best practice to only use azurerm_template_deployment when a resource type doesn't exist in terraform. App service is a first class resource

Related

Cannot determine where this terraform apply error occurs

I'm getting the following error when I do a terraform apply, Error: validating Template Deployment "uksfe-dev-api-office365" (Resource Group "app-sfe-dev-eastus"): requesting validating: resources.DeploymentsClient#Validate: Failure sending request: StatusCode=400 -- Original Error: Code="InvalidRequestContent" Message="The request content was invalid and could not be deserialized: 'Error converting value \"sfe-dev-api-office365\" to type 'Azure.Deployments.Core.Definitions.DeploymentParameterDefinition'. Path 'properties.parameters.connections_office365_name', line 1, position 1590.'.".
Here is the resource the error references:
resource "azurerm_resource_group_template_deployment" "office365" {
name = format( "%s%s-%s-api-office365", var.sfe_names.market, var.sfe_names.product_group, var.sfe_names.environment)
resource_group_name = module.resource_group.name
template_content = file("./refScript/logicapp/Office365.json")
deployment_mode = "Incremental"
parameters_content = jsonencode({
"connections_office365_name" = format( "%s-%s-api-office365", var.sfe_names.product_group, var.sfe_names.environment),
"subscription_id" = data.azurerm_subscription.current.subscription_id
})
}
And here is the ARM template file referenced by the resource shown above, Office365.json:
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"connections_office365_name": {
"defaultValue": "testoffice365",
"type": "String"
}
},
"variables": {},
"resources": [
{
"type": "Microsoft.Web/connections",
"apiVersion": "2016-06-01",
"name": "[parameters('connections_office365_name')]",
"location": "eastus",
"kind": "V1",
"properties": {
"displayName": "rsgfileexchange#mycompanydomain.com",
"statuses": [
{
"status": "Connected"
}
],
"customParameterValues": {},
"nonSecretParameterValues": {},
"createdTime": "2021-03-25T07:41:30.7103666Z",
"changedTime": "2021-09-02T19:26:09.2638641Z",
"api": {
"name": "sfe-dev-api-office365",
"displayName": "Office 365 Outlook",
"description": "Microsoft Office 365 is a cloud-based service that is designed to help meet your organization's needs for robust security, reliability, and user productivity.",
"iconUri": "https://connectoricons-prod.azureedge.net/releases/v1.0.1507/1.0.1507.2528/office365/icon.png",
"brandColor": "#0078D4",
"id": "/subscriptions/dfdbeere-dfda-ghgh-eree-18a838e6ed7a/providers/Microsoft.Web/locations/eastus/managedApis/office365",
"type": "Microsoft.Web/locations/managedApis"
},
"testLinks": [
{
"requestUri": "[concat('https://management.azure.com:443/subscriptions/dfdbeere-dfda-ghgh-eree-18a838e6ed7a/resourceGroups/app-sfe-dev-eastus/providers/Microsoft.Web/connections/', parameters('connections_office365_name'), '/extensions/proxy/testconnection?api-version=2016-06-01')]",
"method": "get"
}
]
}
}
]
}
I believe the last part of the error message tells where the error occurred, i.e. Path 'properties.parameters.connections_office365_name', line 1, position 1590.'
Any help would be most appreciated.
The example here shows that parameters are passed to the template like this:
parameters_content = jsonencode({
"vnetName" = {
value = local.vnet_name
}
})
So, your code would need to be modified as follows:
parameters_content = jsonencode({
"connections_office365_name" = { value = format( "%s-%s-api-office365", var.sfe_names.product_group, var.sfe_names.environment) }
"subscription_id" = { value = data.azurerm_subscription.current.subscription_id }
})

Terraform provider to create cassandra tables

I'm looking to create cassandra DB tables through terraform, on Azure. I already have the relative keyspaces in place.
My deployment is leveraging azurerm, however their provisioner is lacking a cassandra-tables resources.
As of now, I can only deploy cassandra tables through Azure UI on the portal or with Azure CLI scripting, however this isn't the best solution for a variety of reasons.
Is there a provider that could help me with this? I'm giving a look around but it seems that there isn't much I could leverage.
For whatever reason it looks like hashicorp never implemented cassandra table in their provider. Their source code is missing the implementation for it.
I suggest filing a new bug on their repo. You can do that here
Apparently a workaround could be deploying the resource to Azure as ARM using the ARM provider in an "incremental" mode.
resource "azurerm_resource_group_template_deployment" "example" {
depends_on = [module.cassandratest]
name = "example-cassandra-tables"
resource_group_name = azurerm_resource_group.test.name
deployment_mode = "Incremental"
template_content = templatefile("resources/templatecosmos.json", {
cosmos_db_account_name = "test-cassandra-2",
keyspace_name = "keyspace1",
table_name = "test-table-2",
autoscale_max_throughput = 4000
})
}
CosmosDB Cassandra templates are documented here. An example of the contents of resources/templatecosmos.json:
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"accountName": {
"type": "string",
"defaultValue": "${cosmos_db_account_name}",
"metadata": {
"description": "Cosmos DB account name, max length 44 characters"
}
},
"keyspaceName": {
"type": "string",
"defaultValue": "${keyspace_name}",
"metadata": {
"description": "The name for the Cassandra Keyspace"
}
},
"tableName": {
"type": "string",
"defaultValue": "${table_name}",
"metadata": {
"description": "The name for the Cassandra table"
}
},
"autoscaleMaxThroughput": {
"type": "int",
"defaultValue": "[int(${autoscale_max_throughput})]",
"minValue": 4000,
"maxValue": 1000000,
"metadata": {
"description": "Maximum autoscale throughput for the Cassandra table"
}
}
},
"variables": {
"accountName": "[toLower(parameters('accountName'))]",
"databaseRef": "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('accountName'))]",
"keyspaceRef": "[resourceId('Microsoft.DocumentDB/databaseAccounts/cassandraKeyspaces', parameters('accountName'), parameters('keyspaceName'))]"
},
"resources": [
{
"type": "Microsoft.DocumentDb/databaseAccounts/cassandraKeyspaces/tables",
"name": "[concat(variables('accountName'), '/', parameters('keyspaceName'), '/', parameters('tableName'))]",
"apiVersion": "2020-04-01",
"properties": {
"resource": {
"id": "[concat(parameters('tableName'))]",
"schema": {
"columns": [
{
"name": "loadid",
"type": "uuid"
}
],
"partitionKeys": [
{ "name": "machine" },
{ "name": "cpu" },
{ "name": "mtime" }
],
"clusterKeys": [
{
"name": "loadid",
"orderBy": "asc"
}
]
}
},
"options": {
"autoscaleSettings": {
"maxThroughput": "[parameters('autoscaleMaxThroughput')]"
}
}
}
}
]
}

parameters_content value could not be deserialized

I'm using an inline deployment for azurerm_resource_group_template_deployment and I'm passing a parameter that uses the name of another terraform resource.
Code:
resource "azurerm_app_service" "webapi" {
name = "${lower(var.deploymentEnvironment)}-webapi"
location = azurerm_resource_group.frontendResourceGroup.location
resource_group_name = azurerm_resource_group.frontendResourceGroup.name
app_service_plan_id = azurerm_app_service_plan.appSvcPlan.id
}
resource "azurerm_resource_group_template_deployment" "webapi_virtual_directory" {
name = "webapi_virtual_directory"
resource_group_name = azurerm_resource_group.frontendResourceGroup.name
deployment_mode = "Incremental"
template_content = <<TEMPLATE
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"webAppName": {
"type": "String"
},
"virtualApplications":{
"type": "array",
"defaultValue":[
{
"virtualPath": "/",
"physicalPath": "site\\wwwroot",
"preloadEnabled": false,
"virtualDirectories": null
}
]
}
},
"variables": {},
"resources": [
{
"type": "Microsoft.Web/sites/config",
"name": "[concat(parameters('webAppName'), '/web')]",
"apiVersion": "2020-06-01",
"properties": {
"virtualApplications": "[parameters('virtualApplications')]"
},
"dependsOn": []
}
]
}
TEMPLATE
parameters_content = jsonencode({webAppName = azurerm_app_service.webapi.name})
depends_on = [azurerm_app_service.webapi]
}
When I run the terraform apply command, I get the error:
Error: validating Template Deployment "webapi_virtual_directory"
(Resource Group "frontendapps-rg"): requesting validating:
resources.DeploymentsClient#Validate: Failure sending request:
StatusCode=400 -- Original Error: Code="InvalidRequestContent"
Message="The request content was invalid and could not be
deserialized: 'Error converting value "ci-webapi" to type
'Azure.ResourceManager.Deployments.Templates.Definitions.DeploymentParameterDefinition'.
Path 'properties.parameters.webAppName', line 1, position 861.'."
on main.tf line 134, in resource
"azurerm_resource_group_template_deployment"
"webapi_virtual_directory": 134: resource
"azurerm_resource_group_template_deployment"
"webapi_virtual_directory" {
Instead of using the parameters_content property, I added defaultValue to the template_content like this:
resource "azurerm_resource_group_template_deployment" "webapi_virtual_directory" {
name = "webapi_virtual_directory"
resource_group_name = azurerm_resource_group.frontendResourceGroup.name
deployment_mode = "Incremental"
template_content = <<TEMPLATE
{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"webAppName": {
"type": "String",
"defaultValue": "${azurerm_app_service.webapi.name}"
},
"virtualApplications":{
"type": "array",
"defaultValue":[
{
"virtualPath": "/",
"physicalPath": "site\\wwwroot",
"preloadEnabled": false,
"virtualDirectories": null
}
]
}
},
"resources": [
{
"type": "Microsoft.Web/sites/config",
"name": "[concat(parameters('webAppName'), '/web')]",
"apiVersion": "2020-06-01",
"properties": {
"virtualApplications": "[parameters('virtualApplications')]"
}
}
]
}
TEMPLATE
depends_on = [azurerm_app_service.webapi]
}
Now I don't get the error validating template deployment.
The first answer is a workaround, here's what you need to do to make parameters_content work:
This:
parameters_content = jsonencode({webAppName = azurerm_app_service.webapi.name})
Should be:
parameters_content = jsonencode(
{
webAppName = { value = azurerm_app_service.webapi.name }
})
Since the value of the property webAppName is a more complex type containing a property called value.

ARM Return App Service Environment ID to use in Terraform Script

Terraform does not allow for the deployment of App Service Environments so I am using the azurerm_template_deployment as a work around. However, I want to reference the App Service Environment ID in an App Service Plan resource that I am creating later. How would I get and save the ID of the App Service Environment using this method?
I am using the depends_on tag in the app service plan resource to ensure its creation after the app service environment, but I can not figure out how to get the id out of the creation and save to a variable. I think that it would involve the use of the variable and output tags of the ARM template.
resource "azurerm_template_deployment" "ase" {
name = "ILBASE_ARM_template"
resource_group_name = "${azurerm_resource_group.ase.name}"
template_body = <<DEPLOY
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"ilbase_name": {
"type": "string"
},
"ilbase_domain_name": {
"type": "string"
},
"ilbase_subnet_name": {
"type": "string"
},
"ilbase_rglocation": {
"defaultValue": "East US",
"type": "string"
},
"vnet_id": {
"type": "string"
}
},
"variables": {
},
"resources": [
{
"apiVersion": "2016-09-01",
"type": "Microsoft.Web/hostingEnvironments",
"name": "[parameters('ilbase_name')]",
"kind": "ASEV2",
"location": "[parameters('ilbase_rglocation')]",
"properties": {
"name": "[parameters('ilbase_name')]",
"location": "[parameters('ilbase_rglocation')]",
"virtualNetwork": {
"Id": "[parameters('vnet_id')]",
"Subnet": "[parameters('ilbase_subnet_name')]"
},
"internalLoadBalancingMode": "Web, Publishing",
"multiSize": "Standard_D1_V2",
"multiRoleCount": 2,
"workerPools": null,
"ipsslAddressCount": 0,
"dnsSuffix": "[parameters('ilbase_domain_name')]",
"networkAccessControlList": [],
"frontEndScaleFactor": 15,
"apiManagementAccountId": null,
"suspended": false,
"dynamicCacheEnabled": null,
"clusterSettings": null
}
}
],
"outputs": {
}
}
DEPLOY
parameters {
"vnet_id" = "${azurerm_virtual_network.main_vnet.id}"
"ilbase_subnet_name" = "${azurerm_subnet.ase.name}"
"ilbase_name" = "${var.env}-ASE-001"
"ilbase_domain_name" = "${var.dnsName}"
"ilbase_rglocation" = "${var.location}"
}
deployment_mode = "Incremental"
}
resource "azurerm_app_service_plan" "test" {
name = "api-appserviceplan-pro"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.ase.name}"
app_service_environment_id = ????????????????????
sku {
tier = "Isolated"
size = "S1"
}
depends_on = ["azurerm_template_deployment.ase"]
}
Thanks in advance for any help!
In the ARM template, use outputs to set an output to the app service environment ID.
(something like this, didn't have a chance to test, any feedback on changes would be greatly appreciated!)
"outputs": {
"app_service_evironment_id": {
"type": "string",
"value": "[resourceId('Microsoft.Web/hostingEnvironments', parameters('ilbase_name'))]"
}
}
The azurerm_template_deployment supports an outputs map. Using this map, you can then set
azurerm_app_service_plan.test.app_service_environment_id = azurerm_template_deployment.ase.outputs["app_service_evironment_id"]
The depends_on shouldn't be necessary and should be implicit (since the azurerm_app_service_plan uses an output of azurerm_template_deployment)

How to deploy an App Service Extension via Terraform or ARM?

We are using an extension for our AppService. How do I automate adding it via ARM template and/or Terraform? I cannot find it in Azure generated ARM template for the app or service plan.
Thanks!
Here is a template you could refer to, use parameters extensionName AspNetCoreRuntime.2.2.x64 and extensionVersion 2.2.0-preview3-35497 as your desired. You could find the extension info in Azure Resource Explorer.
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"siteName": {
"type": "string",
"metadata": {
"description": "The Azure App Service Name"
}
},
"extensionName": {
"type": "string",
"metadata": {
"description": "The Site Extension Name."
}
},
"extensionVersion": {
"type": "string",
"metadata": {
"description": "The Extension Version"
}
}
},
"resources": [
{
"type": "Microsoft.Web/sites/siteextensions",
"name": "[concat(parameters('siteName'), '/', parameters('extensionName'))]",
"apiVersion": "2015-04-01",
"location": "[resourceGroup().location]",
"properties": {
"version": "[parameters('extensionVersion')]"
}
}
]
}
Result:
You also could use the ARM Template in Terraform. You could add an azurerm_deployment_template block in main.tf. It's like this
resource "azurerm_template_deployment" "extension" {
name = "extension"
resource_group_name = "${azurerm_resource_group.main.name}"
template_body = "${file("arm/siteextensions.json")}"
parameters {
"siteName" = "${azurerm_app_service.main.name}"
"extensionName" = "AspNetCoreRuntime.2.2.x64"
"extensionVersion" = "2.2.0-preview3-35497"
}
deployment_mode = "Incremental"
}
You could get more details from this blog regarding applying Azure App Service extensions with ARM
The azurerm_template_deployment is superseded by azurerm_resource_group_template_deployment and will be deprecated.
Here's the example configuration I used to enable Datadog extension in Azure App Service for reference:
resource "azurerm_resource_group_template_deployment" "dd_extension" {
name = "dd-extension"
resource_group_name = var.resource_group_name
deployment_mode = "Incremental"
parameters_content = jsonencode({
"site_name" = {
value = var.web_app_name
}
})
template_content = "${file("arm/siteextensions.json")}"
}
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"site_name": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.Web/sites/siteextensions",
"apiVersion": "2021-01-15",
"name": "[concat(parameters('site_name'), '/Datadog.AzureAppServices.DotNet')]",
"location": "[resourceGroup().location]"
}
]
}

Resources