Setting up azure virtual machine scale set with LinuxDiagnostic extension - azure

Question

I am trying to setup a Service fabric cluster and while doing so I am creating a azure virtual machine scale set with LinuxDiagnostic as one of the extension. Following is the code for the VM scale set:
resource "azurerm_virtual_machine_scale_set" "sf_scale_set" {
name = "sf-scale-set-${terraform.workspace}"
location = "${var.location}"
resource_group_name = "${azurerm_resource_group.fusion.name}"
# automatic rolling upgrade
automatic_os_upgrade = true
upgrade_policy_mode = "Automatic"
# required when using rolling upgrade policy
health_probe_id = "${azurerm_lb_probe.sf_lb_probe.id}"
sku {
name = "${var.sf_scale_set_vm_config["name"]}"
tier = "${var.sf_scale_set_vm_config["tier"]}"
capacity = "${var.sf_scale_set_vm_config["capacity"]}"
}
storage_profile_image_reference {
publisher = "Canonical"
offer = "UbuntuServer"
sku = "16.04"
version = "6.0.12"
}
storage_profile_os_disk {
name = ""
caching = "ReadWrite"
create_option = "FromImage"
managed_disk_type = "Standard_LRS"
}
os_profile_secrets {
source_vault_id = "${var.sf_vault_id}"
vault_certificates {
certificate_url = "${var.sf_vault_url}"
}
}
storage_profile_data_disk {
lun = 0
caching = "ReadWrite"
create_option = "Empty"
disk_size_gb = 40
}
os_profile {
computer_name_prefix = "sf-vm-${terraform.workspace}"
admin_username = "hachadmin"
}
os_profile_linux_config {
disable_password_authentication = true
ssh_keys {
path = "/home/admin/.ssh/authorized_keys"
key_data = "${file("sshkeys/admin.pub")}"
}
}
network_profile {
name = "sf-vm-net-profile-${terraform.workspace}"
primary = true
ip_configuration {
name = "sf-ip-config-${terraform.workspace}"
primary = true
subnet_id = "${azurerm_subnet.sf_vnet_subnet.id}"
load_balancer_backend_address_pool_ids = ["${azurerm_lb_backend_address_pool.sf_be_vm_set.id}"]
load_balancer_inbound_nat_rules_ids = ["${element(azurerm_lb_nat_pool.sf_nat_vm_set.*.id, count.index)}"]
}
}
extension {
name = "sf-scale-set-extension-${terraform.workspace}"
publisher = "Microsoft.Azure.ServiceFabric"
type = "ServiceFabricLinuxNode"
type_handler_version = "1.0"
settings = "{ \"certificate\": { \"thumbprint\": \"${var.cert_thumbprint}\", \"x509StoreName\": \"My\" } , \"clusterEndpoint\": \"${azurerm_service_fabric_cluster.sf_service.cluster_endpoint}\", \"nodeTypeRef\": \"${terraform.workspace}-sf-node-type\", \"durabilityLevel\": \"${var.sf_reliability}\",\"nicPrefixOverride\": \"${azurerm_subnet.sf_vnet_subnet.address_prefix}\",\"enableParallelJobs\": \"true\"}"
protected_settings = "{\"StorageAccountKey1\": \"${azurerm_storage_account.sf_storage.primary_access_key}\", \"StorageAccountKey2\": \"${azurerm_storage_account.sf_storage.secondary_access_key}\"}"
}
extension {
name = "sf-scale-set-linux-diag-extension-${terraform.workspace}" # This extension connects vms to the cluster.
publisher = "Microsoft.OSTCExtensions"
type = "LinuxDiagnostic"
type_handler_version = "2.3"
auto_upgrade_minor_version = true
protected_settings = "{\"storageAccountName\": \"${azurerm_storage_account.sf_storage_app_diag.primary_access_key}\", \"StorageAccountKey1\": \"${azurerm_storage_account.sf_storage_app_diag.primary_access_key}\", \"StorageAccountKey2\": \"${azurerm_storage_account.sf_storage_app_diag.secondary_access_key}\"}"
settings = "${data.template_file.settings.rendered}"
}
tags {
Region = "${var.location}"
Createdby = "${var.created_by_tag}"
Team = "${var.team_tag}"
Environment = "${terraform.workspace}"
ninetofive = "${var.ninetofivetag}"
}
}
data "template_file" "settings" {
template = "${file("${path.module}/diagnostics/settings2.3.json.tpl")}"
vars {
xml_cfg = "${base64encode(data.template_file.wadcfg.rendered)}"
diag_storage_name = "${azurerm_storage_account.sf_storage_app_diag.name}"
}
}
data "template_file" "wadcfg" {
template = "${file("${path.module}/diagnostics/wadcfg.xml.tpl")}"
vars {
virtual_machine_id = "${azurerm_virtual_machine_scale_set.sf_scale_set.id}"
}
}
The end of Wadcfg file looks like follow:
<WadCfg>
<PerformanceCounters scheduledTransferPeriod="PT1M">
.....
......
</PerformanceCounters>
<Metrics resourceId="${virtual_machine_id}">
<MetricAggregation scheduledTransferPeriod="PT1H"/>
<MetricAggregation scheduledTransferPeriod="PT1M"/>
</Metrics>
</DiagnosticMonitorConfiguration>
</WadCfg>
Settings2.3.json.tpl file is
{
"xmlCfg": "${xml_cfg}",
"storageAccount": "${diag_storage_name}"
}
While trying to run the Terraform code I get the following error:
[+] Found tfvars file ./profiles/eu-sprint/eu-sprint.tfvars
Error: Cycle: data.template_file.wadcfg, data.template_file.settings, azurerm_virtual_machine_scale_set.sf_scale_set
I am assuming that Terraform is trying to render the template wadcfg.xml.tpl without the Azure VM scale set. Following are some of my question:
How can I enforce Terraform to wait until the Azure VM scale set are created before trying to render the wadcfg.xml.tpl file
As part of rendering my wadcfg.xml.tpl I am passing the VM id's, I know this will work if I am only creating one instance but will the code above also work for VM scale set without me explicitly looping through each VM's? If incase I would have to loop through each of them, what would be the recommenced approach?
I saw there's a https://www.terraform.io/docs/providers/azurerm/r/virtual_machine_extension.html resource existing to install VM extension, will this also work for VM scale set? If not is there a better way I could organize my settings and protected_settings part so that they are reader friendly?
I would appreciate some help here.