Azure Kubernetes Service ARM template is not idempotent - azure

Question

I have created an ARM template to deploy an Azure Kubernetes Service instance, which I am trying to plug into a CI/CD pipeline in VSTS. On the first deployment, everything works as expected and the K8s cluster is created successfully. However, upon redeployment, the template fails the validation stage with the following error:
{
"message": "The template deployment 'Microsoft.Template' is not valid according to the validation procedure."
"details": [
{
"code":"PropertyChangeNotAllowed",
"message":"Provisioning of resource(s) for container service <cluster name> in resource group <resource group name> failed. Message:"
{
"code": "PropertyChangeNotAllowed",
"message": "Changing property 'linuxProfile.ssh.publicKeys.keyData' is not allowed.",
"target": "linuxProfile.ssh.publicKeys.keyData"
}
}
]
}
The template is therefore clearly not idempotent which completely dishonours the intended nature of ARM template deployments.
Has anyone managed to find a workaround for this?

Answer 1

The solution to this is to specify the SSH RSA Public Key as a template parameter and use it when configuring the Linux profile. I have posted my ARM template below:
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"clusterName": {
"type": "string",
"metadata": {
"description": "The name of the Kubernetes cluster."
}
},
"location": {
"type": "string",
"metadata": {
"description": "The data center in which to deploy the Kubernetes cluster."
}
},
"dnsPrefix": {
"type": "string",
"metadata": {
"description": "DNS prefix to use with hosted Kubernetes API server FQDN."
}
},
"osDiskSizeGB": {
"defaultValue": 32,
"minValue": 0,
"maxValue": 1023,
"type": "int",
"metadata": {
"description": "Disk size (in GB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize."
}
},
"agentCount": {
"defaultValue": 1,
"minValue": 1,
"maxValue": 50,
"type": "int",
"metadata": {
"description": "The number of agent nodes for the cluster."
}
},
"agentVMSize": {
"defaultValue": "Standard_D1_v2",
"type": "string",
"metadata": {
"description": "The size of the Virtual Machine."
}
},
"servicePrincipalClientId": {
"type": "securestring",
"metadata": {
"description": "The Service Principal Client ID."
}
},
"servicePrincipalClientSecret": {
"type": "securestring",
"metadata": {
"description": "The Service Principal Client Secret."
}
},
"osType": {
"defaultValue": "Linux",
"allowedValues": [
"Linux"
],
"type": "string",
"metadata": {
"description": "The type of operating system."
}
},
"kubernetesVersion": {
"defaultValue": "1.10.6",
"type": "string",
"metadata": {
"description": "The version of Kubernetes."
}
},
"enableOmsAgent": {
"defaultValue": true,
"type": "bool",
"metadata": {
"description": "boolean flag to turn on and off of omsagent addon"
}
},
"enableHttpApplicationRouting": {
"defaultValue": true,
"type": "bool",
"metadata": {
"description": "boolean flag to turn on and off of http application routing"
}
},
"networkPlugin": {
"defaultValue": "kubenet",
"allowedValues": [
"azure",
"kubenet"
],
"type": "string",
"metadata": {
"description": "Network plugin used for building Kubernetes network."
}
},
"enableRBAC": {
"defaultValue": true,
"type": "bool",
"metadata": {
"description": "Flag to turn on/off RBAC"
}
},
"logAnalyticsWorkspaceName": {
"type": "string",
"metadata": {
"description": "Name of the log analytics workspace which will be used for container analytics"
}
},
"logAnalyticsWorkspaceLocation": {
"type": "string",
"metadata": {
"description": "The data center in which the log analytics workspace is deployed"
}
},
"logAnalyticsResourceGroup": {
"type": "string",
"metadata": {
"description": "The resource group in which the log analytics workspace is deployed"
}
},
"vmAdminUsername": {
"type": "string",
"metadata": {
"description": "User name for the Linux Virtual Machines."
}
},
"sshRsaPublicKey": {
"type": "securestring",
"metadata": {
"description": "Configure all linux machines with the SSH RSA public key string. Your key should include three parts, for example: 'ssh-rsa AAAAB...snip...UcyupgH azureuser#linuxvm'"
}
}
},
"variables": {
"logAnalyticsWorkspaceId": "[resourceId(parameters('logAnalyticsResourceGroup'), 'Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]",
"containerInsightsName": "[concat(parameters('clusterName'),'-containerinsights')]"
},
"resources": [
{
"type": "Microsoft.ContainerService/managedClusters",
"name": "[parameters('clusterName')]",
"apiVersion": "2018-03-31",
"location": "[parameters('location')]",
"properties": {
"kubernetesVersion": "[parameters('kubernetesVersion')]",
"enableRBAC": "[parameters('enableRBAC')]",
"dnsPrefix": "[parameters('dnsPrefix')]",
"addonProfiles": {
"httpApplicationRouting": {
"enabled": "[parameters('enableHttpApplicationRouting')]"
},
"omsagent": {
"enabled": "[parameters('enableOmsAgent')]",
"config": {
"logAnalyticsWorkspaceResourceID": "[variables('logAnalyticsWorkspaceId')]"
}
}
},
"agentPoolProfiles": [
{
"name": "agentpool",
"osDiskSizeGB": "[parameters('osDiskSizeGB')]",
"count": "[parameters('agentCount')]",
"vmSize": "[parameters('agentVMSize')]",
"osType": "[parameters('osType')]",
"storageProfile": "ManagedDisks"
}
],
"linuxProfile": {
"adminUsername": "[parameters('vmAdminUsername')]",
"ssh": {
"publicKeys": [
{
"keyData": "[parameters('sshRsaPublicKey')]"
}
]
}
},
"servicePrincipalProfile": {
"clientId": "[parameters('servicePrincipalClientId')]",
"secret": "[parameters('servicePrincipalClientSecret')]"
},
"networkProfile": {
"networkPlugin": "[parameters('networkPlugin')]"
}
},
"dependsOn": [
"[concat('Microsoft.Resources/deployments/', 'SolutionDeployment')]"
]
},
{
"type": "Microsoft.Resources/deployments",
"name": "SolutionDeployment",
"apiVersion": "2017-05-10",
"resourceGroup": "[parameters('logAnalyticsResourceGroup')]",
"properties": {
"mode": "Incremental",
"template": {
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"resources": [
{
"apiVersion": "2015-11-01-preview",
"type": "Microsoft.OperationsManagement/solutions",
"location": "[parameters('logAnalyticsWorkspaceLocation')]",
"name": "[variables('containerInsightsName')]",
"properties": {
"workspaceResourceId": "[variables('logAnalyticsWorkspaceId')]"
},
"plan": {
"name": "[variables('containerInsightsName')]",
"product": "OMSGallery/ContainerInsights",
"promotionCode": "",
"publisher": "Microsoft"
}
}
]
}
}
}
],
"outputs": {
"controlPlaneFQDN": {
"type": "string",
"value": "[reference(concat('Microsoft.ContainerService/managedClusters/', parameters('clusterName'))).fqdn]"
},
"sshMaster0": {
"type": "string",
"value": "[concat('ssh ', parameters('vmAdminUsername'), '#', reference(concat('Microsoft.ContainerService/managedClusters/', parameters('clusterName'))).fqdn, ' -A -p 22')]"
}
}
}