How to send AKS master logs to eventhub using Azurerm terraform ? As Terraform only provides log analytics option only.
In order to send logs to Event Hub using terraform you need to create few resources :
Event Hub Namespace (azurerm_eventhub_namespace)
Event Hub (azurerm_eventhub)
Authorization Rule for an Event Hub Namespace (azurerm_eventhub_namespace_authorization_rule)
Diagnostic Setting for an existing Resource (azurerm_monitor_diagnostic_setting)
The following example based on this repo.
# Create the AKS cluster
resource "azurerm_resource_group" "example" {
name = "example-resources"
location = "West Europe"
}
resource "azurerm_kubernetes_cluster" "example" {
name = "example-aks1"
location = azurerm_resource_group.example.location
resource_group_name = azurerm_resource_group.example.name
dns_prefix = "exampleaks1"
default_node_pool {
name = "default"
node_count = 1
vm_size = "Standard_D2_v2"
}
identity {
type = "SystemAssigned"
}
tags = {
Environment = "Production"
}
}
# Create Event hub namespace
resource "azurerm_eventhub_namespace" "logging" {
name = "logging-eventhub"
location = "${azurerm_resource_group.example.location}"
resource_group_name = "${azurerm_resource_group.example.name}"
sku = "Standard"
capacity = 1
kafka_enabled = false
}
# Create Event hub
resource "azurerm_eventhub" "logging_aks" {
name = "logging-aks-eventhub"
namespace_name = "${azurerm_eventhub_namespace.logging.name}"
resource_group_name = "${azurerm_resource_group.example.name}"
partition_count = 2
message_retention = 1
}
# Create an authorization rule
resource "azurerm_eventhub_namespace_authorization_rule" "logging" {
name = "authorization_rule"
namespace_name = "${azurerm_eventhub_namespace.logging.name}"
resource_group_name = "${azurerm_resource_group.example.name}"
listen = true
send = true
manage = true
}
# Manages a Diagnostic Setting for an existing Resource
resource "azurerm_monitor_diagnostic_setting" "aks-logging" {
name = "diagnostic_aksl"
target_resource_id = "${azurerm_kubernetes_cluster.example.id}"
eventhub_name = "${azurerm_eventhub.logging_aks.name}"
eventhub_authorization_rule_id = "${azurerm_eventhub_namespace_authorization_rule.logging.id}"
log {
category = "kube-scheduler"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-controller-manager"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "cluster-autoscaler"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-audit"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-apiserver"
enabled = true
retention_policy {
enabled = false
}
}
}
Related
I have created an AKS cluster using the following Terraform code
resource "azurerm_virtual_network" "test" {
name = var.virtual_network_name
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
address_space = [var.virtual_network_address_prefix]
subnet {
name = var.aks_subnet_name
address_prefix = var.aks_subnet_address_prefix
}
tags = var.tags
}
data "azurerm_subnet" "kubesubnet" {
name = var.aks_subnet_name
virtual_network_name = azurerm_virtual_network.test.name
resource_group_name = azurerm_resource_group.rg.name
depends_on = [azurerm_virtual_network.test]
}
# Create Log Analytics Workspace
module "log_analytics_workspace" {
source = "./modules/log_analytics_workspace"
count = var.enable_log_analytics_workspace == true ? 1 : 0
app_or_service_name = "log"
subscription_type = var.subscription_type
environment = var.environment
resource_group_name = azurerm_resource_group.rg.name
location = var.location
instance_number = var.instance_number
sku = var.log_analytics_workspace_sku
retention_in_days = var.log_analytics_workspace_retention_in_days
tags = var.tags
}
resource "azurerm_kubernetes_cluster" "k8s" {
name = var.aks_name
location = azurerm_resource_group.rg.location
dns_prefix = var.aks_dns_prefix
resource_group_name = azurerm_resource_group.rg.name
http_application_routing_enabled = false
linux_profile {
admin_username = var.vm_user_name
ssh_key {
key_data = file(var.public_ssh_key_path)
}
}
default_node_pool {
name = "agentpool"
node_count = var.aks_agent_count
vm_size = var.aks_agent_vm_size
os_disk_size_gb = var.aks_agent_os_disk_size
vnet_subnet_id = data.azurerm_subnet.kubesubnet.id
}
service_principal {
client_id = local.client_id
client_secret = local.client_secret
}
network_profile {
network_plugin = "azure"
dns_service_ip = var.aks_dns_service_ip
docker_bridge_cidr = var.aks_docker_bridge_cidr
service_cidr = var.aks_service_cidr
}
# Enabled the cluster configuration to the Azure kubernets with RBAC
azure_active_directory_role_based_access_control {
managed = var.azure_active_directory_role_based_access_control_managed
admin_group_object_ids = var.active_directory_role_based_access_control_admin_group_object_ids
azure_rbac_enabled = var.azure_rbac_enabled
}
oms_agent {
log_analytics_workspace_id = module.log_analytics_workspace[0].id
}
timeouts {
create = "20m"
delete = "20m"
}
depends_on = [data.azurerm_subnet.kubesubnet,module.log_analytics_workspace]
tags = var.tags
}
and I want to send the AKS application Cluster, Node, Pod, Container metrics to Log Analytics workspace so that it will be available in Azure Monitoring.
I have configured the diagnostic setting as mentioned below
resource "azurerm_monitor_diagnostic_setting" "aks_cluster" {
name = "${azurerm_kubernetes_cluster.k8s.name}-audit"
target_resource_id = azurerm_kubernetes_cluster.k8s.id
log_analytics_workspace_id = module.log_analytics_workspace[0].id
log {
category = "kube-apiserver"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-controller-manager"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "cluster-autoscaler"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-scheduler"
enabled = true
retention_policy {
enabled = false
}
}
log {
category = "kube-audit"
enabled = true
retention_policy {
enabled = false
}
}
metric {
category = "AllMetrics"
enabled = false
retention_policy {
enabled = false
}
}
}
Is that all needed? I did come across an article where they were using azurerm_application_insights and I don't understand why azurerm_application_insights is needed to capture the cluster level metrics?
You do not need Application Insights, it really depends if you want application level monitoring.
This is probably want you read:
https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/application_insights
"Manages an Application Insights component."
Application Insights provides complete monitoring of applications running on AKS and other environments.
https://learn.microsoft.com/en-us/azure/aks/monitor-aks#level-4--applications
According to good practice, you need to enable a few others:
guard should be enabled assuming you use AAD.
enable AllMetrics.
consider kube-audit-admin for reduced logging events.
consider csi-azuredisk-controller.
consider cloud-controller-manager for the cloud-node-manager component.
See more here:
https://learn.microsoft.com/en-us/azure/aks/monitor-aks#configure-monitoring
https://learn.microsoft.com/en-us/azure/aks/monitor-aks-reference
I am creating an Azure App Service resource and an App Registration resource (and app service and others that are not relevant to this question as they work fine) via Terraform.
resource "azurerm_app_service" "app" {
name = var.app_service_name
location = var.resource_group_location
resource_group_name = azurerm_resource_group.rg.name
app_service_plan_id = azurerm_app_service_plan.plan-app.id
app_settings = {
"AzureAd:ClientId" = azuread_application.appregistration.application_id
}
site_config {
ftps_state = var.app_service_ftps_state
}
}
resource "azuread_application" "appregistration" {
display_name = azurerm_app_service.app.name
owners = [data.azuread_client_config.current.object_id]
sign_in_audience = "AzureADMyOrg"
fallback_public_client_enabled = true
web {
homepage_url = var.appreg_web_homepage_url
logout_url = var.appreg_web_logout_url
redirect_uris = [var.appreg_web_homepage_url, var.appreg_web_redirect_uri]
implicit_grant {
access_token_issuance_enabled = true
id_token_issuance_enabled = true
}
}
}
output "appreg_application_id" {
value = azuread_application.appregistration.application_id
}
I need to add the App Registration client / application id to the app_settings block in the app service resource.
The error I get with the above configuration is:
{"#level":"error","#message":"Error: Cycle: azuread_application.appregistration, azurerm_app_service.app","#module":"terraform.ui","#timestamp":"2021-09-15T10:54:31.753401Z","diagnostic":{"severity":"error","summary":"Cycle: azuread_application.appregistration, azurerm_app_service.app","detail":""},"type":"diagnostic"}
Note that the output variable displays the application id correctly.
You have a cycle error because you have both resources referencing each other. Terraform builds a directed acyclical graph to work out which order to create (or destroy) resources in with the information from one resource or data source flowing into another normally determining this order.
In your case your azuread_application.appregistration resource is referencing the azurerm_app_service.app.name parameter while the azurerm_app_service.app resource needs the azuread_application.appregistration.application_id attribute.
I don't know a ton about Azure but to me that seems like the azurerm_app_service resource needs to be created ahead of the azuread_application resource and so I'd expect the link to be in that direction.
Because you are already setting the azurerm_app_service.app.name parameter to var.app_service_name then you can just directly pass var.app_service_name to azuread_application.appregistration.display_name to achieve the same result but to break the cycle error.
resource "azurerm_app_service" "app" {
name = var.app_service_name
location = var.resource_group_location
resource_group_name = azurerm_resource_group.rg.name
app_service_plan_id = azurerm_app_service_plan.plan-app.id
app_settings = {
"AzureAd:ClientId" = azuread_application.appregistration.application_id
}
site_config {
ftps_state = var.app_service_ftps_state
}
}
resource "azuread_application" "appregistration" {
display_name = var.app_service_name
owners = [data.azuread_client_config.current.object_id]
sign_in_audience = "AzureADMyOrg"
fallback_public_client_enabled = true
web {
homepage_url = var.appreg_web_homepage_url
logout_url = var.appreg_web_logout_url
redirect_uris = [var.appreg_web_homepage_url, var.appreg_web_redirect_uri]
implicit_grant {
access_token_issuance_enabled = true
id_token_issuance_enabled = true
}
}
}
output "appreg_application_id" {
value = azuread_application.appregistration.application_id
}
I have enabled Private Endpoint for my Azure Cosmos DB. Everytime i go to Cosmos, i see a Red Flag on top which says : Failed to refresh the collection list. Please try again later.
We use Terraform to deploy code.
Also i don't see any container being created even though i have the below code in Terraform
resource "azurerm_cosmosdb_sql_container" "default" {
resource_group_name = module.resourcegroup.resource_group.name
account_name = azurerm_cosmosdb_account.default.name
database_name = azurerm_cosmosdb_sql_database.default.name
name = "cosmosdb_container"
partition_key_path = "/definition/id"
throughput = 400
}
Any idea what can i do to fix this. I don't see these issues when the Cosmos is not behind a Private Endpoint and Private link
My TF Code is provided below :
resource "azurerm_cosmosdb_account" "default" {
resource_group_name = module.resourcegroup.resource_group.name
location = var.location
name = module.name_cosmosdb_account.location.cosmosdb_account.name_unique
tags = module.resourcegroup.resource_group.tags
public_network_access_enabled = false
network_acl_bypass_for_azure_services = true
enable_automatic_failover = true
is_virtual_network_filter_enabled = true
offer_type = "Standard"
kind = "GlobalDocumentDB"
consistency_policy {
consistency_level = "Session"
max_interval_in_seconds = 5
max_staleness_prefix = 100
}
geo_location {
location = module.resourcegroup.resource_group.location
failover_priority = 0
}
geo_location {
location = "eastus2"
failover_priority = 1
}
}
resource "azurerm_cosmosdb_sql_database" "default" {
resource_group_name = module.resourcegroup.resource_group.name
account_name = azurerm_cosmosdb_account.default.name
name = "cosmosdb_db"
throughput = 400
}
resource "azurerm_cosmosdb_sql_container" "default" {
resource_group_name = module.resourcegroup.resource_group.name
account_name = azurerm_cosmosdb_account.default.name
database_name = azurerm_cosmosdb_sql_database.default.name
name = "cosmosdb_container"
partition_key_path = "/definition/id"
throughput = 400
}
Even with the error from Portal the container and resources are being created from terraform . You can use Data explorer to see the database and container created from terraform.
Test:
Terraform code:
provider "azurerm" {
features{}
}
data "azurerm_resource_group" "rg" {
name = "resourcegroup"
}
resource "azurerm_virtual_network" "example" {
name = "cosmos-network"
address_space = ["10.0.0.0/16"]
location = data.azurerm_resource_group.rg.location
resource_group_name = data.azurerm_resource_group.rg.name
}
resource "azurerm_subnet" "example" {
name = "cosmos-subnet"
resource_group_name = data.azurerm_resource_group.rg.name
virtual_network_name = azurerm_virtual_network.example.name
address_prefixes = ["10.0.1.0/24"]
enforce_private_link_endpoint_network_policies = true
}
resource "azurerm_cosmosdb_account" "example" {
name = "ansuman-cosmosdb"
location = data.azurerm_resource_group.rg.location
resource_group_name = data.azurerm_resource_group.rg.name
offer_type = "Standard"
kind = "GlobalDocumentDB"
consistency_policy {
consistency_level = "BoundedStaleness"
max_interval_in_seconds = 10
max_staleness_prefix = 200
}
geo_location {
location = data.azurerm_resource_group.rg.location
failover_priority = 0
}
}
resource "azurerm_private_endpoint" "example" {
name = "cosmosansuman-endpoint"
location = data.azurerm_resource_group.rg.location
resource_group_name = data.azurerm_resource_group.rg.name
subnet_id = azurerm_subnet.example.id
private_service_connection {
name = "cosmosansuman-privateserviceconnection"
private_connection_resource_id = azurerm_cosmosdb_account.example.id
subresource_names = [ "SQL" ]
is_manual_connection = false
}
}
resource "azurerm_cosmosdb_sql_database" "example" {
name = "ansuman-cosmos-mongo-db"
resource_group_name = data.azurerm_resource_group.rg.name
account_name = azurerm_cosmosdb_account.example.name
throughput = 400
}
resource "azurerm_cosmosdb_sql_container" "default" {
resource_group_name = data.azurerm_resource_group.rg.name
account_name = azurerm_cosmosdb_account.example.name
database_name = azurerm_cosmosdb_sql_database.example.name
name = "cosmosdb_container"
partition_key_path = "/definition/id"
throughput = 400
}
Output:
Update: As per the Discussion , the error Failed to refresh the collection list. Please try again later. is by-default in your case as you have disabled public network access to the cosmosdb account while creation. If its set to disabled, public network traffic will be blocked even before the private endpoint is created.
So, for this error the possible solutions will be :
Enable public network traffic to access the account while creating the cosmosdb account from terraform. As , Even you set it to true after the private endpoint is set for cosmosdb , public access to cosmosdb will be automatically disabled , if you go to the firewalls and virtual networks you can see allow access from all networks is grayed out . So, you can check allow access from portal and add your current IP there to get access only for your public network as shown below.(note : as its bydefault set to true so you don't need to add public_network_access_enabled = true in code.)
You can use Data Explorer to check the containers which has been already verified by you .
You can create a VM on the same vnet where the endpoint is residing and
connect the cosmosdb from inside the VM on portal itself. You can refer this Microsoft Document for more details.
See below configuration I am using to add a diagonstic setting to send App service logs to a Log analytics workspace.
https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting
resource "azurerm_app_service" "webapp" {
for_each = local.apsvc_map_with_locations
name = "${var.regional_web_rg[each.value.location].name}-${each.value.apsvc_name}-apsvc"
location = each.value.location
resource_group_name = var.regional_web_rg[each.value.location].name
app_service_plan_id = azurerm_app_service_plan.asp[each.value.location].id
https_only = true
identity {
type = "UserAssigned"
identity_ids = each.value.identity_ids
}
}
resource "azurerm_monitor_diagnostic_setting" "example" {
for_each = local.apsvc_map_with_locations
name = "example"
target_resource_id = "azurerm_app_service.webapp[${each.value.location}-${each.value.apsvc_name}].id"
log_analytics_workspace_id = data.terraform_remote_state.pod_bootstrap.outputs.pod_log_analytics_workspace.id
log {
category = "AuditEvent"
enabled = false
retention_policy {
enabled = false
}
}
metric {
category = "AllMetrics"
retention_policy {
enabled = false
}
}
}
Error:
Can not parse "target_resource_id" as a resource id: Cannot parse Azure ID: parse "azurerm_app_service.webapp[].id": invalid URI for request
2020-11-06T20:19:59.3344089Z
2020-11-06T20:19:59.3346016Z on .terraform\modules\web.web\pipeline\app\apsvc\app_hosting.tf line 127, in resource "azurerm_monitor_diagnostic_setting" "example":
2020-11-06T20:19:59.3346956Z 127: resource "azurerm_monitor_diagnostic_setting" "example" {
2020-11-06T20:19:59.3347091Z
You can try with the following:
target_resource_id = azurerm_app_service.webapp["${each.value.location}-${each.value.apsvc_name}"].id
instead of:
target_resource_id = "azurerm_app_service.webapp[${each.value.location}-${each.value.apsvc_name}].id"
I have the following azurerm_function_app terrform section:
resource "azurerm_function_app" "main" {
name = "${var.storage_function_name}"
location = "${azurerm_resource_group.main.location}"
resource_group_name = "${azurerm_resource_group.main.name}"
app_service_plan_id = "${azurerm_app_service_plan.main.id}"
storage_connection_string = "${azurerm_storage_account.main.primary_connection_string}"
https_only = true
app_settings {
"APPINSIGHTS_INSTRUMENTATIONKEY" = "${azurerm_application_insights.main.instrumentation_key}"
}
}
How can I specify the OS is linux?
Since there is not much documentation, I used following technique to construct terraform template.
Create the type of function app you want in azure portal
Import same resource using terraform import command.
terraform import azurerm_function_app.functionapp1
/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.Web/sites/functionapp1
following information will be retrieved
id = /subscriptions/xxxx/resourceGroups/xxxxxx/providers/Microsoft.Web/sites/xxxx
app_service_plan_id = /subscriptions/xxx/resourceGroups/xxxx/providers/Microsoft.Web/serverfarms/xxxx
app_settings.% = 3
app_settings.FUNCTIONS_WORKER_RUNTIME = node
app_settings.MACHINEKEY_DecryptionKey = xxxxx
app_settings.WEBSITE_NODE_DEFAULT_VERSION = 10.14.1
client_affinity_enabled = false
connection_string.# = 0
default_hostname = xxxx.azurewebsites.net
enable_builtin_logging = false
enabled = true
https_only = false
identity.# = 0
kind = functionapp,linux,container
location = centralus
name = xxxxx
outbound_ip_addresses = xxxxxx
resource_group_name = xxxx
site_config.# = 1
site_config.0.always_on = true
site_config.0.linux_fx_version = DOCKER|microsoft/azure-functions-node8:2.0
site_config.0.use_32_bit_worker_process = true
site_config.0.websockets_enabled = false
site_credential.# = 1
site_credential.0.password =xxxxxx
site_credential.0.username = xxxxxx
storage_connection_string = xxxx
tags.% = 0
version = ~2
From this I build following terraform template
provider "azurerm" {
}
resource "azurerm_resource_group" "linuxnodefunction" {
name = "azure-func-linux-node-rg"
location = "westus2"
}
resource "azurerm_storage_account" "linuxnodesa" {
name = "azurefunclinuxnodesa"
resource_group_name = "${azurerm_resource_group.linuxnodefunction.name}"
location = "${azurerm_resource_group.linuxnodefunction.location}"
account_tier = "Standard"
account_replication_type = "LRS"
}
resource "azurerm_app_service_plan" "linuxnodesp" {
name = "azure-func-linux-node-sp"
location = "${azurerm_resource_group.linuxnodefunction.location}"
resource_group_name = "${azurerm_resource_group.linuxnodefunction.name}"
kind = "Linux"
reserved = true
sku {
capacity = 1
size = "P1v2"
tier = "PremiunV2"
}
}
resource "azurerm_function_app" "linuxnodefuncapp" {
name = "azure-func-linux-node-function-app"
location = "${azurerm_resource_group.linuxnodefunction.location}"
resource_group_name = "${azurerm_resource_group.linuxnodefunction.name}"
app_service_plan_id = "${azurerm_app_service_plan.linuxnodesp.id}"
storage_connection_string = "${azurerm_storage_account.linuxnodesa.primary_connection_string}"
app_settings {
FUNCTIONS_WORKER_RUNTIME = "node"
WEBSITE_NODE_DEFAULT_VERSION = "10.14.1"
}
site_config {
always_on = true
linux_fx_version = "DOCKER|microsoft/azure-functions-node8:2.0"
use_32_bit_worker_process = true
websockets_enabled = false
}
}
Let us know your experience with this. I will try to test few things with this.
I think you need to specify that in app_service_plan block
Kind = "Linux"
kind - (Optional) The kind of the App Service Plan to create. Possible values are Windows (also available as App), Linux and FunctionApp (for a Consumption Plan). Defaults to Windows. Changing this forces a new resource to be created.
NOTE: When creating a Linux App Service Plan, the reserved field must be set to true.
Example from Terraform doc
resource "azurerm_resource_group" "test" {
name = "azure-functions-cptest-rg"
location = "westus2"
}
resource "azurerm_storage_account" "test" {
name = "functionsapptestsa"
resource_group_name = "${azurerm_resource_group.test.name}"
location = "${azurerm_resource_group.test.location}"
account_tier = "Standard"
account_replication_type = "LRS"
}
resource "azurerm_app_service_plan" "test" {
name = "azure-functions-test-service-plan"
location = "${azurerm_resource_group.test.location}"
resource_group_name = "${azurerm_resource_group.test.name}"
kind = "Linux"
sku {
tier = "Dynamic"
size = "Y1"
}
properties {
reserved = true
}
}
resource "azurerm_function_app" "test" {
name = "test-azure-functions"
location = "${azurerm_resource_group.test.location}"
resource_group_name = "${azurerm_resource_group.test.name}"
app_service_plan_id = "${azurerm_app_service_plan.test.id}"
storage_connection_string = "${azurerm_storage_account.test.primary_connection_string}"
}