I'm currently implementing Azure Application Insights logging with Serilog which is working fine except for when I use an output template in my Serilog configuration. It seems like the template is ignored when passing the Serilog data to Application insights.
My serilog config in appsetting.json:
"Serilog": {
"Using": [ "Serilog.Sinks.ApplicationInsights" ],
"MinimumLevel": "Debug",
"WriteTo": [
{ "Name": "Console" },
{
"Name": "RollingFile",
"Args": {
"pathFormat": "logs\\log-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}"
}
},
{
"Name": "ApplicationInsights",
"Args": {
"restrictedToMinimumLevel": "Information",
"telemetryConverter": "Serilog.Sinks.ApplicationInsights.Sinks.ApplicationInsights.TelemetryConverters.TraceTelemetryConverter, Serilog.Sinks.ApplicationInsights",
"outputTemplate": "Test Template - {Message}"
}
}
],
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ],
"Properties": {
"Application": "app"
}
},
The logging statement:
logger.Error("Test Serilog Error For AI - " + DateTime.Now);
The output within application insights:
Is this the correct approach to customising an error message for Application insights?
After checking the source code serilog-sinks-applicationinsights, you will find it did not read the outputTemplate from appsetting.json.
For a workaround, you may implement custom TemplateTraceTelemetryConverter.
TemplateTraceTelemetryConverter
public class TemplateTraceTelemetryConverter : TraceTelemetryConverter
{
public override IEnumerable<ITelemetry> Convert(LogEvent logEvent, IFormatProvider formatProvider)
{
var templateParser = new MessageTemplateParser();
var template = templateParser.Parse($"Test Template - {logEvent.MessageTemplate.Text}");
LogEvent newLogEvent = new LogEvent(logEvent.Timestamp
, logEvent.Level
, logEvent.Exception
, template
, logEvent.Properties.Select(p => new LogEventProperty(p.Key, p.Value)));
return base.Convert(newLogEvent, formatProvider);
}
}
Use TemplateTraceTelemetryConverter
"Serilog": {
"Using": [
"Serilog.Sinks.ApplicationInsights",
],
"MinimumLevel": "Debug",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "Test Template - {Message}"
}
},
{
"Name": "RollingFile",
"Args": {
"pathFormat": "logs\\log-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}"
}
},
{
"Name": "ApplicationInsights",
"Args": {
"restrictedToMinimumLevel": "Error",
"telemetryConverter": "YourProjectNamespace.TemplateTraceTelemetryConverter, YourProjectNamespace"
//"outputTemplate": "Test Template - {Message}"
}
}
],
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ],
"Properties": {
"Application": "app"
}
}
Related
I have an issue with serilog when i want to write to a database based on different environments.
Serilog override my appsettings.production.json and always take the settings in appsettings.json instead even in a production mode !
My code in my local appsettings.json
"Serilog": {
"Using": [ "Serilog.Sinks.MSSqlServer" ],
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Microsoft": "Warning",
"System": "Warning"
}
},
"WriteTo": {
"Console" : {"Name": "Console"},
"Sql": {
"Name": "MSSqlServer",
"Args": {
"connectionString": "Server=MyServerInDev\\SQLSERVER;Database=MyDBDev;user id=ui;password=pw_;ConnectRetryCount=0;MultipleActiveResultSets=true",
"tableName": "Log",
"autoCreateSqlTable": true
}
}
},
appsettings.production.json
"Serilog": {
"Using": [ "Serilog.Sinks.MSSqlServer" ],
"MinimumLevel": {
"Default": "Warning",
"Override": {
"Microsoft": "Warning",
"System": "Warning"
}
},
"WriteTo": {
"Console" : {"Name": "Console"},
"Sql": {
"Name": "MSSqlServer",
"Args": {
"connectionString": "Server=MyServerInProd\\SQLSERVER;Database=MyDBProdv;user id=ui;password=pw_;ConnectRetryCount=0;MultipleActiveResultSets=true",
"tableName": "Log",
"autoCreateSqlTable": true
}
}
},
My program.cs
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"}.json", optional: true)
.AddEnvironmentVariables()
.Build();
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.CreateLogger();
I am missing something ? and why serilog take the appsettings.json settings and not appsettings.production.json even in prod mode
Note that the other settings in the appsettings.production.json work fine. Only the settings in serilog is causing the problem
Found it. The reason is that Serilog adds sink configs to array, and each of two files of appsettings just adds another sinks instead of replacing them. Solution is explicitly indicate array indexes for each sink. Credits for answer: nblumhardt. Tested, works.
"WriteTo": {
"0": {
"Name": "File",
"Args": {
"path": "./logs/myapp-.txt",
"rollingInterval": "Day"
}
}
},
I'm trying to get the default Service Fabric app logging to work in Azure the same way it does locally.
In Visual Studio 2019 v16.4.5 I created the sample Service Fabric stateless application .Net Core 3.1.
Everything built and runs locally OK.
In the Diagnostics Event window I can see the application log messages "Working-1522" this if from the sample source code:
ServiceEventSource.Current.ServiceMessage(this.Context, "Working-{0}", ++iterations);
But.. When I deploy to Azure it deploys OK and runs OK but I no longer see the "Working-1522" messages in any of the storage WADServiceFabricSystemEventTable or WADServiceFabricReliableServiceEventTable tables.
I don't see the messages anywhere.
I am not using Application Insights. The nodes have the IaaSDiagnostics Microsoft.Azure.Diagnostics Extension, with these Settings:
{
"WadCfg": {
"DiagnosticMonitorConfiguration": {
"overallQuotaInMB": "50000",
"sinks": "applicationInsights",
"EtwProviders": {
"EtwEventSourceProviderConfiguration": [
{
"provider": "Microsoft-ServiceFabric-Actors",
"scheduledTransferKeywordFilter": "1",
"scheduledTransferPeriod": "PT5M",
"DefaultEvents": {
"eventDestination": "ServiceFabricReliableActorEventTable"
}
},
{
"provider": "Microsoft-ServiceFabric-Services",
"scheduledTransferPeriod": "PT5M",
"DefaultEvents": {
"eventDestination": "ServiceFabricReliableServiceEventTable"
}
}
],
"EtwManifestProviderConfiguration": [
{
"provider": "cbd93bc2-71e5-4566-b3a7-595d8eeca6e8",
"scheduledTransferLogLevelFilter": "Information",
"scheduledTransferKeywordFilter": "4611686018427387904",
"scheduledTransferPeriod": "PT5M",
"DefaultEvents": {
"eventDestination": "ServiceFabricSystemEventTable"
}
}
]
}
},
"SinksConfig": {
"Sink": [
{
"name": "applicationInsights",
"ApplicationInsights": ""
}
]
}
},
"StorageAccount": "wad34xxxxxxxxxxx"
}
Any Suggestions?
Did you follow this walkthrough?
You seem to be missing the storage account details:
"type": "IaaSDiagnostics",
"autoUpgradeMinorVersion": true,
"protectedSettings": {
"storageAccountName": "[parameters('applicationDiagnosticsStorageAccountName')]",
"storageAccountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('applicationDiagnosticsStorageAccountName')),'2015-05-01-preview').key1]",
"storageAccountEndPoint": "https://core.windows.net/"
},
I am following Microsoft documentation and trying to set the logs for edge Agent through the container options. My deployment.template.json file is as follows:
{
"$schema-template": "2.0.0",
"modulesContent": {
"$edgeAgent": {
"properties.desired": {
"schemaVersion": "1.0",
"runtime": {
"type": "docker",
"settings": {
"minDockerVersion": "v1.25",
"loggingOptions": "",
"registryCredentials": {
"myRegistryName": {
"username": "$CONTAINER_REGISTRY_USERNAME",
"password": "$CONTAINER_REGISTRY_PASSWORD",
"address": "myRegistryAddress.azurecr.io"
}
}
}
},
"systemModules": {
"edgeAgent": {
"type": "docker",
"settings": {
"image": "mcr.microsoft.com/azureiotedge-agent:1.0",
"createOptions": {
"HostConfig": {
"LogConfig": {
"Type": "json-file",
"Config": {
"max-size": "10m",
"max-file": "3"
}
}
}
}
}
},
"edgeHub": {
"type": "docker",
"status": "running",
"restartPolicy": "always",
"settings": {
"image": "mcr.microsoft.com/azureiotedge-hub:1.0",
"createOptions": {
"HostConfig": {
"PortBindings": {
"5671/tcp": [
{
"HostPort": "5671"
}
],
"8883/tcp": [
{
"HostPort": "8883"
}
],
"443/tcp": [
{
"HostPort": "443"
}
]
},
"LogConfig": {
"Type": "json-file",
"Config": {
"max-size": "10m",
"max-file": "3"
}
}
}
}
}
}
},
"modules": {
"Module_Name": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"settings": {
"image": "${MODULES.Module_Name}",
"createOptions": {
"HostConfig": {
"LogConfig": {
"Type": "json-file",
"Config": {
"max-size": "10m",
"max-file": "3"
}
}
}
}
}
}
}
}
},
"$edgeHub": {
"properties.desired": {
"schemaVersion": "1.0",
"routes": {
"route": "FROM /messages/* INTO $upstream"
},
"storeAndForwardConfiguration": {
"timeToLiveSecs": 7200
}
}
}
}
}
When I build and deploy it on my device, edge Hub and my module log rotation are applied but edgeAgent log rotation is not applied. I check the log rotation settings in the
/var/lib/docker/containers/{container_id}/hostconfig.json file.
What I have done so far:
Removed the image through sudo docker rmi sudo docker rmi mcr.microsoft.com/azureiotedge-agent:1.0 and removed all the containers including the edgeAgent container and restarted the edge environment by sudo systemctl restart iotedge. The log rotation is still not applied to the new container created by the edge run time. I am not sure, what am I missing ? Any help is appreciated. Please note, I don't want to apply log rotation by creating a daemon.json file and placing it in the edge run time folder. I need to do it through the container options specified in the deployment.template.json file.
This is a known bug where edge agent deployment does not apply if version number is identical to the one in config.yaml; please help to create a github issue for it. To workaround, please set the options in config.yaml.
I've created a simple Azure Function with the intension of using Serililog for logging to Azure Blob Storage.
When using inline configuration for the Serilog sink, that works perfectly fine, the sink is created and Serilog happily wirtes to Blob storage.
This works:
Log.Logger = new LoggerConfiguration()
.WriteTo.AzureBlobStorage(
"STORAGEACCOUNT CONNECTION STRING", // <-- inline config
LogEventLevel.Information,
null,
"{yyyy}/{MM}/{dd}/MyLogFile.txt")
.CreateLogger();
The problem is that I would like to configure this all via appsettings.json, but when I try this (both locally running in the emulator & in the cloud), it fails to bind the sink settings, and as a result its failing to log to to the Blog storage.
If I debug, I can see that the config values are being loaded as expected into the configuration object, but they are not being applied to the logging configuration.
This doesnt work:
Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.CreateLogger();
appsettings.json in this snippet:
{
"Serilog": {
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Information"
}
},
"WriteTo": [{
"Name": "AzureBlobStorage",
"Args": {
"connectionString": " --- REPLACE WITH STORAGEACCOUNT BLOB CONNECTION STRING --- ",
"formatter": "Serilog.Formatting.Compact.RenderedCompactJsonFormatter, Serilog.Formatting.Compact",
"storageFileName": "{yyyy}/{MM}/{dd}/MyLogFile.txt",
"retainedFileCountLimit": 31
}
}
],
"Properties": {
"Application": "int-test-logging",
"Environment": "int"
}
}
}
I'm not sure what I'm doing incorrectly but any help would be appreciated. The following github repo contains code to implement the above (both approaches), and reproduces this behaviour. https://github.com/oneiltomlinson/AzureFunctionsLogging
Just add
"Using": [ "Serilog.Sinks.AzureBlobStorage" ] //under Serilog Config
and it will work!
Working Config
{
"Serilog": {
"Using": [
"Serilog.Sinks.AzureBlobStorage"
],
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Information"
}
},
"WriteTo": [
{
"Name": "AzureBlobStorage",
"Args": {
"connectionString": " --- REPLACE WITH STORAGEACCOUNT BLOB CONNECTION STRING --- ",
"formatter": "Serilog.Formatting.Compact.RenderedCompactJsonFormatter, Serilog.Formatting.Compact",
"storageFileName": "{yyyy}/{MM}/{dd}/MyLogFile.txt",
"retainedFileCountLimit": 31
}
}
],
"Properties": {
"Application": "int-test-logging",
"Environment": "int"
}
}
}
I've the same issue and even with all the required "using" of the used sinks, nothing change. This is my configuration file (just about the Serilog part):
"Serilog": {
"Using": [
"Serilog.Sinks.RollingFile",
"Serilog.Sinks.AzureTableStorage"
],
"MinimumLevel": "Information",
"Override": {
"Microsoft": "Warning",
"Microsoft.EntityFrameworkCore": "Information"
},
"WriteTo": [
{
"Name": "Logger",
"Args": {
"configureLogger": {
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] [TID:{ThreadId}] - {Message:lj}{NewLine}{Exception}"
},
{
"Name": "Debug",
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] [TID:{ThreadId}] - {Message:lj}{NewLine}{Exception}"
},
{
"Name": "AzureTableStorage",
"Args": {
"storageTableName": "HeroFunctionsLogs",
"connectionString": "DefaultEndpointsProtocol=https;AccountName=herobugari;AccountKey=NYJcdoCg9mQxqbQaTUdIxOHYQGdcKAwMsuWwzDub29UweHR1c+wd6Sh3IDQNB+eCx3DAe/ccobxu67sJvqQB5g==",
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] [TID:{ThreadId}] - {Message:lj}{NewLine}{Exception}",
"restrictedToMinimumLevel": "Information",
"writeInBatches": true,
"batchPostingLimit": 100
}
},
{
"Name": "RollingFile",
"Args": {
"pathFormat": "logs\\Bulgari.Hero.Functions.log",
"retainedFileCountLimit": 10,
"buffered": true,
"flushToDiskInterval": 5,
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff}] [{Level:u3}] [TID:{ThreadId}] - {Message:lj}{NewLine}{Exception}"
}
}
],
"Filter": [
{
"Name": "ByExcludingOnly",
"Args": {
"expression": "SourceContext like '%Microsoft.EntityFrameworkCore%'"
}
}
]
}
}
}
]
}
and this the Startup.cs code that reads and try to use it:
var config = new ConfigurationBuilder()
.SetBasePath(Environment.CurrentDirectory)
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables().Build();
// Registering Serilog provider
logger = new LoggerConfiguration()
//.WriteTo.Console()
//.WriteTo.File("logs\\herofunctions.log", rollingInterval: RollingInterval.Day)
.ReadFrom.Configuration(config)
.CreateLogger();
builder.Services.AddLogging(lb => lb.AddSerilog(logger));
The logger object has no "sinks" registered as it happens in the question.
I have solved exactly the same issue here by adding entries in local.settings.json.
"Serilog:Using:0": "Serilog.Sinks.Console",
"Serilog:MinimumLevel": "Information",
"Serilog:Override:Microsoft": "Warning",
"Serilog:Override:System": "Warning",
"Serilog:WriteTo:Console:Name": "Console"
then in StartUp.cs:
var provider = builder.Services.BuildServiceProvider();
var config = provider.GetRequiredService<IConfiguration>();
Log.Logger = new LoggerConfiguration().ReadFrom.Configuration(config).CreateLogger();
then the function:
public class TestFunction
{
private readonly ILogger _logger;
public TestFunction()
{
_logger = Log.Logger;
}
[FunctionName("TestFunction")]
public async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req)
{
_logger.Information("Hello");
return new OkResult();
}
}
}
I can see the config entries registered and entries are logged!
To write to SQL Server I had to add:
"Using": [
"Serilog.Sinks.MSSqlServer"
],
I'm using .net Core 2.0.9 and Serilog.Filters.Expressions 2.0.0.
I configured my appsettings.json to write in a log table in the Database.The data are recorded successfuly in the database but the RequestPath property is always null :
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Microsoft": "Debug"
}
},
"WriteTo": [
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "myconnectionString",
"tableName": "Log"
}
}
],
"WriteTo:Async": {
"Name": "Async",
"Args": {
"configure": [
{
"Name": "File",
"Args": {
"path": "..\\output\\log.txt",
"rollingInterval": "Day"
}
}
]
}
},
"Using": [ "Serilog.Settings.Configuration" ]
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "RequestPath like '%/api/book%'"
}
}
]
},
But i want to filter and save only log entries that have a specific api path. In this case, just entries that contain the api/user path in the RequestPath. But no data are saved anymore and i have no log errors, any idea why ?
Here are working steps for me, check the difference:
appsettings.json
"Serilog": {
"MinimumLevel": "Information",
"Override": {
"Microsoft": "Critical"
},
"WriteTo": [
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "Data Source=xx",
"autoCreateSqlTable ": true,
"tableName": "Logs",
"autoCreateSqlTable": true,
"columnOptionsSection": {
"removeStandardColumns": [ "Properties" ],
"customColumns": [
{
"ColumnName": "Release",
"DataType": "varchar",
"DataLength": 32
},
{
"ColumnName": "RequestPath",
"DataType": "varchar"
},
{
"ColumnName": "ConnectionId",
"DataType": "varchar"
}
]
}
}
},
{
"Name": "RollingFile",
"Args": {
"pathFormat": "Logs/app-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message} {UserName} {ActionName} {NewLine} {Exception}"
}
}
],
"Using": [ "Serilog.Settings.Configuration" ],
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "RequestPath like '%/api%'"
}
}
]
},
Startup.cs
Log.Logger = new LoggerConfiguration()
.ReadFrom.ConfigurationSection(Configuration.GetSection("Serilog"))
.CreateLogger();
For checking serilog error, adding code below:
Log.Logger = new LoggerConfiguration()
.ReadFrom.ConfigurationSection(Configuration.GetSection("Serilog"))
.CreateLogger();
Serilog.Debugging.SelfLog.Enable(msg =>
{
Debug.Print(msg);
Debugger.Break();
});