The .Net core extension SeriLog filtering is not working - asp.net-core-2.0

I'm using .net Core 2.0.9 and Serilog.Filters.Expressions 2.0.0.
I configured my appsettings.json to write in a log table in the Database.The data are recorded successfuly in the database but the RequestPath property is always null :
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Microsoft": "Debug"
}
},
"WriteTo": [
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "myconnectionString",
"tableName": "Log"
}
}
],
"WriteTo:Async": {
"Name": "Async",
"Args": {
"configure": [
{
"Name": "File",
"Args": {
"path": "..\\output\\log.txt",
"rollingInterval": "Day"
}
}
]
}
},
"Using": [ "Serilog.Settings.Configuration" ]
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "RequestPath like '%/api/book%'"
}
}
]
},
But i want to filter and save only log entries that have a specific api path. In this case, just entries that contain the api/user path in the RequestPath. But no data are saved anymore and i have no log errors, any idea why ?

Here are working steps for me, check the difference:
appsettings.json
"Serilog": {
"MinimumLevel": "Information",
"Override": {
"Microsoft": "Critical"
},
"WriteTo": [
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "Data Source=xx",
"autoCreateSqlTable ": true,
"tableName": "Logs",
"autoCreateSqlTable": true,
"columnOptionsSection": {
"removeStandardColumns": [ "Properties" ],
"customColumns": [
{
"ColumnName": "Release",
"DataType": "varchar",
"DataLength": 32
},
{
"ColumnName": "RequestPath",
"DataType": "varchar"
},
{
"ColumnName": "ConnectionId",
"DataType": "varchar"
}
]
}
}
},
{
"Name": "RollingFile",
"Args": {
"pathFormat": "Logs/app-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message} {UserName} {ActionName} {NewLine} {Exception}"
}
}
],
"Using": [ "Serilog.Settings.Configuration" ],
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "RequestPath like '%/api%'"
}
}
]
},
Startup.cs
Log.Logger = new LoggerConfiguration()
.ReadFrom.ConfigurationSection(Configuration.GetSection("Serilog"))
.CreateLogger();
For checking serilog error, adding code below:
Log.Logger = new LoggerConfiguration()
.ReadFrom.ConfigurationSection(Configuration.GetSection("Serilog"))
.CreateLogger();
Serilog.Debugging.SelfLog.Enable(msg =>
{
Debug.Print(msg);
Debugger.Break();
});

Related

Azure Resource Manager Query with multiple dynamic tag filters

I'm trying to query the Azure Cost Management API and I want to be able to filter the results based off of 2 different types of resource tags but I am having trouble figuring out the format. I can get the single tag filter working, but I'm blanking on the format for multiple. Can anyone throw in their 2 cents?
Working single filter query:
{
"type": "Usage",
"timeframe": "{TimeFrame}",
"dataset": {
"granularity": "None",
"filter": {
"tags": {
"name": "Environment",
"operator": "In",
"values": [
{Environment}
]
}
},
"aggregation": {
"totalCost": {
"name": "PreTaxCost",
"function": "Sum"
}
},
"grouping": [
{
"type": "Dimension",
"name": "{Aggregation}"
}
]
}
}
My attempt at adding more than one filter:
{
"type": "Usage",
"timeframe": "{TimeFrame}",
"dataset": {
"granularity": "None",
"filter": {
"tags": [
{
"name": "Environment",
"operator": "In",
"values": [
{Environment}
]
},
{
"name": "Location",
"operator": "In",
"values": [
{Location}
]
}
]
},
"aggregation": {
"totalCost": {
"name": "PreTaxCost",
"function": "Sum"
}
},
"grouping": [
{
"type": "Dimension",
"name": "{Aggregation}"
}
]
}
}
I am very new to Azure so please don't roast me too hard lol.
Thank you to everyone who took a look at my question, much appreciated even if you don't have an answer for me.
There was an issue with the way my parameters were set causing a bad query. Here is the working code with multiple tag attributes for filtering:
{
"type": "Usage",
"timeframe": "{TimeFrame}",
"dataset": {
"granularity": "None",
"filter": {
"and": [
{
"tags": {
"name": "Location",
"operator": "In",
"values": [{LocationTag}]
}
},
{
"tags": {
"name": "Environment",
"operator": "In",
"Values": [{EnvironmentTag}]
}
},
{
"tags": {
"name": "Integrated-System",
"operator": "In",
"Values": [{IntegratedSystemTag}]
}
}
]
},
"aggregation": {
"totalCost": {
"name": "PreTaxCost",
"function": "Sum"
}
},
"grouping": [
{
"type": "Dimension",
"name": "{Aggregation}"
}
]
}
}

Cosmos Array_Contains Behavior in Different Partitions

In a Cosmos db, I have the following query:
SELECT
value app
FROM
Appointments app
where
app.PartitionKey = 'PartitionKey1'
and array_contains(app.Document.participant,
{
"type": [
{
"coding": [
{
"system": "http://hl7.org/fhir/v3/ParticipationType",
"code": "LOC",
"display": "Location"
}
]
}
]
}, true)
This works properly returning expected results.
However, if I remove the "system" and "display" items under "coding", making it
"coding": [
{
"code": "LOC"
}
]
the query works for Partition1 but not for Partition2, even though documents are structured the same, similar to:
[
{
"Document": {
"participant": [
{
"type": [
{
"coding": [
{
"system": "http://hl7.org/fhir/v3/ParticipationType",
"code": "LOC",
"display": "Location"
}
]
}
],
"actor": {
"reference": "Lo1",
"display": "Location1"
},
"required": "required"
},
{
"type": [
{
"coding": [
{
"system": "http://hl7.org/fhir/v3/ParticipationType",
"code": "SBJ",
"display": "Patient"
}
]
}
],
"actor": {
"reference": "Pa1",
"display": "Patient1"
},
"required": "required"
},
{
"type": [
{
"coding": [
{
"system": "http://hl7.org/fhir/v3/ParticipationType",
"code": "PRF",
"display": "Provider"
}
]
}
],
"actor": {
"reference": "Pr1",
"display": "Provider1"
},
"required": "required"
}
]
},
"PartitionKey": "Partition1"
}
]
The mysterious part is why this would work with one partition key but not another. It's almost as if one partition respects the true parameter of array_contains but the other partition does not.
Is there a setting on a partition, or some other property, that changes the behavior of array_contains?

Using outputTemplate argument in Serilog with Azure Application Insights

I'm currently implementing Azure Application Insights logging with Serilog which is working fine except for when I use an output template in my Serilog configuration. It seems like the template is ignored when passing the Serilog data to Application insights.
My serilog config in appsetting.json:
"Serilog": {
"Using": [ "Serilog.Sinks.ApplicationInsights" ],
"MinimumLevel": "Debug",
"WriteTo": [
{ "Name": "Console" },
{
"Name": "RollingFile",
"Args": {
"pathFormat": "logs\\log-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}"
}
},
{
"Name": "ApplicationInsights",
"Args": {
"restrictedToMinimumLevel": "Information",
"telemetryConverter": "Serilog.Sinks.ApplicationInsights.Sinks.ApplicationInsights.TelemetryConverters.TraceTelemetryConverter, Serilog.Sinks.ApplicationInsights",
"outputTemplate": "Test Template - {Message}"
}
}
],
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ],
"Properties": {
"Application": "app"
}
},
The logging statement:
logger.Error("Test Serilog Error For AI - " + DateTime.Now);
The output within application insights:
Is this the correct approach to customising an error message for Application insights?
After checking the source code serilog-sinks-applicationinsights, you will find it did not read the outputTemplate from appsetting.json.
For a workaround, you may implement custom TemplateTraceTelemetryConverter.
TemplateTraceTelemetryConverter
public class TemplateTraceTelemetryConverter : TraceTelemetryConverter
{
public override IEnumerable<ITelemetry> Convert(LogEvent logEvent, IFormatProvider formatProvider)
{
var templateParser = new MessageTemplateParser();
var template = templateParser.Parse($"Test Template - {logEvent.MessageTemplate.Text}");
LogEvent newLogEvent = new LogEvent(logEvent.Timestamp
, logEvent.Level
, logEvent.Exception
, template
, logEvent.Properties.Select(p => new LogEventProperty(p.Key, p.Value)));
return base.Convert(newLogEvent, formatProvider);
}
}
Use TemplateTraceTelemetryConverter
"Serilog": {
"Using": [
"Serilog.Sinks.ApplicationInsights",
],
"MinimumLevel": "Debug",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "Test Template - {Message}"
}
},
{
"Name": "RollingFile",
"Args": {
"pathFormat": "logs\\log-{Date}.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}"
}
},
{
"Name": "ApplicationInsights",
"Args": {
"restrictedToMinimumLevel": "Error",
"telemetryConverter": "YourProjectNamespace.TemplateTraceTelemetryConverter, YourProjectNamespace"
//"outputTemplate": "Test Template - {Message}"
}
}
],
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ],
"Properties": {
"Application": "app"
}
}

Configure log4js logger to log only INFO to console and everything to file

With below-mentioned log4js configuration, I intend to log everything to a file in logs/test/log-timestamp.txt file and only INFO level to console, but I see that the console prints all the levels. How do I fix this?
{
"appenders": {
"test": {
"type": "dateFile",
"filename": "logs/test/log",
"pattern": "-yyyy-MM-dd-hh-mm-ss.txt",
"alwaysIncludePattern": true,
"keepFileExt": true
},
"test-filter": {
"type": "logLevelFilter",
"appender": "test",
"level": "trace",
"maxLevel": "fatal"
},
"default": {
"type": "dateFile",
"filename": "logs/default",
"pattern": "-yyyy-MM-dd.txt",
"alwaysIncludePattern": true,
"keepFileExt": true
},
"console": {
"type": "console",
"level": "info",
"maxLevel": "info"
}
},
"categories": {
"default": {
"appenders": [
"default"
],
"level": "info"
},
"test": {
"appenders": [
"test-filter",
"console"
],
"level": "debug"
}
}
}

ElasticSearch query stops working with big amount of data

The problem: I have 2 identical in terms of settings and mappings indexes.
The first index contains only 1 document.
The second index contains the same document + 16M of others.
When I'm running the query on the first index it returns the document, but when I do the same query on the second — I receive nothing.
Indexes settings:
{
"tasks_test": {
"settings": {
"index": {
"analysis": {
"analyzer": {
"tag_analyzer": {
"filter": [
"lowercase",
"tag_filter"
],
"tokenizer": "whitespace",
"type": "custom"
}
},
"filter": {
"tag_filter": {
"type": "word_delimiter",
"type_table": "# => ALPHA"
}
}
},
"creation_date": "1444127141035",
"number_of_replicas": "2",
"number_of_shards": "5",
"uuid": "wTe6WVtLRTq0XwmaLb7BLg",
"version": {
"created": "1050199"
}
}
}
}
}
Mappings:
{
"tasks_test": {
"mappings": {
"Task": {
"dynamic": "false",
"properties": {
"format": "dateOptionalTime",
"include_in_all": false,
"type": "date"
},
"is_private": {
"type": "boolean"
},
"last_timestamp": {
"type": "integer"
},
"name": {
"analyzer": "tag_analyzer",
"type": "string"
},
"project_id": {
"include_in_all": false,
"type": "integer"
},
"user_id": {
"include_in_all": false,
"type": "integer"
}
}
}
}
}
The document:
{
"_index": "tasks_test",
"_type": "Task",
"_id": "1",
"_source": {
"is_private": false,
"name": "135548- test with number",
"project_id": 2,
"user_id": 1
}
}
The query:
{
"query": {
"filtered": {
"query": {
"bool": {
"must": [
[
{
"match": {
"_all": {
"query": "135548",
"type": "phrase_prefix"
}
}
}
]
]
}
},
"filter": {
"bool": {
"must": [
{
"term": {
"is_private": false
}
},
{
"terms": {
"project_id": [
2
]
}
},
{
"terms": {
"user_id": [
1
]
}
}
]
}
}
}
}
}
Also, some findings:
if I replace _all with name everything works
if I replace match_phrase_prefix with match_phrase works too
ES version: 1.5.1
So, the question is: how to make the query work for the second index without mentioned hacks?

Resources