Logging with winston-mongodb, express-winston and winston - node.js

I'm using winston, expressWinston, winston-mongoDB to manage the logs in my NodeJs application.
While using winston-mongodb transport with express-winston; i can store the logs to my mongoDB collection in the following format. (meta:true)
{
"_id" : ObjectId("5a124f9781d911337ebeb98d"),
"timestamp" : ISODate("2017-11-20T03:44:23.336Z"),
"level" : "info",
"message" : "GET /stylesheets/bootstrap.css 304 2ms",
"meta" : {
"res" : {
"statusCode" : 304
},
"req" : {
"url" : "/stylesheets/bootstrap.css",
"headers" : {
"host" : "localhost:3000",
"connection" : "keep-alive",
},
"method" : "GET",
"httpVersion" : "1.1",
"originalUrl" : "/stylesheets/bootstrap.css",
"query" : {}
},
"responseTime" : 2
}
I can get the request/response info in the meta.
Is it possible that these details be a part of the collection directly ? , something like this:
{
"_id" : ObjectId("5a12537d81b6b634cb7d4696"),
"timestamp" : ISODate("2017-11-20T04:01:01.229Z"),
"level" : "info",
"message" : "GET /stylesheets/bootstrap.css 304 11ms",
"status": 200,
"requestUrl": '/',
"requestMethod": 'GET',
"remoteIp": '::1'
"meta" : {}
}

you can override the default log function implementation to save whatever you want exactly
let logger = new winston.Logger(options);
logger.log = function () {
var args = arguments;
// here you can transform your meta obj
winston.Logger.prototype.log.apply(this, args);
};
Hope it will help you the idea

Related

VDM call to a function import returns null without throwing an exception

I am using VDM to query a function import. It returns null, which is not expected. It does not go to exception flow, so not able to figure out whether the call was a success or not.
PurchaseRequisitionHeaderBOView resultPojo = null;
try {
resultPojo = sspprService.purchaserequisition_WdActivation(purchaseRequisition, draftUUID, isActiveEntity)
.execute(new ErpConfigContext(s4destination));
if (resultPojo == null)
logger.error("VDM call returned null!");
} catch (ODataException e) {
logger.error(EXCEPTION_ACTIVATING_CART, e);
throw new org.apache.olingo.odata2.api.exception.ODataException(EXCEPTION_ACTIVATING_CART, e);
}
Expected:If call was a success a non-null POJO. If call failed it should hit the catch block.
Response of the function import
{
"d" : {
"__metadata" : {
"id" : "https://uyt700-er9003.wdf.sap.corp/sap/opu/odata/sap/MMPUR_REQ_GPR_MAINTAIN_SRV/I_Purchaserequisition_Wd(PurchaseRequisition='12444947',DraftUUID=guid'00000000-0000-0000-0000-000000000000',IsActiveEntity=true)",
"uri" : "https://uyt700-er9003.wdf.sap.corp/sap/opu/odata/sap/MMPUR_REQ_GPR_MAINTAIN_SRV/I_Purchaserequisition_Wd(PurchaseRequisition='12444947',DraftUUID=guid'00000000-0000-0000-0000-000000000000',IsActiveEntity=true)",
"type" : "MMPUR_REQ_GPR_MAINTAIN_SRV.I_Purchaserequisition_WdType",
"etag" : "W/\"20190920072451.9970020m\""
},
"Activation_ac" : false,
"Edit_ac" : true,
"Preparation_ac" : false,
"Validation_ac" : false,
"PurReqnDescription_fc" : 3,
"PurReqnSSPRequestor_fc" : 3,
"PurchaseRequisition" : "12444947",
"PurReqnSSPRequestor" : "00001076",
"PurReqnSSPAuthor" : "CSAC",
"PurchaseRequisitionType" : "NB",
"CreationDate" : "\/Date(1568937600000)\/",
"LastChangeDateTime" : "20190920072451.9970020",
"IsOnBehalfCart" : "",
"BusinessUser" : "CSAC",
"PurReqnIsCreatedInExpertMode" : false,
"PurReqnDescription" : "",
"EmployeeFullName" : "Don Draper",
"PurReqnOrigin" : "S",
"IsSrchEnabled" : false,
"PurReqnLifeCycleStatus" : "",
"PurReqnLifeCycleStatusName" : "",
"NumberOfItems" : 0,
"TotalNetAmount" : "0.00",
"Currency" : "",
"IsExtPurgScenario" : false,
"PurReqnIsCopyDraft" : false,
"HasDraftEntity" : false,
"DraftUUID" : "00000000-0000-0000-0000-000000000000",
"DraftEntityCreationDateTime" : null,
"DraftEntityLastChangeDateTime" : null,
"HasActiveEntity" : false,
"IsActiveEntity" : true,
"DraftAdministrativeData" : {
"__deferred" : {
"uri" : "https://uyt700-er9003.wdf.sap.corp/sap/opu/odata/sap/MMPUR_REQ_GPR_MAINTAIN_SRV/I_Purchaserequisition_Wd(PurchaseRequisition='12444947',DraftUUID=guid'00000000-0000-0000-0000-000000000000',IsActiveEntity=true)/DraftAdministrativeData"
}
},
"SiblingEntity" : {
"__deferred" : {
"uri" : "https://uyt700-er9003.wdf.sap.corp/sap/opu/odata/sap/MMPUR_REQ_GPR_MAINTAIN_SRV/I_Purchaserequisition_Wd(PurchaseRequisition='12444947',DraftUUID=guid'00000000-0000-0000-0000-000000000000',IsActiveEntity=true)/SiblingEntity"
}
}
}
}
Please update your dependency of SAP Cloud SDK to 3.6.0 as it fixes the issue of parsing OData function import responses. A customization is no longer required. Please let me know, if it works for you.

filebeat to logstash read json file with multiline

Trying to parse this multiline JSON file
{
"eventSource" : { "objectName": "SYSTEM.ADMIN.CHANNEL.EVENT",
"objectType" : "Queue" },
"eventType" : {
"name" : "Channel Event",
"value" : 46
},
"eventReason" : {
"name" : "Channel Blocked",
"value" : 2577
},
"eventCreation" : {
"timeStamp" : "2018/03/07 05:50:19.06 GMT",
"epoch" : 1520401819
},
"eventData" : {
"queueMgrName" : "QMG1",
"connectionName" : "localhost (192.168.10.1)",
"connectionNameList" : [
"localhost"
],
"reasonQualifier" : "Channel Blocked Noaccess",
"channelName" : "SVR.TEST",
"clientUserId" : "test1",
"applName" : "WebSphere MQ Client for Java",
"applType" : "Java"
}
}
filebeat is configured as
filebeat.prospectors:
- type: log
paths:
- /var/log/test2.log
fields:
tags: ['json']
logsource: mqjson
fields_under_root: true
input beats conf is as below.
input {
beats {
port => 5400
host => "192.168.205.11"
ssl => false
#ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
#ssl_key => "/etc/pki/tls/private/logstash-beats.key"
}
}
filter {
if [tags][json] {
json {
source => "message"
}
}
}
In elastic each line is a record.
Question:
How do i parse this multi line json
Also is there an option to extract certain keys, like eventData section.
by adding as below converts the json. There was an issue opened in elastic which was corrected in 6.0
processors:
- decode_json_fields:
fields: ['message']
target: json

How to retrieve the result of Pregel ArangoDB?

I am a newbie and using ArangoDB 3.2.5. I would like to retrieve the result of pregel in ArangoDB. Here is a simple example in arangosh.
var pregel = require("#arangodb/pregel");
var params = {source: "Country/Tunesia"};
var execution = pregel.start("sssp", "CountryGraph", params);
In the document, it dumps the result of pregel in a field in the collection. Is there an alternative way like the following code to retrieve the result? I look for it in the document but didn't find it. By the way, you can load the dataset example used by the above code by loading this. Thanks.
var result = pregel.getResult(execution);
The result can not be returned directly. You have two options however:
Let ArangoDB write the Pregel result into the documents
Don't store the result, but only hold it temporarily in memory
To avoid persisting the result, there is an option store which you have to set to false. You can access the volatile result with AQL through the function PREGEL_RESULT(<handle>).
The flow is like this:
Start Pregel execution
Check status and wait until it changes to "done" or "canceled"
Perform an AQL query to access the result if Pregel succeeded
var pregel = require("#arangodb/pregel");
var params = {source: "Country/Algeria", store: false};
var handle = pregel.start("sssp", "CountryGraph", params);
while (!["done", "canceled"].includes(pregel.status(handle).state)) {
print("waiting for result");
require("internal").wait(0.5); // TODO: make this more clever
}
var status = pregel.status(handle);
print(status);
if (status.state == "done") {
var query = db._query("FOR doc IN PREGEL_RESULT(#handle) RETURN doc", {handle: handle});
print(query.toArray());
}
The output looks like this:
shell>arangosh --javascript.execute pregel.js
waiting for result
{
"state" : "done",
"gss" : 2,
"totalRuntime" : 0.00583648681640625,
"aggregators" : {
},
"sendCount" : 1,
"receivedCount" : 1,
"vertexCount" : 10,
"edgeCount" : 8,
"code" : 200
}
[
{
"_key" : "Germany",
"result" : 9223372036854776000
},
{
"_key" : "Switzerland",
"result" : 9223372036854776000
},
{
"_key" : "Brasil",
"result" : 9223372036854776000
},
{
"_key" : "Marocco",
"result" : 9223372036854776000
},
{
"_key" : "Argentina",
"result" : 9223372036854776000
},
{
"_key" : "Austria",
"result" : 9223372036854776000
},
{
"_key" : "Algeria",
"result" : 0
},
{
"_key" : "Uruguay",
"result" : 9223372036854776000
},
{
"_key" : "Tunesia",
"result" : 1
},
{
"_key" : "Australia",
"result" : 9223372036854776000
}
]

how to find data in array key name nodejs from mongodb

how to find data from mongodb in nodejs i have below query but its return null, i want to find data in child_categories array Key name child_categoryname please help me i am very thanful to you
i am using nodejs i want to find 'truck repair' and in response i want to get child_categoryhrs in response related to truck repair
example = 4
JobCategories.findOne({ child_categories : { child_categoryname : 'truck repair'} })
.exec().then(result => { console.log(result); })
.catch(handleError(res));
//mongodb
{
"_id" : ObjectId("58cea64cfd70bb1aa472ef2c"),
"category_name" : "Repairing",
"date_updated" : ISODate("2017-03-20T12:04:16.323Z"),
"child_categories" : [
{
"child_categoryname" : "truck repair",
"child_categoryhrs" : 4,
"_id" : ObjectId("58cea64cfd70bb1aa472ef2d")
},
{
"child_categoryname" : "car repair",
"child_categoryhrs" : 8,
"_id" : ObjectId("58cfc5405895461b286238fa")
}
],
"__v" : 0
}

mongodb query on nodeJS say cannot read property remove of 'null'

This is my entry in database in mongodb which is of type object in schema
"_id" : ObjectId("5539bed4b417d75d1fee5df7"),
"favMovies" : {
"alternate_ids" : {
"imdb" : "2820852"
},
"studio" : "Universal Pictures",
"abridged_directors" : [
{
"name" : "James Wan"
}
],
"abridged_cast" : [
{
"characters" : [
"Dominic Toretto"
],
"id" : "162652472",
"name" : "Vin Diesel"
},
{
"characters" : [
"Brian O'Conner"
],
"id" : "162654234",
"name" : "Paul Walker"
},
{
"characters" : [
"Louie Tran"
],
"id" : "162684066",
"name" : "Tony Jaa"
},
{
"characters" : [
"Deckard Shaw"
],
"id" : "162653720",
"name" : "Jason Statham"
},
{
"characters" : [
"Luke Hobbs"
],
"id" : "770893686",
"name" : "Dwayne \"The Rock\" Johnson"
}
],
"synopsis" : "Continuing the global exploits in the unstoppable franchise built on speed, Vin Diesel, Paul Walker and Dwayne Johnson lead the returning cast of Fast & Furious 7. James Wan directs this chapter of the hugely successful series that also welcomes back favorites Michelle Rodriguez, Jordana Brewster, Tyrese Gibson, Chris \"Ludacris\" Bridges, Elsa Pataky and Lucas Black. They are joined by international action stars new to the franchise including Jason Statham, Djimon Hounsou, Tony Jaa, Ronda Rousey and Kurt Russell.",
"ratings" : {
"audience_score" : 88,
"audience_rating" : "Upright",
"critics_score" : 82,
"critics_rating" : "Certified Fresh"
},
"release_dates" : {
"theater" : "2015-04-03"
},
"critics_consensus" : "",
"runtime" : 140,
"mpaa_rating" : "PG-13",
"genres" : [
"Mystery & Suspense",
"Action & Adventure"
],
"year" : 2015,
"title" : "Furious 7",
"id" : 771354922
},
"username" : "punk",
"__v" : 0
}
In my Node JS code I use the following query
app.delete('/favMovies/:user/:movid',function(req, res){
var user = req.params.user;
var mid = req.params.movid;
console.log(mid);
console.log(user);
MovModel.find({username:user,'favMovies.id':mid}, function (err, doc) {
doc.remove();
MovModel.find({username: user},function (err, data) {   
res.json(data);
});
});
});
In the above snippet mid is movie id. For the above entry in database mov
"id" : 771354922
and user is username but I am getting following error for my query which is working fine in mongo client.
/Users/pankajtripathi/Documents/ECLIPSE-FILES/MyProject/server.js:132
doc.remove();
^
TypeError: Cannot read property 'remove' of null
at /Users/pankajtripathi/Documents/ECLIPSE-FILES/MyProject/server.js:132:5
at /Users/pankajtripathi/Documents/ECLIPSE-FILES/MyProject/node_modules/mongoose/lib/query.js:1169:16
at /Users/pankajtripathi/Documents/ECLIPSE-FILES/MyProject/node_modules/mongoose/node_modules/kareem/index.js:103:16
at process._tickCallback (node.js:355:11)
You should use findOneAndRemove()
MovModel.findOneAndRemove({username:user,'favMovies.id':mid}, function (err, doc) {
if (err) console.log(err);
res.json(doc);
}
Finds a matching document, removes it, passing the found document (if
any) to the callback. Executes immediately if callback is passed.
I changed the query and its working fine now.
MovModel.findOneAndRemove({username:user,_id:mid}, function (err, doc) {
console.log(doc);
MovModel.find({username: user},function (err, data) {
res.json(data);
});
});

Resources