I'm trying to parse some logs with grok but am having some trouble doing it when the log lines don't look the same sometimes...
My log file lets say looks like this:
[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)
[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)
[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)
[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)
[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)
My current node code looks like this:
'use strict';
var nodegrok = require('node-grok');
var Regex = require("regex");
var zlib = require('zlib');
var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)'
console.log('message: ', msg);
var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}'
var lines = msg.toString().split('\n');
for(var i = 0;i < lines.length;i++){
console.log('line [i]:', lines[i])
var str = lines[i]
var patterns = require('node-grok').loadDefaultSync();
var pattern = patterns.createPattern(p2)
console.log('pattern:', pattern.parseSync(lines[i]));
}
but the last two seem to output null...since its missing the 3rd part in the pattern.
line [i]: [2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)
pattern: { timestamp: '2017-02-03 19:15:51,112',
level: 'INFO',
message1: 'Group Metadata Manager on Broker 1',
message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' }
line [i]: [2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)
pattern: { timestamp: '2017-02-03 19:25:51,112',
level: 'INFO',
message1: 'Group Metadata Manager on Broker 1',
message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' }
line [i]: [2017-02-03 19:26:20,605] INFO Rolled new log segment for 'omega-replica-sync-dev-8' in 21 ms. (kafka.log.Log)
pattern: null
line [i]: [2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)
pattern: null
line [i]: [2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)
pattern: null
How can you format lines with varying formats then in grok?
so here is one way of doing it that I got to work...essentially looking to see if the pattern matches with an if statement and then evaluating but, what if there are 6 potential formats of the log? Do I have to right 6 if statements that are nested then? Does sound like an efficient way to me...is there a better way?
'use strict';
var nodegrok = require('node-grok');
var Regex = require("regex");
var zlib = require('zlib');
var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)'
console.log('message: ', msg);
var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}'
var lines = msg.toString().split('\n');
for(var i = 0;i < lines.length;i++){
console.log('line [i]:', lines[i])
var str = lines[i]
var p = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}'
var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} %{GREEDYDATA:message2}'
var patterns = require('node-grok').loadDefaultSync();
var pattern = patterns.createPattern(p)
if (pattern.parseSync(lines[i]) == null ) {
var pattern = patterns.createPattern(p2)
console.log('patternf:', pattern.parseSync(lines[i]));
} else {
console.log('pattern:', pattern.parseSync(lines[i]));
}
}
Related
I have an azure app service that reads from a azure storage queue through camel route version 3.14.0. Below is my code:
queue code:
QueueServiceClient client = new QueueServiceClientBuilder()
.connectionString(storageAccountConnectionString)
.buildClient();
getContext().getRegistry().bind("client", client);
errorHandler(deadLetterChannel(SEND_TO_POISON_QUEUE)
.useOriginalBody()
.log("Message sent to poison queue for handling")
.retryWhile(method(new RetryRuleset(), "shouldRetry"))
.maximumRedeliveries(24)
.asyncDelayedRedelivery()
.redeliveryDelay(3600 * 1000L) // initial delay
);
// Route to retrieve a message from storage queue.
from("azure-storage-queue:" + storageAccountName + "/" + QUEUE_NAME + "?serviceClient=#client&maxMessages=1&visibilityTimeout=P2D")
.id(QUEUE_ROUTE_CONSUMER)
.log("Message received from queue with messageId: ${headers.CamelAzureStorageQueueMessageId} and ${headers.CamelAzureStorageQueueInsertionTime} in UTC")
.bean(cliFacilityService, "processMessage(${body}, ${headers.CamelAzureStorageQueueInsertionTime})")
.end();
RetryRuleset code:
public boolean shouldRetry(#Header(QueueConstants.MESSAGE_ID) String messageId,
#Header(Exchange.REDELIVERY_COUNTER) Integer counter,
#Header(QueueConstants.INSERTION_TIME) OffsetDateTime insertionTime) {
OffsetDateTime futureRetryOffsetDateTime = OffsetDateTime.now(Clock.systemUTC()).plusHours(1); //because redelivery delay is 1hr
OffsetDateTime insertionTimePlus24hrs = insertionTime.plusHours(24);
if (futureRetryOffsetDateTime.isAfter(insertionTimePlus24hrs)) {
log.info("Facility queue message: {} done retrying because next time to retry {}. Redelivery count: {}, enqueue time: {}",
messageId, futureRetryOffsetDateTime, counter, insertionTime);
return false;
}
return true;
}
the redeliveryDelay is 1hr and maximumRedeliveries is 24, because i want to try once an hour for about 24 hrs. so not necessarily needs to be 24 times, just as many as it can do with 24hrs. and if it passes 24hrs, send it to the poison queue (this code is in the retry ruleset)
The problem is the app service retrying for first lets say 2 - 5 times normally once an hour. but after that the app service retries after 2 days later. So the message is expired and not retried because of the ruleset and sent to poison queue. Sometimes the app service does the first read from queue and the next retry is after 2 days. so very unstable. so total it is retrying tops 1-10 times and the last retry is always 2 days later in the same time from the first retry.
Is there anything i am doing wrong?
Thank you for you help!
I'm new here and try to build my webaplication with node.js to read and write modbus data from a door operator.
So far I have successfully used modbus-serial package to send and receive data with function code 3 and 16.
var ModbusRTU = require("modbus-serial");
var client = new ModbusRTU();
client.connectRTUBuffered("/dev/ttyUSB0", { baudRate : 9600, dataBits : 8, stopBits : 1, parity : "none", });
client.setID(2);
socket.on('op_mode', function (data, err) {
if (err){
console.log(err);
} else {
console.log('op_mode : ' + data);
client.writeRegisters(20, [data, 0]); //data is a number between 1 and 6
}
});
This represents the equivalent of typing "2 16 0 20 0 2 4 0 2 0 0" into Realterm on Windows.
With "2 3 0 20 0 2" I'm able to read data with Realterm. An the equivalent in node.js is:
client.readHoldingRegisters(20, 2, function(err, data) {
if (err){
console.log(err);
} else {
console.log('Reg 20: ' + data.data[0]);
}
});
Now I need to send the command "2 100 252 249 0 0" with node.js which I don't know how to do as the function code 100 is not present. The same command in Realterm is working fine.
For me it would also be ok to use the SerialPort package but need some help to get the code right to receive an answer from my operator.
Thanks in advance.
Additional information from 08.02.2021
Message information from documentation:
Slave address Byte 0 = 2
Function code Byte 1 = 100
Byte 2 = 252
Byte 3 = 249
Byte 4 = 0
Byte 5 = 0
Modbus16 CRC must be added. Response has the same structure.
Sometimes, kafka-node consumer starts consuming from offset 0, while its default behavior it to consume only newer messages. Then it will not switch back to its default behavior. Do you know how to solve this and what happens and its behavior suddenly changes? The code is very simple and this happens without altering the code.
var kafka = require("kafka-node") ;
Consumer = kafka.Consumer;
client = new kafka.KafkaClient();
consumer = new Consumer(client, [{ topic: "Topic_23", partition: 0}
]);
consumer.on("message", function(message) {
console.log(message)
});
The only solution I have found so far is to change the kafka topic. Then everything works fine again. Any ideas ?
In Kafka, offsets are not associated to specific consumers but instead, they are linked to the Consumer Groups. In your code, you don't provide the Consumer Group therefore, every time you fire up the consumer, it is being assigned to a different Consumer Group and thus, the offset starts from 0.
The following should do the trick (obviously the first time you are going to read all the messages):
var kafka = require("kafka-node") ;
Consumer = kafka.Consumer;
client = new kafka.KafkaClient();
payload = [{
topic: "Topic_23",
partition: 0
}]
var options = {
groupId: 'test-consumer-group',
fromOffset: 'latest'
};
consumer = new Consumer(client, payload, options);
consumer.on("message", function(message) {
console.log(message)
});
I have java application in which a Quartz Task runs every 300ms and try to retrieve a number of messages from ActiveMQ Queue let's say 5 and process them each requested bulk needs about 1 minute to be processed to free the Quartz Scheduler worker
All threads use the same connection which is locked by a synchronized block on the receive method like this:
protected List<?> receive(int bulkSize, String queueName, long receiveTimeoutMillis) {
LinkedList messages = new LinkedList();
try {
QueueReceiver receiver = (QueueReceiver)this.receivers.get(queueName);
if (receiver != null) {
ObjectMessage message = null;
int index = 1;
do {
message = (ObjectMessage)receiver.receive(receiveTimeoutMillis);
if (message != null) {
messages.add(message.getObject());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Message received from: " + receiver.getQueue().getQueueName() + " jms message id:" + message.getJMSMessageID());
}
message.acknowledge();
}
if (message == null) {
break;
}
++index;
} while(index <= bulkSize);
LOGGER.info("Consumed " + (index - 1) + " messages from " + queueName + ", elapsed time: " + stopWatch.getTime() + " ms");
} else {
LOGGER.warn("Queue not found " + queueName);
}
} catch (Exception var10) {
LOGGER.warn("error in performing receive: " + var10.getMessage(), var10);
}
return messages;
}
receiveTimeoutMillis is always 150 ms , the bulkSize is 5 as I said.
when I start my application each thread uses this method gets a bulk of 5 messages but after a couple of minutes they all start to get 0 to 2 messages although the Queue is Full with messages
I don't have any clue why this is happening!,so please give me any hint to check it!
Notes
even though I increase the timeout of the receive method from the Queue still the receiver doesn't get any messages!
in the log I see the following messages from each thread running the above code:
2019-01-29 14:59:23,893 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-2] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:24,329 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-2] Consumed 1 messages from TRANSLATOR, elapsed time: 282 ms
2019-01-29 14:59:24,500 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:24,793 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:25,097 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-18] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:25,403 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-18] Consumed 1 messages from TRANSLATOR, elapsed time: 153 ms
2019-01-29 14:59:25,693 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:25,996 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:26,300 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:26,595 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:26,898 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:27,193 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
2019-01-29 14:59:27,496 INFO com.peer39.commons.pattern.jms.JMSService - [DefaultQuartzScheduler_Worker-15] Consumed 0 messages from TRANSLATOR, elapsed time: 150 ms
I am adding the daemon thread that runs every 300 ms
public class Daemon extends ParallelQuartzDaemon {
#Override
protected ICollectorAgent.ProcessStatus executeWork(JobExecutionContext jobExecutionContext,
Map<String, Double> properties,
Map<String, String> alerts) throws Exception {
ICollectorAgent.ProcessStatus processStatus = ICollectorAgent.ProcessStatus.SUCCESS;
List<FlowRequest> flowRequests = getFlowRequestsForTranslation();
if (!flowRequests.isEmpty()) {
//DO Work! //takes 1-2 minutes!
}
return processStatus;
}
private List<FlowRequest> getFlowRequestsForTranslation() {
translatorContext.getTranslatorJMSService().getFlowRequestsToTranslate(5);
}
}
and the JMS class:
public class TranslatorJMSService extends JMSService {
public List<FlowRequest> getFlowRequestsToTranslate(int count) {
final Long jmsReceiveTimeoutMillis = translatorConfiguration.getJmsReceiveTimeoutMillis();
return (List<FlowRequest>) receive(count, queueName, jmsReceiveTimeoutMillis);
}
}
and last, the receive method is mentioned above.
thanks!
I am writing this block device driver. From the logs I can see that all requests the block has received were served.
Here is the code for fetching requests from the queue:
/* Gets the current request from the dispatch queue */
while ((req = blk_fetch_request(q)) != NULL) {
start_sector = blk_rq_pos(req);
sector_cnt = blk_rq_sectors(req);
dir = rq_data_dir(req);
printk("Received a %s request starting at %d and length %d sectors!\n",
(dir == READ ? "READ" : "WRITE"), start_sector, sector_cnt);
kthread_run(request_handler_thread, req, "request_handler");
//ret = rb_transfer(req);
}
And here is the code for ending the requests:
int start_sector;
int sector_cnt;
int dir;
int error = req->errors ? -EIO : 0;
start_sector = blk_rq_pos(req);
sector_cnt = blk_rq_sectors(req);
dir = rq_data_dir(req);
printk("Ending a %s request starting at %d and length %d sectors!\n",
(dir == READ ? "READ" : "WRITE"), start_sector, sector_cnt);
__blk_end_request_cur(req, error);
From the log messages, I can see there is one-to-one match between fetched and ended block requests. However, I can see that the process is hung and cat /proc/diskstats reports pending requests.