Parsing Protokoll mit grok, aber mit unterschiedlichen Zeilenformate in NodeJS

Question

Ich versuche, einige Protokolle mit grok zu analysieren, aber ein paar Probleme es zu tun habe, wenn die Protokollzeilen die gleiche nicht mal schauen ...

Meine log-Datei können sagen, sieht wie folgt aus:

[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log) 
[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log) 
[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log) 

Mein aktueller Knoten Code sieht wie folgt aus:

'use strict'; 

var nodegrok = require('node-grok'); 
var Regex = require("regex"); 
var zlib = require('zlib'); 

var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)' 

console.log('message: ', msg); 

var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 

var lines = msg.toString().split('\n'); 

for(var i = 0;i < lines.length;i++){ 

    console.log('line [i]:', lines[i]) 
    var str = lines[i] 

    var patterns = require('node-grok').loadDefaultSync(); 
    var pattern = patterns.createPattern(p2) 
    console.log('pattern:', pattern.parseSync(lines[i])); 

} 

aber die letzten beiden scheinen Ausgang null ... seit seiner den dritten Teil im Muster fehlt .

line [i]: [2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
pattern: { timestamp: '2017-02-03 19:15:51,112', 
    level: 'INFO', 
    message1: 'Group Metadata Manager on Broker 1', 
    message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' } 
line [i]: [2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager) 
pattern: { timestamp: '2017-02-03 19:25:51,112', 
    level: 'INFO', 
    message1: 'Group Metadata Manager on Broker 1', 
    message2: 'Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)' } 
line [i]: [2017-02-03 19:26:20,605] INFO Rolled new log segment for 'omega-replica-sync-dev-8' in 21 ms. (kafka.log.Log) 
pattern: null 
line [i]: [2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log) 
pattern: null 
line [i]: [2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log) 
pattern: null 

Wie können Sie Zeilen mit verschiedenen Formaten dann in grok formatieren?

Answer 1

also hier ist eine Möglichkeit, es zu tun, dass ich arbeiten musste ... im Wesentlichen zu sehen, ob das Muster mit einer if-Anweisung übereinstimmt und dann auswerten, aber was ist, wenn es 6 mögliche Formate des Protokolls gibt? Muss ich 6 Recht haben, wenn Aussagen, die dann verschachtelt sind? Klingt für mich nach einem effizienten Weg ... gibt es einen besseren Weg?

'use strict'; 

var nodegrok = require('node-grok'); 
var Regex = require("regex"); 
var zlib = require('zlib'); 

var msg = '[2017-02-03 19:15:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:25:51,112] INFO [Group Metadata Manager on Broker 1]: Removed 0 expired offsets in 0 milliseconds. (kafka.coordinator.GroupMetadataManager)\n[2017-02-03 19:26:20,605] INFO Rolled new log segment for \'omega-replica-sync-dev-8\' in 21 ms. (kafka.log.Log)\n[2017-02-03 19:26:20,605] INFO Scheduling log segment 1 for log omega-replica-sync-dev-8 for deletion. (kafka.log.Log)\n[2017-02-03 19:27:20,606] INFO Deleting segment 1 from log omega-replica-sync-dev-8. (kafka.log.Log)' 

console.log('message: ', msg); 

var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 

var lines = msg.toString().split('\n'); 

for(var i = 0;i < lines.length;i++){ 

    console.log('line [i]:', lines[i]) 
    var str = lines[i] 
    var p = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} \\[%{DATA:message1}\\]: %{GREEDYDATA:message2}' 
    var p2 = '\\[%{TIMESTAMP_ISO8601:timestamp}\\] %{LOGLEVEL:level} %{GREEDYDATA:message2}' 

    var patterns = require('node-grok').loadDefaultSync(); 
    var pattern = patterns.createPattern(p) 

    if (pattern.parseSync(lines[i]) == null) { 
     var pattern = patterns.createPattern(p2) 
     console.log('patternf:', pattern.parseSync(lines[i])); 

    } else { 
     console.log('pattern:', pattern.parseSync(lines[i])); 
    } 

}