Elasticsearch Mapping-Konfiguration von Logstash

Question

Ich versuche, Index Vorlage-Management für Elasticsearch von Logstash unter Windows zu konfigurieren.

Ich habe c:\ulyaoth\logstash-2.3.1\bin\logstash.json Datei:

input { 
     beats { 
     port => 5044 
     type => "log" 
     } 
    } 

    filter { 
     grok { 
      match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"] 
      remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 
     } 

     mutate { 
      remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 
     } 

     date { 
      match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"] 
      target => "timestamp_match" 
     } 
    } 

    output { 
     elasticsearch { 
     hosts => "localhost:9200" 
     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" 
     document_type => "%{[@metadata][type]}" 
     template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json" 
     template_name => "elasticsearch-template" 
     manage_template => true 
     template_overwrite => true 
     } 
    } 

und die Vorlagendatei c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json. Ich grub diese Datei von c:\ulyaoth\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\ und bearbeitet sie so, dass:

"source":{"index": "not_analyzed"} 

Dies ist die gesamte Datei:

{ 
     "template" : "logstash-*", 
     "settings" : { 
     "index.refresh_interval" : "5s" 
     }, 
     "mappings" : { 
     "_default_" : { 
      "_all" : {"enabled" : true, "omit_norms" : true}, 
      "dynamic_templates" : [ { 
      "message_field" : { 
       "match" : "message", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" } 
       } 
      } 
      }, { 
      "string_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" }, 
       "fields" : { 
        "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256} 
       } 
       } 
      } 
      }, { 
      "float_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "float", 
       "mapping" : { "type" : "float", "doc_values" : true } 
      } 
      }, { 
      "double_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "double", 
       "mapping" : { "type" : "double", "doc_values" : true } 
      } 
      }, { 
      "byte_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "byte", 
       "mapping" : { "type" : "byte", "doc_values" : true } 
      } 
      }, { 
      "short_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "short", 
       "mapping" : { "type" : "short", "doc_values" : true } 
      } 
      }, { 
      "integer_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "integer", 
       "mapping" : { "type" : "integer", "doc_values" : true } 
      } 
      }, { 
      "long_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "long", 
       "mapping" : { "type" : "long", "doc_values" : true } 
      } 
      }, { 
      "date_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "date", 
       "mapping" : { "type" : "date", "doc_values" : true } 
      } 
      }, { 
      "geo_point_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "geo_point", 
       "mapping" : { "type" : "geo_point", "doc_values" : true } 
      } 
      } ], 
      "properties" : { 
      "@timestamp": { "type": "date", "doc_values" : true }, 
      "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 
      "source":{"index": "not_analyzed"} 
      "geoip" : { 
       "type" : "object", 
       "dynamic": true, 
       "properties" : { 
       "ip": { "type": "ip", "doc_values" : true }, 
       "location" : { "type" : "geo_point", "doc_values" : true }, 
       "latitude" : { "type" : "float", "doc_values" : true }, 
       "longitude" : { "type" : "float", "doc_values" : true } 
       } 
      } 
      } 
     } 
     } 
    } 

Mein Problem ist, dass die Vorlage nicht registriert ist. Die REST-Abfrage gibt ein leeres Objekt zurück. Außerdem sehe ich, dass das Feld in Kibana noch analysiert wird.

GET /_template HTTP/1.1 
    Host: 127.0.0.1:9200 

weiteres Problem ist, dass remove_field auch nicht funktioniert - ich immer noch all diese Felder sehen.

remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"] 

Ich sehe keine logstash Protokolle (ironisch :) und in ES-Protokolle Ich sehe keine Fehler oder eine Vorlage Probleme.

Wie können diese Probleme gelöst werden?

EDIT:

Der letzte Arbeitskonfiguration ist:

{ 
     "template" : "filebeat-*", 
     "settings" : { 
     "index.refresh_interval" : "5s" 
     }, 
     "mappings" : { 
     "_default_" : { 
      "_all" : {"enabled" : true, "omit_norms" : true}, 
      "dynamic_templates" : [ { 
      "message_field" : { 
       "match" : "message", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" } 
       } 
      } 
      }, { 
      "string_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "string", 
       "mapping" : { 
       "type" : "string", "index" : "analyzed", "omit_norms" : true, 
       "fielddata" : { "format" : "disabled" }, 
       "fields" : { 
        "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256} 
       } 
       } 
      } 
      }, { 
      "float_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "float", 
       "mapping" : { "type" : "float", "doc_values" : true } 
      } 
      }, { 
      "double_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "double", 
       "mapping" : { "type" : "double", "doc_values" : true } 
      } 
      }, { 
      "byte_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "byte", 
       "mapping" : { "type" : "byte", "doc_values" : true } 
      } 
      }, { 
      "short_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "short", 
       "mapping" : { "type" : "short", "doc_values" : true } 
      } 
      }, { 
      "integer_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "integer", 
       "mapping" : { "type" : "integer", "doc_values" : true } 
      } 
      }, { 
      "long_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "long", 
       "mapping" : { "type" : "long", "doc_values" : true } 
      } 
      }, { 
      "date_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "date", 
       "mapping" : { "type" : "date", "doc_values" : true } 
      } 
      }, { 
      "geo_point_fields" : { 
       "match" : "*", 
       "match_mapping_type" : "geo_point", 
       "mapping" : { "type" : "geo_point", "doc_values" : true } 
      } 
      } ], 
      "properties" : { 
      "@timestamp": { "type": "date", "doc_values" : true }, 
      "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 
      "source":{ "type": "string", "index": "not_analyzed"} 
      "geoip" : { 
       "type" : "object", 
       "dynamic": true, 
       "properties" : { 
       "ip": { "type": "ip", "doc_values" : true }, 
       "location" : { "type" : "geo_point", "doc_values" : true }, 
       "latitude" : { "type" : "float", "doc_values" : true }, 
       "longitude" : { "type" : "float", "doc_values" : true } 
       } 
      } 
      } 
     } 
     } 
    } 

• 
  "template" : "filebeat-*", geändert und "source":{ "type": "string", "index": "not_analyzed"}

Answer 1

Die source Fiel d hat keine type. Vielleicht meintest du:

"source":{ "type": "string", "index": "not_analyzed"},