elasticsearch 来自logstash的Elasticsearch映射配置

我正在尝试从Windows上的

Logstash

为

Elasticsearch

配置索引模板管理

我有

c:\ulyaoth\logstash-2.3.1\bin\logstash.json

文件：

    input {
      beats {
       port => 5044
       type => "log"
      }
    }

    filter {
        grok {
            match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"]
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        mutate {
            remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
        }

        date {
            match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"]
            target => "timestamp_match"
        }
    }

    output {
      elasticsearch {
        hosts => "localhost:9200"
        index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
        document_type => "%{[@metadata][type]}"
        template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json"
        template_name => "elasticsearch-template"
        manage_template => true
        template_overwrite => true
      }
    }

以及模板文件

c:/ulyaoth/logstash-2.3.1/bin/elasticsearch template.custom.json

。我从

c:\ulyath\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\

中找到了这个文件，并对其进行了编辑，以便：

    "source":{"index": "not_analyzed"}

这是整个文件：

    {
      "template" : "logstash-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{"index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

我的问题是模板没有注册。REST查询返回空对象，我还看到Kibana中仍然分析的字段

    GET /_template HTTP/1.1
    Host: 127.0.0.1:9200

另一个问题是

remove\u字段

也不起作用-我仍然可以看到所有这些字段

    remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]

我没有看到任何日志存储日志（讽刺的是：），在ES日志中我没有看到任何错误或模板问题

如何解决这些问题

编辑：

最终工作配置为：

    {
      "template" : "filebeat-*",
      "settings" : {
        "index.refresh_interval" : "5s"
      },
      "mappings" : {
        "_default_" : {
          "_all" : {"enabled" : true, "omit_norms" : true},
          "dynamic_templates" : [ {
            "message_field" : {
              "match" : "message",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" }
              }
            }
          }, {
            "string_fields" : {
              "match" : "*",
              "match_mapping_type" : "string",
              "mapping" : {
                "type" : "string", "index" : "analyzed", "omit_norms" : true,
                "fielddata" : { "format" : "disabled" },
                "fields" : {
                  "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
                }
              }
            }
          }, {
            "float_fields" : {
              "match" : "*",
              "match_mapping_type" : "float",
              "mapping" : { "type" : "float", "doc_values" : true }
            }
          }, {
            "double_fields" : {
              "match" : "*",
              "match_mapping_type" : "double",
              "mapping" : { "type" : "double", "doc_values" : true }
            }
          }, {
            "byte_fields" : {
              "match" : "*",
              "match_mapping_type" : "byte",
              "mapping" : { "type" : "byte", "doc_values" : true }
            }
          }, {
            "short_fields" : {
              "match" : "*",
              "match_mapping_type" : "short",
              "mapping" : { "type" : "short", "doc_values" : true }
            }
          }, {
            "integer_fields" : {
              "match" : "*",
              "match_mapping_type" : "integer",
              "mapping" : { "type" : "integer", "doc_values" : true }
            }
          }, {
            "long_fields" : {
              "match" : "*",
              "match_mapping_type" : "long",
              "mapping" : { "type" : "long", "doc_values" : true }
            }
          }, {
            "date_fields" : {
              "match" : "*",
              "match_mapping_type" : "date",
              "mapping" : { "type" : "date", "doc_values" : true }
            }
          }, {
            "geo_point_fields" : {
              "match" : "*",
              "match_mapping_type" : "geo_point",
              "mapping" : { "type" : "geo_point", "doc_values" : true }
            }
          } ],
          "properties" : {
            "@timestamp": { "type": "date", "doc_values" : true },
            "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
            "source":{ "type": "string", "index": "not_analyzed"}
            "geoip"  : {
              "type" : "object",
              "dynamic": true,
              "properties" : {
                "ip": { "type": "ip", "doc_values" : true },
                "location" : { "type" : "geo_point", "doc_values" : true },
                "latitude" : { "type" : "float", "doc_values" : true },
                "longitude" : { "type" : "float", "doc_values" : true }
              }
            }
          }
        }
      }
    }

• 已更改
  

  “模板”：“文件节拍-*”

  ， 和

  “源”：{“类型”：“字符串”，“索引”：“未分析”}

---

源代码字段没有

类型。也许你的意思是：

"source":{ "type": "string", "index": "not_analyzed"},

也许您在“整个文件”qoute的第131行遗漏了一个“？”？（在geoip之前）？确实缺少一个逗号，但不幸的是它没有解决问题：（仍然没有模板…谢谢。你认为类型字段是必需的吗？我添加了“类型”：“字符串”，但得到了相同的结果。是的。类型是必需的。检查ES是否有模板，就像你在LS文件中保存模板的方式一样。