elasticsearch Logstash->；Elasticsearch未正确映射

因此，我最近创建了一个麋鹿集群，使用此网站作为模板

我遇到了一个问题，Logstash处理节点上的json模板没有在实际的Elasticsearch数据节点上使用。我可以看到映射已经在HQ中创建，但另一个映射是使用一些动态创建的映射创建的。正确完成的映射在数据节点上称为“Sourcefire”，但它也创建了一个称为“sourcfire”的映射

我无法理解这一点，我正在学习这方面的知识，因此非常感谢您的帮助。请参阅下面的代码片段

Logstash.conf

input {
    tcp {
        port => 5170
        type => "sourcefire"
    }
}

filter {

    mutate{
        split => ["message", "|"]
        add_field => {
            "event" => "%{message[5]}"
            "eventSource" => "%{message[1]}"
        }
    }

    kv {
        include_keys => ["dhost", "dst", "dpt", "shost", "src", "spt", "rt"]
    }

    mutate {
        rename => [ "dhost", "destinationHost" ]
        rename => [ "dst", "destinationAddress" ]
        rename => [ "dpt", "destinationPort" ]
        rename => [ "shost", "sourceHost" ]
        rename => [ "src", "sourceAddress" ]
        rename => [ "spt", "sourcePort" ]
    }

    date {
        match => ["rt","UNIX_MS"]
        target => "eventDate"
    }

    geoip {
        add_tag => [ "sourceGeo" ]
        source => "src"
        database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
    }

    geoip {
        add_tag => [ "destinationGeo" ]
        source => "src"
        database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
    }
}

output {
    if [type] == "sourcefire" {
        elasticsearch {
            cluster => "XXX-cluster"
            flush_size => 1
            manage_template => true
            template => "/opt/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-sourcefire.json"
        }
    }
}

Elasticsearch json模板

{
    "template": "logstash-*",
    "settings": {
        "index.refresh_interval": "5s"
    },
    "mappings": {
        "Sourcefire": {
            "_all": {
                "enabled": true
            },
            "properties": {
                "@timestamp": {
                    "type": "date",
                    "format": "basicDateTimeNoMillis"
                },
                "@version": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "geoip": {
                    "type": "object",
                    "dynamic": true,
                    "path": "full",
                    "properties": {
                        "location": {
                            "type": "geo_point"
                        }
                    }
                },
                "event": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "eventDate": {
                    "type": "date",
                    "format": "basicDateTimeNoMillis"
                },
                "destinationAddress": {
                    "type": "ip"
                },
                "destinationHost": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "destinationPort": {
                    "type": "integer",
                    "index": "not_analyzed"
                },
                "sourceAddress": {
                    "type": "ip"
                },
                "sourceHost": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "sourcePort": {
                    "type": "integer",
                    "index": "not_analyzed"
                }
            }
        }
    }
}

---

您可以使用elasticsearch输出的template_overwrite属性。但是，它不能保证总是正常工作，特别是当您有多个同时工作的日志存储实例时。此外，根据elasticsearch映射配置设置，特别是动态映射和默认设置（），您可能会得到与预期不同的结果

---

根据我的经验，我发现最好在elasticsearch中手动控制索引映射（使用fiddler或elasticsearch head management site等工具）。这是因为当多个logstash实例一起重写映射时，我遇到了各种意外结果，禁用了我设置的特殊elasticsearch字段（如_ttl）。

只需删除存储的模板，它就会重新创建它： 例如，如果模板名称为

logstash

：

curl-XDELETE localhost:9200/\u template/logstash

此外，如果要写入同一索引，则无法更改映射。您需要重新创建索引（确保首先停止日志存储，以防止任何飞行）