elasticsearch Logstash geoip随机故障

我正在使用logstash处理来自dnsmasq的日志。 在DNS响应中，我使用“geoip”过滤器用位置信息丰富请求。不幸的是，有些请求包含地理位置信息，而其他请求则不包含

地理位置代码： 不带地理位置的示例IP列表 成功进入： 输入失败： 完整日志存储配置：

---

最新的Logstash版本（2.3.2）捆绑了2.0.7，而2.0.7又包含了一个旧的Maxmind的GeoIP数据库，从调试日志中可以看出，该数据库来自2013年（

GeoLiteCity-2013-01-18.dat

）

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"}

Maxmind有一个新的GeoIP2服务，您可以尝试，它将正确地对上面列表中的所有IP进行地理编码

---

就Logstash

geoip

而言，从geoip到GeoIP2的切换将在.

OMG中提供。这个回答让我明白了。在我的例子中，我试图解析日志中的10.101.xxx.xxx地址。我总是发现geoip查找失败。。。这是有原因的。这些是私人地址。@Wexoni是的，Logstash无法对私人IP地址进行地理编码。

104.156.81.217
104.156.85.217
104.16.92.65
104.16.93.65
104.16.94.65
104.16.95.65
104.16.96.65
104.20.5.131
104.20.6.131
104.20.77.18
104.20.78.18
104.244.43.135
104.244.43.167
104.244.43.231
104.244.43.39
104.244.43.7
104.28.30.27
104.28.31.27
104.40.196.5
104.41.231.130
104.45.95.112
104.47.151.128
104.71.97.80
104.84.200.206
104.90.129.122
104.90.176.199
104.90.176.77
104.94.60.210
104.98.119.204
104.98.150.212
162.255.119.124
185.118.208.20
185.19.196.101
185.54.150.54
185.63.147.12
191.232.139.13
191.233.80.151
191.239.8.125
192.229.233.25
23.101.51.170
23.196.235.245
23.196.247.114
23.196.249.86
23.196.255.139
23.197.0.60
23.199.209.223
23.235.33.217
23.235.37.217
23.97.173.24

{
         "message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:17:17.147Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:17:16",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "www.google.com",
        "function" => "is",
        "serverip" => "216.58.213.228",
           "geoip" => {
                      "ip" => "216.58.213.228",
           "country_code2" => "US",
           "country_code3" => "USA",
            "country_name" => "United States",
          "continent_code" => "NA",
             "region_name" => "CA",
               "city_name" => "Mountain View",
             "postal_code" => "94043",
                "latitude" => 37.41919999999999,
               "longitude" => -122.0574,
                "dma_code" => 807,
               "area_code" => 650,
                "timezone" => "America/Los_Angeles",
        "real_region_name" => "California",
                "location" => [
            [0] -122.0574,
            [1] 37.41919999999999
        ]
    }
}

{
         "message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:15:51.697Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:15:50",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "e5884.d.akamaiedge.net",
        "function" => "is",
        "serverip" => "23.197.8.251"
}

input {
  file {
    path => "/var/log/dnsmasq.log"
    start_position => "beginning"
    type => "dnsmasq"
  }
}

# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140
# Mar  2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2

filter {
  if [type] == "dnsmasq" {
    grok {
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"]
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"]

      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"]
    }

    if [action] =~ "DHCPACK" {
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "No name" }
        }
      }
      aggregate {
        task_id => "%{clientip}"
        code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];"
        map_action => "create_or_update"
        # timeout = 0 sets the timeout to the default value 1800 seconds.
        timeout => 172800
      }
    } else if [action] == "query" {
      aggregate {
        task_id => "%{clientip}"
        code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']"
        map_action => "update"
      }
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "%{clientip}" }
        }
      }
      if ![clientmac] {
        mutate {
          add_field => { "clientmac" => "%{clientip}" }
        }
      }
    } else if [action] == "reply" {
        mutate {
          rename => { "clientip" => "serverip" }
        }
      geoip {
        source => "serverip"
      }
    } else
    {
      drop{}
    }
  }
}
output {
# elasticsearch { hosts => ["localhost:9200"] }
 stdout { codec => rubydebug }
}

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"}