Logstash 数小时后随机停止日志存储
运行logstash时，我收到一个随机错误： 16:30:26.240[[main]>worker0]错误logstash.pipeline-中出现异常 pipelineworker，管道已停止处理新事件，请 检查过滤器配置并重新启动Logstash。 {“异常”=>#， “backtrace”=>[“org/jruby/RubyString.java:3101:inLogstash 数小时后随机停止日志存储,logstash,Logstash,运行logstash时，我收到一个随机错误： 16:30:26.240[[main]>worker0]错误logstash.pipeline-中出现异常 pipelineworker，管道已停止处理新事件，请 检查过滤器配置并重新启动Logstash。 {“异常”=>#， “backtrace”=>[“org/jruby/RubyString.java:3101:ingsub'”， “org/jruby/RubyString.java:3069:ingsub'”， “/usr/share/log
gsub'”，
“org/jruby/RubyString.java:3069:ingsub\u动态\u字段“，
“/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:308:in
each'，
“/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:290:in
filter'，
“/usr/share/logstash/logstash core/lib/logstash/filters/base.rb:145:in
multi_filter'，“org/jruby/RubyArray.java:1613:inmulti_filter'，
“/usr/share/logstash/logstash core/lib/logstash/filter_delegator.rb:41:in
initialize'，
“org/jruby/RubyArray.java:1613:ininitialize'，“org/jruby/RubyProc.java:281:infilter_func'，
“/usr/share/logstash/logstash core/lib/logstash/pipeline.rb:295:in
call'，
“/usr/share/logstash/logstash core/lib/logstash/util/wrapped_synchronous_queue.rb:192:in
each'，
“/usr/share/logstash/logstash core/lib/logstash/util/wrapped_synchronous_queue.rb:191:in
filter_batch'，
“/usr/share/logstash/logstash core/lib/logstash/pipeline.rb:282:in
start_workers'”}16:30:26.542[LogStash:：Runner]致命
logstash.runner-发生意外错误！
{：错误=>#，
：backtrace=>[“org/jruby/RubyString.java:3101:ingsub'”，
“/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:317:in
gsub'，“org/jruby/RubyArray.java:1613:ingsub'，
“/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:207:in
do_filter'，
“/usr/share/logstash/logstash core/lib/logstash/filters/base.rb:164:in
each'，
“/usr/share/logstash/logstash core/lib/logstash/filters/base.rb:161:in
multi_filter'”（eval）：4135:ineach'”（eval）：4131:in
call'”（eval）：997:in
filter_batch'，“org/jruby/RubyProc.java:281:ineach'，“org/jruby/RubyHash.java:1342:ineach'，
“/usr/share/logstash/logstash core/lib/logstash/pipeline.rb:294:in
worker_loop'，
“/usr/share/logstash/logstash core/lib/logstash/pipeline.rb:258:in
input {
file {
type => "SystemError"
path => "/app/systemerr/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^\s"
what => "previous"
}
}
file {
type => "SystemOut"
path => "/app/systemout/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
file {
type => "Errorlog"
path => "/app/error/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^FATAL"
negate => true
what => "previous"
}
}
file {
type => "Messagelog"
path => "/app/message/**/*"
start_position => "beginning"
codec => multiline {
pattern => "^ERROR"
negate => true
what => "previous"
}
}
}
filter {
if [type] == "SystemError" {
grok {
match => { "message" => "\[%{DATA:timestamp}] %{BASE16NUM:threadID} (?<shortname>\b[A-Za-z0-9\$]{2,}\b)%{SPACE}%{WORD:loglevel}%{SPACE} %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
if ([message] =~ "^\tat") {
drop {}
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "SystemOut" {
grok {
match => { "message" => "\[%{DATA:timestamp}] %{BASE16NUM:threadID} (?<shortname>\b[A-Za-z0-9\$]{2,}\b)%{SPACE}%{WORD:loglevel}%{SPACE} %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "Errorlog" {
grok {
match => { "message" => "%{LOGLEVEL:loglevel} \| %{TIMESTAMP_ISO8601:timestamp} \| %{DATA:string} \: %{DATA:WebContainer} \| %{DATA:code} \| %{DATA:country} \| %{DATA:user} \| %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
date {
match => ["timestamp", "yyyy-M-dd HH:mm:ss,SSS"]
}
mutate { remove_field => [ "string" ] }
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
if [type] == "Messagelog" {
grok {
match => { "message" => "%{LOGLEVEL:loglevel} \| %{TIMESTAMP_ISO8601:timestamp} \| %{DATA:string} \: %{DATA:WebContainer} \| %{DATA:code} \| %{DATA:country} \| %{DATA:user} \| %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
date {
match => ["timestamp", "yyyy-M-dd HH:mm:ss,SSS"]
}
mutate {
remove_field => [ "string" ]
}
if ([path] =~ "113") {
mutate {
add_field => { "server" => "113" }
}
} else {
mutate {
add_field => { "server" => "117" }
}
}
}
}
输入{
文件{
类型=>“系统错误”
路径=>“/app/systemerr/***”
开始位置=>“开始”
编解码器=>多行{
模式=>“^\s”
什么=>“以前的”
}
}
文件{
类型=>“SystemOut”
路径=>“/app/systemout/***”
开始位置=>“开始”
编解码器=>多行{
模式=>“^\[”
否定=>true
什么=>“以前的”
}
}
文件{
类型=>“错误日志”
路径=>“/app/error/***”
开始位置=>“开始”
编解码器=>多行{
模式=>“^FATAL”
否定=>true
什么=>“以前的”
}
}
文件{
类型=>“消息日志”
路径=>“/app/message/***”
开始位置=>“开始”
编解码器=>多行{
模式=>“^ERROR”
否定=>true
什么=>“以前的”
}
}
}
滤器{
如果[type]=“SystemError”{
格罗克{
match=>{“message”=>“\[%{DATA:timestamp}]]{BASE16NUM:threadID}（？\b[A-Za-z0-9\$]{2，}\b）%{SPACE}%{WORD:loglevel}%{SPACE}%{greedyddata:message}}
覆盖=>[“消息”]
}
变异{
gsub=>[“时间戳”，“格林尼治标准时间\+05\：30”，“”]
}
日期{
匹配=>[“时间戳”，“年月日HH:mm:ss:SSS”]
if "_grokparsefailure" not in [tags] {
mutate {
gsub => ["timestamp", " GMT\+05\:30", ""]
}
date {
match => ["timestamp", "M/dd/yy HH:mm:ss:SSS"]
}
}