Synchronization 将两个elasticsearch群集与logstash同步
我正在尝试使用logstash同步两个elasticsearch集群 我的日志存储管道如下所示：Synchronization 将两个elasticsearch群集与logstash同步,synchronization,connection,logstash,elastic-stack,Synchronization,Connection,Logstash,Elastic Stack,我正在尝试使用logstash同步两个elasticsearch集群 我的日志存储管道如下所示： input { elasticsearch { hosts => ["host1:9200", "host2:9200", "host3:9200","host4:9200", "host4:9200" ] index => "*" siz
input {
elasticsearch {
hosts => ["host1:9200", "host2:9200", "host3:9200","host4:9200", "host4:9200" ]
index => "*"
size => 500
scroll => "5m"
docinfo => true
query => '{"query": {"range": {"update_date": {"gte": "now-5m"}}}, "_source" : {"exclude" : ["content"]} }'
schedule => "*/5 * * * *"
}
}
output {
elasticsearch {
hosts => ["cloud_es_host"]
user => "user"
password => "password"
index => "index_to_insert"
document_id => "%{[@metadata][_id]}"
doc_as_upsert => true
}
}
{ 2056 rufus-scheduler intercepted an error:
9/29/2020 11:45:10 AM 2056 job:
9/29/2020 11:45:10 AM 2056 Rufus::Scheduler::CronJob "*/5 * * * *" {}
9/29/2020 11:45:10 AM 2056 error:
9/29/2020 11:45:10 AM 2056 2056
9/29/2020 11:45:10 AM 2056 Manticore::ConnectTimeout
9/29/2020 11:45:10 AM 2056 connect timed out
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:37:in `block in initialize'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:79:in `call'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:274:in `call_once'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:158:in `code'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/http/manticore.rb:84:in `block in perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/base.rb:262:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/client.rb:131:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-api-5.0.5/lib/elasticsearch/api/actions/search.rb:183:in `search'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:340:in `search_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:268:in `do_run_slice'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:246:in `do_run'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:229:in `block in run'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:234:in `do_call'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:258:in `do_trigger'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:300:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:299:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 org/jruby/RubyKernel.java:1442:in `loop'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:289:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 tz:
9/29/2020 11:45:10 AM 2056 ENV['TZ']:
9/29/2020 11:45:10 AM 2056 Time.now: 2020-09-29 08:45:10 UTC
9/29/2020 11:45:10 AM 2056 scheduler:
9/29/2020 11:45:10 AM 2056 object_id: 2054
9/29/2020 11:45:10 AM 2056 opts:
9/29/2020 11:45:10 AM 2056 {:max_work_threads=>1}
9/29/2020 11:45:10 AM 2056 frequency: 0.3
9/29/2020 11:45:10 AM 2056 scheduler_lock: #<Rufus::Scheduler::NullLock:0x5491d760>
9/29/2020 11:45:10 AM 2056 trigger_lock: #<Rufus::Scheduler::NullLock:0x1ecdc57>
9/29/2020 11:45:10 AM 2056 uptime: 125.011625 (2m5s14)
9/29/2020 11:45:10 AM 2056 down?: false
9/29/2020 11:45:10 AM 2056 threads: 2
9/29/2020 11:45:10 AM 2056 thread: #<Thread:0x23035a1d>
9/29/2020 11:45:10 AM 2056 thread_key: rufus_scheduler_2054
9/29/2020 11:45:10 AM 2056 work_threads: 1
9/29/2020 11:45:10 AM 2056 active: 1
9/29/2020 11:45:10 AM 2056 vacant: 0
9/29/2020 11:45:10 AM 2056 max_work_threads: 1
9/29/2020 11:45:10 AM 2056 mutexes: {}
9/29/2020 11:45:10 AM 2056 jobs: 1
9/29/2020 11:45:10 AM 2056 at_jobs: 0
9/29/2020 11:45:10 AM 2056 in_jobs: 0
9/29/2020 11:45:11 AM 2056 every_jobs: 0
9/29/2020 11:45:11 AM 2056 interval_jobs: 0
9/29/2020 11:45:11 AM 2056 cron_jobs: 1
9/29/2020 11:45:11 AM 2056 running_jobs: 1
9/29/2020 11:45:11 AM 2056 work_queue: 0
9/29/2020 11:45:11 AM } 2056 .
input {
elasticsearch {
hosts => ["host1:9200", "host2:9200", "host3:9200","host4:9200", "host4:9200" ]
index => "*"
size => 500
scroll => "5m"
docinfo => true
query => '{"query": {"range": {"update_date": {"gte": "now-5m"}}}, "_source" : {"exclude" : ["content"]} }'
schedule => "*/5 * * * *"
}
}
output {
elasticsearch {
hosts => ["cloud_es_host"]
user => "user"
password => "password"
index => "index_to_insert"
document_id => "%{[@metadata][_id]}"
doc_as_upsert => true
}
}
{ 2056 rufus-scheduler intercepted an error:
9/29/2020 11:45:10 AM 2056 job:
9/29/2020 11:45:10 AM 2056 Rufus::Scheduler::CronJob "*/5 * * * *" {}
9/29/2020 11:45:10 AM 2056 error:
9/29/2020 11:45:10 AM 2056 2056
9/29/2020 11:45:10 AM 2056 Manticore::ConnectTimeout
9/29/2020 11:45:10 AM 2056 connect timed out
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:37:in `block in initialize'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:79:in `call'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:274:in `call_once'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/manticore-0.6.4-java/lib/manticore/response.rb:158:in `code'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/http/manticore.rb:84:in `block in perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/base.rb:262:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/transport/http/manticore.rb:67:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-transport-5.0.5/lib/elasticsearch/transport/client.rb:131:in `perform_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/elasticsearch-api-5.0.5/lib/elasticsearch/api/actions/search.rb:183:in `search'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:340:in `search_request'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:268:in `do_run_slice'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:246:in `do_run'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-elasticsearch-4.7.1/lib/logstash/inputs/elasticsearch.rb:229:in `block in run'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:234:in `do_call'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:258:in `do_trigger'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:300:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:299:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 org/jruby/RubyKernel.java:1442:in `loop'
9/29/2020 11:45:10 AM 2056 /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/rufus-scheduler-3.0.9/lib/rufus/scheduler/jobs.rb:289:in `block in start_work_thread'
9/29/2020 11:45:10 AM 2056 tz:
9/29/2020 11:45:10 AM 2056 ENV['TZ']:
9/29/2020 11:45:10 AM 2056 Time.now: 2020-09-29 08:45:10 UTC
9/29/2020 11:45:10 AM 2056 scheduler:
9/29/2020 11:45:10 AM 2056 object_id: 2054
9/29/2020 11:45:10 AM 2056 opts:
9/29/2020 11:45:10 AM 2056 {:max_work_threads=>1}
9/29/2020 11:45:10 AM 2056 frequency: 0.3
9/29/2020 11:45:10 AM 2056 scheduler_lock: #<Rufus::Scheduler::NullLock:0x5491d760>
9/29/2020 11:45:10 AM 2056 trigger_lock: #<Rufus::Scheduler::NullLock:0x1ecdc57>
9/29/2020 11:45:10 AM 2056 uptime: 125.011625 (2m5s14)
9/29/2020 11:45:10 AM 2056 down?: false
9/29/2020 11:45:10 AM 2056 threads: 2
9/29/2020 11:45:10 AM 2056 thread: #<Thread:0x23035a1d>
9/29/2020 11:45:10 AM 2056 thread_key: rufus_scheduler_2054
9/29/2020 11:45:10 AM 2056 work_threads: 1
9/29/2020 11:45:10 AM 2056 active: 1
9/29/2020 11:45:10 AM 2056 vacant: 0
9/29/2020 11:45:10 AM 2056 max_work_threads: 1
9/29/2020 11:45:10 AM 2056 mutexes: {}
9/29/2020 11:45:10 AM 2056 jobs: 1
9/29/2020 11:45:10 AM 2056 at_jobs: 0
9/29/2020 11:45:10 AM 2056 in_jobs: 0
9/29/2020 11:45:11 AM 2056 every_jobs: 0
9/29/2020 11:45:11 AM 2056 interval_jobs: 0
9/29/2020 11:45:11 AM 2056 cron_jobs: 1
9/29/2020 11:45:11 AM 2056 running_jobs: 1
9/29/2020 11:45:11 AM 2056 work_queue: 0
9/29/2020 11:45:11 AM } 2056 .