elasticsearch 脚本在upsert中失败，使用logstash输出elasticsearch插件

环境

DB:Sybase Logstash:2.2.0，带有JDBC插件、Elasticsearch输出插件

SQL查询：

select res.id as 'res.id', res.name as 'res.name', tag.name as 'tag.name'
from Res res, ResTags rt, Tags tag
where res.id *= rt.resrow and rt.tagid *= tag.id

res.id | res.name | tag.name
0 | result0 | null
0 | result0 | tagA
1 | result1 | tagA
1 | result1 | tagB
2 | result2 | tagA
2 | result2 | tagC

{ 
 "mappings": { 
  "res": { 
   "properties": { 
    "id": { "type": "long"},
    "name": { "type": "string" },
    "tags": {
      "type": "nested",
      "properties": { "tagname": { "type": "string" }}
    }
   }
 }
}

input {
 jdbc {
  jdbc_driver_library => "jtds-1.3.1.jar"
  jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
  jdbc_connection_string => "jdbc:jtds:sybase://hostname.com:1234/schema"
  jdbc_user => "george"
  jdbc_password => "monkey"
  jdbc_fetch_size => 100
  statement_filepath => "/home/george/sql"
 }
}
output {
 elasticsearch {
  action => "update"
  index => "myres"
  document_type => "res"
  document_id => "%{res.id}"
  script_lang => "groovy"
  hosts => [ "my.other.host.com:5921" ]
  upsert => ' {
    "id" : %{res.id},
    "name" : "%{res.name}",
    "tags" :[{ "tagname": "%{tag.name}" }]
  }'
  script => '
    if (ctx._source.res.tags.containsValue(null)) {
      // if null has been added replace it with actual value
      cts._source.res.tags = [{"tagname": "%{tag.name}" }];
    else {
      // if you find the tag, then do nothing
      if (ctx._source.res.tags.containsValue("%{tag.name}")) {}
      else {
        // if the value you try to add is not null
        if (%{tag.name} != null)
           // add it as a new object into the tag array
           ctx._source.res.tags += {"tagname": "%{tag.name}"};
        }
     }
   '
 }
}

    [400] {"error":"ActionRequestValidationException[Validation Failed: 
1: script or doc is missing;
2: script or doc is missing;
3: script or doc is missing;],"status":400]} {:class=> ... bla bla ...}

SQL结果：

select res.id as 'res.id', res.name as 'res.name', tag.name as 'tag.name'
from Res res, ResTags rt, Tags tag
where res.id *= rt.resrow and rt.tagid *= tag.id

res.id | res.name | tag.name
0 | result0 | null
0 | result0 | tagA
1 | result1 | tagA
1 | result1 | tagB
2 | result2 | tagA
2 | result2 | tagC

{ 
 "mappings": { 
  "res": { 
   "properties": { 
    "id": { "type": "long"},
    "name": { "type": "string" },
    "tags": {
      "type": "nested",
      "properties": { "tagname": { "type": "string" }}
    }
   }
 }
}

input {
 jdbc {
  jdbc_driver_library => "jtds-1.3.1.jar"
  jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
  jdbc_connection_string => "jdbc:jtds:sybase://hostname.com:1234/schema"
  jdbc_user => "george"
  jdbc_password => "monkey"
  jdbc_fetch_size => 100
  statement_filepath => "/home/george/sql"
 }
}
output {
 elasticsearch {
  action => "update"
  index => "myres"
  document_type => "res"
  document_id => "%{res.id}"
  script_lang => "groovy"
  hosts => [ "my.other.host.com:5921" ]
  upsert => ' {
    "id" : %{res.id},
    "name" : "%{res.name}",
    "tags" :[{ "tagname": "%{tag.name}" }]
  }'
  script => '
    if (ctx._source.res.tags.containsValue(null)) {
      // if null has been added replace it with actual value
      cts._source.res.tags = [{"tagname": "%{tag.name}" }];
    else {
      // if you find the tag, then do nothing
      if (ctx._source.res.tags.containsValue("%{tag.name}")) {}
      else {
        // if the value you try to add is not null
        if (%{tag.name} != null)
           // add it as a new object into the tag array
           ctx._source.res.tags += {"tagname": "%{tag.name}"};
        }
     }
   '
 }
}

    [400] {"error":"ActionRequestValidationException[Validation Failed: 
1: script or doc is missing;
2: script or doc is missing;
3: script or doc is missing;],"status":400]} {:class=> ... bla bla ...}

索引映射：

select res.id as 'res.id', res.name as 'res.name', tag.name as 'tag.name'
from Res res, ResTags rt, Tags tag
where res.id *= rt.resrow and rt.tagid *= tag.id

res.id | res.name | tag.name
0 | result0 | null
0 | result0 | tagA
1 | result1 | tagA
1 | result1 | tagB
2 | result2 | tagA
2 | result2 | tagC

{ 
 "mappings": { 
  "res": { 
   "properties": { 
    "id": { "type": "long"},
    "name": { "type": "string" },
    "tags": {
      "type": "nested",
      "properties": { "tagname": { "type": "string" }}
    }
   }
 }
}

input {
 jdbc {
  jdbc_driver_library => "jtds-1.3.1.jar"
  jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
  jdbc_connection_string => "jdbc:jtds:sybase://hostname.com:1234/schema"
  jdbc_user => "george"
  jdbc_password => "monkey"
  jdbc_fetch_size => 100
  statement_filepath => "/home/george/sql"
 }
}
output {
 elasticsearch {
  action => "update"
  index => "myres"
  document_type => "res"
  document_id => "%{res.id}"
  script_lang => "groovy"
  hosts => [ "my.other.host.com:5921" ]
  upsert => ' {
    "id" : %{res.id},
    "name" : "%{res.name}",
    "tags" :[{ "tagname": "%{tag.name}" }]
  }'
  script => '
    if (ctx._source.res.tags.containsValue(null)) {
      // if null has been added replace it with actual value
      cts._source.res.tags = [{"tagname": "%{tag.name}" }];
    else {
      // if you find the tag, then do nothing
      if (ctx._source.res.tags.containsValue("%{tag.name}")) {}
      else {
        // if the value you try to add is not null
        if (%{tag.name} != null)
           // add it as a new object into the tag array
           ctx._source.res.tags += {"tagname": "%{tag.name}"};
        }
     }
   '
 }
}

    [400] {"error":"ActionRequestValidationException[Validation Failed: 
1: script or doc is missing;
2: script or doc is missing;
3: script or doc is missing;],"status":400]} {:class=> ... bla bla ...}

Conf文件：

select res.id as 'res.id', res.name as 'res.name', tag.name as 'tag.name'
from Res res, ResTags rt, Tags tag
where res.id *= rt.resrow and rt.tagid *= tag.id

res.id | res.name | tag.name
0 | result0 | null
0 | result0 | tagA
1 | result1 | tagA
1 | result1 | tagB
2 | result2 | tagA
2 | result2 | tagC

{ 
 "mappings": { 
  "res": { 
   "properties": { 
    "id": { "type": "long"},
    "name": { "type": "string" },
    "tags": {
      "type": "nested",
      "properties": { "tagname": { "type": "string" }}
    }
   }
 }
}

input {
 jdbc {
  jdbc_driver_library => "jtds-1.3.1.jar"
  jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
  jdbc_connection_string => "jdbc:jtds:sybase://hostname.com:1234/schema"
  jdbc_user => "george"
  jdbc_password => "monkey"
  jdbc_fetch_size => 100
  statement_filepath => "/home/george/sql"
 }
}
output {
 elasticsearch {
  action => "update"
  index => "myres"
  document_type => "res"
  document_id => "%{res.id}"
  script_lang => "groovy"
  hosts => [ "my.other.host.com:5921" ]
  upsert => ' {
    "id" : %{res.id},
    "name" : "%{res.name}",
    "tags" :[{ "tagname": "%{tag.name}" }]
  }'
  script => '
    if (ctx._source.res.tags.containsValue(null)) {
      // if null has been added replace it with actual value
      cts._source.res.tags = [{"tagname": "%{tag.name}" }];
    else {
      // if you find the tag, then do nothing
      if (ctx._source.res.tags.containsValue("%{tag.name}")) {}
      else {
        // if the value you try to add is not null
        if (%{tag.name} != null)
           // add it as a new object into the tag array
           ctx._source.res.tags += {"tagname": "%{tag.name}"};
        }
     }
   '
 }
}

    [400] {"error":"ActionRequestValidationException[Validation Failed: 
1: script or doc is missing;
2: script or doc is missing;
3: script or doc is missing;],"status":400]} {:class=> ... bla bla ...}

目标是将从数据库返回的多行添加到ES中，将标记连接为新对象（这是一个简化的示例，因此add_标记和过滤器不起作用，因为我的json结构深度超过2层（嵌套或嵌套等））

批量上传到ES后的预期结果是：

{
 "hits": {
   "total": 3,
   "max_score": 1,
   "hits": [ {
     "_index": "myres",
     "_type": "res",
     "_id": 0,
     "_score": 1,
     "_source": {
       "res": {
         "id":0,
         "name": "result0",
         "tags": [{"tagname": "tagA"}],
         "@version": "2",
         "@timestamp": "2016-xx-yy..."
       }
      },{
     "_index": "myres",
     "_type": "res",
     "_id": 1,
     "_score": 1,
     "_source": {
       "res": {
         "id":1,
         "name": "result1",
         "tags": [{"tagname": "tagA"},{"tagname": "tagB"}],
         "@version": "2",
         "@timestamp": "2016-xx-yy..."
       }
      }{
     "_index": "myres",
     "_type": "res",
     "_id": 2,
     "_score": 1,
     "_source": {
       "res": {
         "id":2,
         "name": "result2",
         "tags": [{"tagname": "tagA"},{"tagname": "tagC"],
         "@version": "2",
         "@timestamp": "2016-xx-yy..."
       }
      }
     }
...

问题：如果在conf，output部分脚本没有注释掉，则会弹出以下错误。如果不包括脚本，则只导入初始标记（如预期的那样），而不导入第二个标记

看起来脚本在elasticsearch输出中不起作用

错误消息：

select res.id as 'res.id', res.name as 'res.name', tag.name as 'tag.name'
from Res res, ResTags rt, Tags tag
where res.id *= rt.resrow and rt.tagid *= tag.id

res.id | res.name | tag.name
0 | result0 | null
0 | result0 | tagA
1 | result1 | tagA
1 | result1 | tagB
2 | result2 | tagA
2 | result2 | tagC

{ 
 "mappings": { 
  "res": { 
   "properties": { 
    "id": { "type": "long"},
    "name": { "type": "string" },
    "tags": {
      "type": "nested",
      "properties": { "tagname": { "type": "string" }}
    }
   }
 }
}

input {
 jdbc {
  jdbc_driver_library => "jtds-1.3.1.jar"
  jdbc_driver_class => "Java::net.sourceforge.jtds.jdbc.Driver"
  jdbc_connection_string => "jdbc:jtds:sybase://hostname.com:1234/schema"
  jdbc_user => "george"
  jdbc_password => "monkey"
  jdbc_fetch_size => 100
  statement_filepath => "/home/george/sql"
 }
}
output {
 elasticsearch {
  action => "update"
  index => "myres"
  document_type => "res"
  document_id => "%{res.id}"
  script_lang => "groovy"
  hosts => [ "my.other.host.com:5921" ]
  upsert => ' {
    "id" : %{res.id},
    "name" : "%{res.name}",
    "tags" :[{ "tagname": "%{tag.name}" }]
  }'
  script => '
    if (ctx._source.res.tags.containsValue(null)) {
      // if null has been added replace it with actual value
      cts._source.res.tags = [{"tagname": "%{tag.name}" }];
    else {
      // if you find the tag, then do nothing
      if (ctx._source.res.tags.containsValue("%{tag.name}")) {}
      else {
        // if the value you try to add is not null
        if (%{tag.name} != null)
           // add it as a new object into the tag array
           ctx._source.res.tags += {"tagname": "%{tag.name}"};
        }
     }
   '
 }
}

    [400] {"error":"ActionRequestValidationException[Validation Failed: 
1: script or doc is missing;
2: script or doc is missing;
3: script or doc is missing;],"status":400]} {:class=> ... bla bla ...}

注释

• 为了避免浪费人们的时间，doc\u as\u upsert=>true也不能按预期工作。它只会不断更新/覆盖数据库，只会保留数据库的最新一行
• 另外，jdbc到ES的river插件不支持嵌套结构的嵌套，因此无法正常工作

---

您是否尝试过启用

脚本化插入

标志？对它仍然失败。看起来upsert和script是互斥的。另外，当您编写了upsert脚本时，您需要保留upsert:{}，即使它是空的（至少这是我在文档0中的印象实际上，

upsert

可以根据

script

使用和不使用

script

工作。在

script

的情况下，

upsert

仅在

scripted\u upsert

为false时使用。你应该尝试将脚本存储在一个文件中，并改用

script\u type:file

。我反对。）格力。即使脚本是空的，它也不适用于脚本。如果您相信，请提供上述场景的一个可靠示例作为答案。