Java 将Elasticsearch JSON响应转换为Elasticsearch BulkResponse
由于Elasticsearch正在从其API中删除_type,因此我需要在不同版本的ES上保持产品的兼容性。Java 将Elasticsearch JSON响应转换为Elasticsearch BulkResponse,java,elasticsearch,Java,elasticsearch,由于Elasticsearch正在从其API中删除_type,因此我需要在不同版本的ES上保持产品的兼容性。 为了解决2.x、5.x和6.x与不支持_类型的更高版本之间的冲突,我只需统一api,但对发送到ES的请求编写不同的实现。对于支持_类型的版本,我使用ES低rest客户端向ES发送原始dsl,而对于不支持的版本,我调用高rest客户端。不幸的是,当我实现bulk()时,我根本不知道应该如何处理BulkResponse。 对于2.x和5.x,我的方法返回是一个JSONObject,实现如下:
为了解决2.x、5.x和6.x与不支持_类型的更高版本之间的冲突,我只需统一api,但对发送到ES的请求编写不同的实现。对于支持_类型的版本,我使用ES低rest客户端向ES发送原始dsl,而对于不支持的版本,我调用高rest客户端。不幸的是,当我实现bulk()时,我根本不知道应该如何处理BulkResponse。
对于2.x和5.x,我的方法返回是一个JSONObject,实现如下:
/**
*@param bulkRequest要发送的请求
*@param type目标类型,对于es 2.x和5.x,此字段由用户设置
*@param refresh强制es刷新并创建新的lucene段
*@param选项
*@param头文件
*@返回
*@抛出异常
*/
@凌驾
公共BulkResponse批量(BulkRequest BulkRequest、字符串类型、布尔刷新、RequestOptions选项、标头…标头)引发IOException{
字符串端点=connection.getHttpSchema()+“:/”+connection.getConnectedHosts()
+“:“+connection.getConnectedPort()+”/_bulk”;
HttpEntity实体=新的NStringEntity(wrapBulkRequests(bulkRequest,type),ContentType.APPLICATION_JSON);
请求=新请求(POST、端点);
请求。设置选项(选项);
请求。设置实体(实体);
响应=this.connection.getLowLevelClient().performRequest(请求);
HttpEntity HttpEntity=response.getEntity();
JSONObject res=JSON.parseObject(EntityUtils.toString(httpEntity,“utf-8”);
debug(“批量响应为{}”,res);
返回新的响应(res);
}
私有字符串wrapBulkRequests(BulkRequest,字符串类型){
StringBuilder bulkRequestBody=新的StringBuilder();
for(DocWriteRequest请求:bulkRequest.requests()){
String requestStr=request.toString();
字符串targetIndex=request.index();
字符串_id=request.id();
字符串opType=request.opType().getLowercase();
if(索引类型等于(操作类型)){
如果(_id==null | |“”.equals(_id)){
bulkRequestBody.append(String.format(“{\”index\”:{\“\u index\”:\%s\”,\“\u type\”:\%s\”}}\n”,targetIndex,type));
}否则{
bulkRequestBody.append(String.format(“{\”index\”:{\“\u index\”:\%s\”,\“\u type\”:\%s\”,\“\u id\”:\%s\”}\n”,targetIndex,type,\u id));
}
append(matchSourceFromIndexRequest(requestStr,“source\\[(\\{.+\})\\]”)+“\n”);
}else if(创建_OP_TYPE.equals(opType)){
如果(_id==null | |“”.equals(_id)){
bulkRequestBody.append(String.format(“{\”create\“:{\”\u index\“:\%s\”,\“\u type\”:\%s\“}}\n”,targetIndex,type));
}否则{
bulkRequestBody.append(String.format(“{\”create\“:{\”\u index\“:\%s\”,\“\u type\”:\%s\”,\“\u id\”:\%s\”}\n”,targetIndex,type,\u id));
}
append(matchSourceFromIndexRequest(requestStr,“source\\[(\\{.+\})\\]”)+“\n”);
}else if(UPDATE_OP_TYPE.equals(opType)){
int retryOnConflict=((UpdateRequest)请求).retryOnConflict();
如果(retryOnConflict==0){
bulkRequestBody.append(String.format(“{\”update\“:{\”\u index\“:\%s\”,\“\u type\”:\%s\”,\“\u id\”:\%s\”}\n”,targetIndex,type,\u id));
}否则{
bulkRequestBody.append(String.format(“{”update\“:{”u index\“:\%s\”,\“\u type\”:\%s\”,\“\u id\”:\%s\”,\“在\u冲突中重试”:%d}\n”,targetIndex,type,\u id,retryOnConflict));
}
布尔值isUpsert=((UpdateRequest)请求).docAsUpsert();
append(matchSourceFromUpdateRequest(requestStr,isUpsert));
}else if(DELETE_OP_TYPE.equals(opType)){//DELETE ops
bulkRequestBody.append(String.format(“{\”delete\”:{\“\u index\”:\%s\”,\“\u type\”:\%s\”,\“\u id\”:\%s\”}\n”,targetIndex,type,\u id));
}否则{
错误(“错误的批量请求op_类型{}”,opType);
}
}
debug(“批量请求体为{}”,bulkRequestBody.toString());
返回bulkRequestBody.toString();
}
私有字符串匹配SourceFromUpdateRequest(字符串请求str,布尔值isUpsert){
如果(isupert){
返回“{\“doc\”:“+matchSourceFromIndexRequest(requestStr,“source\\[(\\{.+\})\\]\\\\\\\\\\\\\\\\\\\]”+”,““doc\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\;
}否则{
返回“{\“doc\”:“+matchSourceFromIndexRequest(requestStr,“source\\[(\\{.+\})\\]\\\\\\\\\\\\]”+“}\n”;
}
}
私有字符串匹配SourceFromIndexRequest(字符串请求str,字符串正则表达式){
Pattern p=Pattern.compile(regex);
匹配器m=p.Matcher(requestStr);
if(m.find()){
debug(“索引批量请求源为{}”,m.group(1));
返回m组(1);
}
错误(“无法从索引批量请求中匹配源,requestStr为{}”,requestStr);
返回“”;
}
为了与6.x和7.x兼容,我需要返回BulkResponse
。对于像GetResponse
,UpdateResponse
,IndexResponse
这样的情况,我可以简单地编写我们自己的包装类来保存基于当前ES版本的JSONObject或真实ES响应,因为上面常用的api返回类型只是主要类型。但是对于BulkResponse,它返回许多嵌套类,如B
/**
* @param bulkRequest requests to be sent
* @param type target type, for es 2.x and 5.x, this field is set by user
* @param refresh force es to refresh and create new lucene segment or not
* @param options
* @param headers
* @return
* @throws IOException
*/
@Override
public BulkResponse bulk(BulkRequest bulkRequest, String type, boolean refresh, RequestOptions options, Header... headers) throws IOException {
String endpoint = connection.getHttpSchema() + "://" + connection.getConnectedHosts()
+ ":" + connection.getConnectedPort() + "/_bulk";
HttpEntity entity = new NStringEntity(wrapBulkRequests(bulkRequest, type), ContentType.APPLICATION_JSON);
Request request = new Request(POST, endpoint);
request.setOptions(options);
request.setEntity(entity);
Response response = this.connection.getLowLevelClient().performRequest(request);
HttpEntity httpEntity = response.getEntity();
JSONObject res = JSON.parseObject(EntityUtils.toString(httpEntity, "utf-8"));
LOG.debug("bulk response is {}", res);
return new BulkResponse(res);
}
private String wrapBulkRequests(BulkRequest bulkRequest, String type) {
StringBuilder bulkRequestBody = new StringBuilder();
for (DocWriteRequest<?> request : bulkRequest.requests()) {
String requestStr = request.toString();
String targetIndex = request.index();
String _id = request.id();
String opType = request.opType().getLowercase();
if (INDEX_OP_TYPE.equals(opType)) {
if (_id == null || "".equals(_id)) {
bulkRequestBody.append(String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", targetIndex, type));
} else {
bulkRequestBody.append(String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\" } }\n", targetIndex, type, _id));
}
bulkRequestBody.append(matchSourceFromIndexRequest(requestStr, "source\\[(\\{.+\\})\\]") + "\n");
} else if (CREATE_OP_TYPE.equals(opType)) {
if (_id == null || "".equals(_id)) {
bulkRequestBody.append(String.format("{ \"create\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", targetIndex, type));
} else {
bulkRequestBody.append(String.format("{ \"create\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\" } }\n", targetIndex, type, _id));
}
bulkRequestBody.append(matchSourceFromIndexRequest(requestStr, "source\\[(\\{.+\\})\\]") + "\n");
} else if (UPDATE_OP_TYPE.equals(opType)) {
int retryOnConflict = ((UpdateRequest) request).retryOnConflict();
if (retryOnConflict == 0) {
bulkRequestBody.append(String.format("{ \"update\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\" } }\n", targetIndex, type, _id));
} else {
bulkRequestBody.append(String.format("{ \"update\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\", \"retry_on_conflict\" : %d} }\n", targetIndex, type, _id, retryOnConflict));
}
boolean isUpsert = ((UpdateRequest) request).docAsUpsert();
bulkRequestBody.append(matchSourceFromUpdateRequest(requestStr, isUpsert));
} else if (DELETE_OP_TYPE.equals(opType)) { // delete ops
bulkRequestBody.append(String.format("{ \"delete\" : { \"_index\" : \"%s\", \"_type\" : \"%s\", \"_id\" : \"%s\" } }\n", targetIndex, type, _id));
} else {
LOG.error("bad bulk request op_type {}", opType);
}
}
LOG.debug("bulk request body is {}", bulkRequestBody.toString());
return bulkRequestBody.toString();
}
private String matchSourceFromUpdateRequest(String requestStr, boolean isUpsert) {
if (isUpsert) {
return "{ \"doc\" : " + matchSourceFromIndexRequest(requestStr, "source\\[(\\{.+\\})\\]\\}\\]") + ", \"doc_as_upsert\" : true }\n";
} else {
return "{ \"doc\" : " + matchSourceFromIndexRequest(requestStr, "source\\[(\\{.+\\})\\]\\}\\]") + " }\n";
}
}
private String matchSourceFromIndexRequest(String requestStr, String regex) {
Pattern p = Pattern.compile(regex);
Matcher m = p.matcher(requestStr);
if (m.find()) {
LOG.debug("index bulk request source is {}", m.group(1));
return m.group(1);
}
LOG.error("cannot match source from index bulk request, requestStr is {}", requestStr);
return "";
}