使用java将数据从Dynamo DB推送到elasticsearch

嗨，我已经用java创建了一个处理程序，用于从dynamo DB获取事件

这是我的代码

package com.Lambda.dynamodb;

import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;

public class DDBEventProcessor implements
        RequestHandler<DynamodbEvent, String> {

    public String handleRequest(DynamodbEvent ddbEvent, Context context) {
        for (DynamodbStreamRecord record : ddbEvent.getRecords()){
            System.out.println(record.getEventID());
            System.out.println(record.getEventName());
            System.out.println(record.getDynamodb().toString());
        }
        return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
    }
}

包com.Lambda.dynamodb；
导入com.amazonaws.services.lambda.runtime.Context；
导入com.amazonaws.services.lambda.runtime.LambdaLogger；
导入com.amazonaws.services.lambda.runtime.RequestHandler；
导入com.amazonaws.services.lambda.runtime.events.DynamodbEvent；
导入com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord；
公共类DDP处理器实现
请求处理程序{
公共字符串handleRequest（DynamodbEvent ddbEvent，上下文）{
对于（DynamodbStreamRecord记录：ddbEvent.getRecords（））{
System.out.println（record.getEventID（））；
System.out.println（record.getEventName（））；
System.out.println（record.getDynamodb（）.toString（））；
}
返回“已成功处理”+ddbEvent.getRecords（）.size（）+“记录”；
}
}

Lambda函数能够在cloudwatch中写入事件，但挑战是我必须将所有流式记录索引到AWS elasticsearch服务端点并对其进行索引。 在搜索博客时，我得到了一些python和node.js中的代码示例，但我的要求是我必须用java构建这个lambda函数

---

有谁能建议如何在java lambda函数中实现这一点吗？

您好，我已经包含了下面的代码，可能对某些人有所帮助。Dynamo DB streams在AWS内部和外部的elasticsearch中索引文档

package com.Firstlambda;

import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;

import com.amazonaws.auth.AWS4Signer;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemUtils;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent;
import com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.json.JSONObject;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;


public class HelloWorld implements RequestHandler<DynamodbEvent, String> {
    private static String serviceName = "es";
    private static String region = "us-east-1";
    private static String aesEndpoint = ""
    private static String index = "";
    private static String type = "_doc";

    static final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();

    public String handleRequest(DynamodbEvent ddbEvent, Context context) {
        for (DynamodbStreamRecord record : ddbEvent.getRecords()) {
            System.out.println("EventName : " + record.getEventName());
            System.out.println("EventName : " + record.getDynamodb());

            //AWS outside
            RestHighLevelClient esClient = esClient();
            //AWS outside

            //AWS Inside
            //RestHighLevelClient esClient = esClient(serviceName, region);
            //AWS Inside
            if (record.getEventName().toLowerCase().equals("insert")) {
                String JsonString = getJsonstring(record.getDynamodb().getNewImage());
                String JsonUniqueId = GetIdfromJsonString(JsonString);
                IndexRequest indexRequest = new IndexRequest(index, type, JsonUniqueId);
                indexRequest.source(JsonString, XContentType.JSON);
                try {
                    IndexResponse indexResponse = esClient.index(indexRequest, RequestOptions.DEFAULT);
                    System.out.println(indexResponse.toString());
                    return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
                } catch (IOException e) {
                    System.out.println(e.getMessage());
                }
            } else if (record.getEventName().toLowerCase().equals("modify")) {
                String JsonString = getJsonstring(record.getDynamodb().getNewImage());
                String JsonUniqueId = GetIdfromJsonString(JsonString);
                UpdateRequest request = new UpdateRequest(index, type, JsonUniqueId);
                String jsonString = JsonString;
                request.doc(jsonString, XContentType.JSON);
                try {
                    UpdateResponse updateResponse = esClient.update(
                            request, RequestOptions.DEFAULT);
                    System.out.println(updateResponse.toString());
                    return "Successfully processed " + ddbEvent.getRecords().size() + " records.";
                } catch (IOException e) {
                    System.out.println(e.getMessage());
                }
            } else {
                System.out.println("remove");
                System.out.println("KEYID : " + record.getDynamodb().getKeys().get("ID").getN());
                String deletedId = record.getDynamodb().getKeys().get("ID").getN();
                DeleteRequest request = new DeleteRequest(index, type, deletedId);
                try {
                    DeleteResponse deleteResponse = esClient.delete(
                            request, RequestOptions.DEFAULT);
                } catch (IOException e) {
                    System.out.println(e.getMessage());
                }
            }
        }
        return "Successfullyprocessed";
    }

    public String getJsonstring(Map<String, AttributeValue> newIma) {
        String json = null;
        Map<String, AttributeValue> newImage = newIma;
        List<Map<String, AttributeValue>> listOfMaps = new ArrayList<Map<String, AttributeValue>>();
        listOfMaps.add(newImage);
        List<Item> itemList = ItemUtils.toItemList(listOfMaps);
        for (Item item : itemList) {
            json = item.toJSON();
        }
        return json;
    }

    public String GetIdfromJsonString(String Json) {
        JSONObject jsonObj = new JSONObject(Json);
        return String.valueOf(jsonObj.getInt("ID"));
    }

    // Adds the interceptor to the ES REST client
//    public static RestHighLevelClient esClient(String serviceName, String region) {
//        AWS4Signer signer = new AWS4Signer();
//        signer.setServiceName(serviceName);
//        signer.setRegionName(region);
//        HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor(serviceName, signer, credentialsProvider);
//        return new RestHighLevelClient(RestClient.builder(HttpHost.create(aesEndpoint)).setHttpClientConfigCallback(hacb -> hacb.addInterceptorLast(interceptor)));
//    }

    public static RestHighLevelClient esClient() {
        String host = "d9bc7cbca5ec49ea96a6ea683f70caca.eastus2.azure.elastic-cloud.com";

        int port = 9200;

        String userName = "elastic";

        String password = "L4Nfnle3wxLmV95lffwsf$Ub46hp";

        String protocol = "https";

        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(userName, password));

        RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, protocol))
                .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));

        RestHighLevelClient client = new RestHighLevelClient(builder);
        return client;
    }

}

package com.Firstlambda；
导入com.amazonaws.services.lambda.runtime.Context；
导入com.amazonaws.services.lambda.runtime.RequestHandler；
导入com.amazonaws.auth.AWS4Signer；
导入com.amazonaws.auth.AWSCredentialsProvider；
导入com.amazonaws.auth.DefaultAWSCredentialsProviderChain；
导入com.amazonaws.services.dynamodbv2.document.Item；
导入com.amazonaws.services.dynamodbv2.document.ItemUtils；
导入com.amazonaws.services.dynamodbv2.model.AttributeValue；
导入com.amazonaws.services.lambda.runtime.events.DynamodbEvent；
导入com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord；
导入org.apache.http.HttpHost；
导入org.apache.http.HttpRequestInterceptor；
导入org.apache.http.auth.AuthScope；
导入org.apache.http.auth.UsernamePasswordCredentials；
导入org.apache.http.client.CredentialsProvider；
导入org.apache.http.impl.client.BasicCredentialsProvider；
导入org.elasticsearch.action.delete.DeleteRequest；
导入org.elasticsearch.action.delete.DeleteResponse；
导入org.elasticsearch.action.index.IndexRequest；
导入org.elasticsearch.action.index.IndexResponse；
导入org.elasticsearch.action.update.UpdateRequest；
导入org.elasticsearch.action.update.UpdateResponse；
导入org.elasticsearch.client.RequestOptions；
导入org.elasticsearch.client.RestClient；
导入org.elasticsearch.client.RestClientBuilder；
导入org.elasticsearch.client.RestHighLevelClient；
导入org.elasticsearch.common.xcontent.XContentType；
导入org.json.JSONObject；
导入java.io.IOException；
导入java.util.ArrayList；
导入java.util.List；
导入java.util.Map；
公共类HelloWorld实现RequestHandler{
私有静态字符串serviceName=“es”；
私有静态字符串区域=“us-east-1”；
私有静态字符串aesEndpoint=“”
私有静态字符串索引=”；
私有静态字符串类型=“\u doc”；
静态最终AWSCredentialsProvider credentialsProvider=新的默认AWSCredentialsProviderChain（）；
公共字符串handleRequest（DynamodbEvent ddbEvent，上下文）{
对于（DynamodbStreamRecord记录：ddbEvent.getRecords（））{
System.out.println（“EventName:+record.getEventName（））；
System.out.println（“EventName:+record.getDynamodb（））；
//室外自动气象站
RestHighLevelClient esClient=esClient（）；
//室外自动气象站
//AWS内部
//RestHighLevelClient esClient=esClient（服务名称，区域）；
//AWS内部
if（record.getEventName（）.toLowerCase（）.equals（“插入”））{
字符串JsonString=getJsonstring（record.getDynamodb（）.getNewImage（））；
字符串JsonUniqueId=GetIdfromJsonString（JsonString）；
IndexRequest IndexRequest=新的IndexRequest（索引、类型、JsonUniqueId）；
source（JsonString，XContentType.JSON）；
试一试{
IndexResponse IndexResponse=esClient.index（indexRequest，RequestOptions.DEFAULT）；
System.out.println（indexResponse.toString（））；
返回“已成功处理”+ddbEvent.getRecords（）.size（）+“记录”；
}捕获（IOE异常）{
System.out.println（e.getMessage（））；
}
}else if（record.getEventName（）.toLowerCase（）.equals（“modify”））{
字符串JsonString=getJsonstring（record.getDynamodb（）.getNewImage（））；
字符串JsonUniqueId=GetIdfromJsonString（JsonString）；
UpdateRequest=新的UpdateRequest（索引、类型、JsonUniqueId）；
字符串jsonString=jsonString；
doc（jsonString，XContentType.JSON）；
试一试{
UpdateResponse UpdateResponse=esClient.update(
request，RequestOptions.DEFAULT）；
System.out.println（updateResponse.toString（））；
返回“已成功处理”+ddbEvent.getRecords（）.size（）+“记录”；
}捕获（IOE异常）{
System.out.println（e.getMessage（））；
}
}否则{
系统输出打印项次（“删除”）；
System.out.println（“KEYID:+record.getDynamodb（）.getKeys（）.get（“ID”）.getN（））；
字符串deletedId=record.getDynamodb（）.getKeys（）.get（“ID”）。