Spring integration 调用高并发性http调用时丢失消息
嗨，我有如下的流定义。我从s3中提取文件，逐行分割，调用http客户端并放入命名通道。我的传输是rabbit，预取是10，http的并发性是100，在3个容器和1个管理员上运行Spring integration 调用高并发性http调用时丢失消息,spring-integration,spring-xd,Spring Integration,Spring Xd,嗨，我有如下的流定义。我从s3中提取文件，逐行分割，调用http客户端并放入命名通道。我的传输是rabbit，预取是10，http的并发性是100，在3个容器和1个管理员上运行 stream aws-s3|custom processor| custom-http-client --url1=https://test1.com --url2=https://test1.com --filterAttribute=messageAttribute --httpMethod=POST --nonRe
stream aws-s3|custom processor| custom-http-client --url1=https://test1.com --url2=https://test1.com --filterAttribute=messageAttribute --httpMethod=POST --nonRetryErrorCodes=400,401,404,500 --charset=UTF-8 --replyTimeout=30000 --mapHeaders=Api-Key,Content-Type --requestTimeOut=30000 |processor> queue:testQueue
<beans:beans xmlns="http://www.springframework.org/schema/integration"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:beans="http://www.springframework.org/schema/beans"
xmlns:int-http="http://www.springframework.org/schema/integration/http"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration
http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/http
http://www.springframework.org/schema/integration/http/spring-integration-http.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-4.0.xsd">
<!-- <context:property-placeholder location="${xd.module.config.location}\processor\${xd.module.name}\batch-http.properties"
ignore-resource-not-found="true" local-override="true"/> -->
<context:property-placeholder />
<!-- logger changes start -->
<channel-interceptor pattern="*" order="3">
<beans:bean class="org.springframework.integration.channel.interceptor.WireTap">
<beans:constructor-arg ref="loggingChannel" />
</beans:bean>
</channel-interceptor>
<logging-channel-adapter id="loggingChannel" log-full-message="true" level="ERROR"/>
<!-- logger changes end -->
<header-filter input-channel="input"
output-channel="inputX" header-names="x-death"/>
<service-activator input-channel="inputX" ref="gw" />
<gateway id="gw" default-request-channel="toHttp" default-reply-timeout="0" error-channel="errors" />
<beans:bean id="inputfields" class="test.HTTPInputProperties">
<beans:property name="nonRetryErrorCodes" value="${nonRetryErrorCodes}"/>
</beans:bean>
<beans:bean id="responseInterceptor" class="test.ResponseInterceptor">
<beans:property name="inputProperties" ref="inputfields" />
</beans:bean>
<chain input-channel="errors" output-channel="output">
<!-- examine payload.cause (http status code etc) and decide whether
to throw an exception or return the status code for sending to output -->
<header-filter header-names="replyChannel, errorChannel" />
<transformer ref="responseInterceptor" />
</chain>
<int-http:outbound-gateway id='batch-http' header-mapper="headerMapper"
request-channel='toHttp'
rest-template="batchRestTemplate"
url-expression="payload.contains('${filterAttribute}') ? '${url1}' : '${url2}'" http-method="${httpMethod}"
expected-response-type='java.lang.String' charset='${charset}'
reply-timeout='${replyTimeout}' reply-channel='output'>
</int-http:outbound-gateway>
<beans:bean id="batchHTTPConverter" class="org.springframework.http.converter.StringHttpMessageConverter" >
<beans:constructor-arg index="0" value="${charset}"/>
<beans:property name="supportedMediaTypes" value = "application/json;UTF-8" />
</beans:bean>
<beans:bean id="batchRestTemplate" class="testBatchRestTemplate" >
<beans:constructor-arg name="requestTimeOut" value="${requestTimeOut}"/>
<beans:constructor-arg name="maxConnectionPerRoute" value="${maxConnectionPerRoute}"/>
<beans:constructor-arg name="totalMaxConnections" ref="${totalMaxConnections}"/>
</beans:bean>
<beans:bean id="headerMapper" class="org.springframework.integration.http.support.DefaultHttpHeaderMapper"
factory-method="outboundMapper">
<beans:property name="outboundHeaderNames" value="${mapHeaders}"/>
<beans:property name="userDefinedHeaderPrefix" value=""/>
</beans:bean>
<channel id="output" />
<channel id="input" />
<channel id="inputX" />
<channel id="toHttp" />
</beans:beans>
public class BatchRestTemplate extends RestTemplate{
private static final Logger LOGGER = LoggerFactory
.getLogger(BatchRestTemplate.class);
private static Integer requestTimeOut;
private static Integer totalMaxConnections;
private static Integer maxConnectionPerRoute;
public BatchRestTemplate(Integer requestTimeOut,Integer totalMaxConnections,Integer maxConnectionPerRoute) throws NoSuchAlgorithmException {
super(createBatchHttpRequestFactory());
List<HttpMessageConverter<?>> messageConverters= new ArrayList<HttpMessageConverter<?>>();
messageConverters.addAll(getMessageConverters());
messageConverters.add(0,new StringHttpMessageConverter(Charset.forName("UTF-8")));
setMessageConverters(messageConverters);
}
private static ClientHttpRequestFactory createBatchHttpRequestFactory() throws NoSuchAlgorithmException {
CloseableHttpClient httpClient;
HttpComponentsClientHttpRequestFactory httpRequestFactory;
SSLConnectionSocketFactory socketFactory;
socketFactory = new SSLConnectionSocketFactory(
SSLContext.getDefault(),
new String[] {"TLSv1"},
null,
SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", socketFactory)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setMaxTotal(250);
cm.setDefaultMaxPerRoute(100);
cm.closeExpiredConnections();
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(30000)
.setConnectionRequestTimeout(30000).setSocketTimeout(30000).build();
httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).setConnectionManager(cm).build();
httpRequestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
return httpRequestFactory;
}
}
公共类BatchRestTemplate扩展了RestTemplate{
专用静态最终记录器记录器=记录器工厂
.getLogger（BatchRestTemplate.class）；
私有静态整数请求超时；
私有静态整数totalMaxConnections；
私有静态整数maxConnectionPerRoute；
公共BatchRestTemplate（Integer requestTimeOut、Integer totalMaxConnections、Integer maxConnectionPerRoute）抛出NoSuchAlgorithmException{
super（createBatchHttpRequestFactory（））；
列表>（）；
addAll（getMessageConverters（））；
添加（0，新的StringHttpMessageConverter（Charset.forName（“UTF-8”）））；
设置消息转换器（messageConverters）；
}
私有静态ClientHttpRequestFactory createBatchHttpRequestFactory（）抛出NoSuchAlgorithmException{
可关闭的httpClient httpClient；
Http组件客户端Http请求工厂Http请求工厂；
SSLConnectionSocketFactory socketFactory；
socketFactory=新SSLConnectionSocketFactory(
SSLContext.getDefault（），
新字符串[]{“TLSv1”}，
无效的
SSLConnectionSocketFactory。允许\u所有\u主机名\u验证程序）；
注册表socketFactoryRegistry=RegistryBuilder.create（）
.register（“http”，PlainConnectionSocketFactory.getSocketFactory（））
.注册（“https”，socketFactory）
.build（）；
PoolightPClientConnectionManager cm=新的PoolightPClientConnectionManager（socketFactoryRegistry）；
cm.setMaxTotal（250）；
cm.setDefaultMaxPerRoute（100）；
cm.closeExpiredConnections（）；
RequestConfig RequestConfig=RequestConfig.custom（）.setConnectTimeout（30000）
.setConnectionRequestTimeout（30000）.setSocketTimeout（30000）.build（）；
httpClient=HttpClients.custom（）.setDefaultRequestConfig（requestConfig）.setConnectionManager（cm.build（）；
httpRequestFactory=新的HttpComponents客户端httpRequestFactory（httpClient）；
返回httpRequestFactory；
}
}
public class ResponseInterceptor {
private HTTPInputProperties inputProperties;
private static final Logger LOGGER = LoggerFactory.getLogger(ResponseInterceptor.class);
/**
* Intercepts the errorMessage from the API response and sends appropriate
* information to the Output channel.
*
* @param errorMessage
* @return Message
*/
public Message<String> transform(Message<MessagingException> errorMessage) {
LOGGER.error("Inside Response Interceptor !");
Message<String> responseMessage = null;
try {
if (null != errorMessage && null != errorMessage.getPayload()
&& null != errorMessage.getPayload().getCause()) {
LOGGER.error("Cause is - " + errorMessage.getPayload().getCause().getMessage());
if (errorMessage.getPayload().getCause() instanceof HttpClientErrorException) {
HttpClientErrorException clientError = (HttpClientErrorException) errorMessage.getPayload()
.getCause();
LOGGER.error("Error in ResponseInceptor", clientError);
List<String> errorCodeList = getErrorCodes(inputProperties.getNonRetryErrorCodes());
// intercept Only those errors that are defined as
// nonRetryErrorCodes options in stream definition
if (null != clientError.getStatusCode()
&& errorCodeList.contains(clientError.getStatusCode().toString())) {
LOGGER.error("Error in Response Body", clientError.getResponseBodyAsString());
LOGGER.debug("Non retry message found. Sending to output channel without retrying");
responseMessage = MessageBuilder.withPayload((null == clientError.getResponseBodyAsString() || clientError.getResponseBodyAsString().isEmpty())
? getDefaultPayload(clientError.getStatusCode().toString()) : clientError.getResponseBodyAsString())
.setHeader(BatchHttpClientConstants.HTTP_STATUS, clientError.getStatusCode().toString())
.setHeader(BatchHttpClientConstants.REQUEST_OBJECT,
getFailedMessagePayload(errorMessage))
.copyHeaders(errorMessage.getPayload().getFailedMessage().getHeaders())
.setReplyChannelName(BatchHttpClientConstants.OUTPUT).setErrorChannelName(null).build();
} else {
LOGGER.debug("Status code from API is not present in the nonRetryCodes");
}
} else if (errorMessage.getPayload().getCause() instanceof HttpServerErrorException) {
LOGGER.error("Error is Instance of HttpServerErrorException");
HttpServerErrorException serverError = (HttpServerErrorException) errorMessage.getPayload()
.getCause();
responseMessage = MessageBuilder
.withPayload((null == serverError.getResponseBodyAsString()
|| serverError.getResponseBodyAsString().isEmpty())
? getDefaultPayload(serverError.getStatusCode().toString())
: serverError.getResponseBodyAsString())
.setHeader(BatchHttpClientConstants.HTTP_STATUS, serverError.getStatusCode().toString())
.setHeader(BatchHttpClientConstants.REQUEST_OBJECT, getFailedMessagePayload(errorMessage))
.copyHeaders(errorMessage.getPayload().getFailedMessage().getHeaders())
.setReplyChannelName(BatchHttpClientConstants.OUTPUT).setErrorChannelName(null).build();
}
}
} catch (Exception exception) {
LOGGER.error("Exception occured while transforming errorResponse", exception);
}
// returning null will send the message back to previous module
return responseMessage;
}
private String getDefaultPayload(String httpStatusCode) {
JSONObject jsonResponse = new JSONObject();
if (BatchHttpClientConstants.INTERNAL_SERVER_ERROR.equalsIgnoreCase(httpStatusCode)) {
jsonResponse.put(BatchHttpClientConstants.ID, BatchHttpClientConstants.INTERNAL_SERVER_ERROR_SUBCODE);
jsonResponse.put(BatchHttpClientConstants.TEXT, "Internal Server Error");
} else if (BatchHttpClientConstants.RESOURCE_NOT_FOUND.equalsIgnoreCase(httpStatusCode)) {
jsonResponse.put(BatchHttpClientConstants.ID, BatchHttpClientConstants.RESOURCE_NOT_FOUND_SUBCODE);
jsonResponse.put(BatchHttpClientConstants.TEXT, "Empty Response From the API");
}else{
jsonResponse.put(BatchHttpClientConstants.ID, BatchHttpClientConstants.GENERIC_ERROR_SUBCODE);
jsonResponse.put(BatchHttpClientConstants.TEXT, "Generic Error Occured.");
}
return jsonResponse.toString();
}
/**
* Get Individual error codes using delimiter
*
* @param nonRetryErrorCodes
* @return List of Error Codes as string
*/
private List<String> getErrorCodes(String nonRetryErrorCodes) {
List<String> errorCodeList = new ArrayList<String>();
StringTokenizer st = new StringTokenizer(nonRetryErrorCodes, BatchHttpClientConstants.DELIMITER);
while (st.hasMoreElements()) {
errorCodeList.add(st.nextToken());
}
return errorCodeList;
}
/**
* returns failed Message Payload
*
* @param errorMessage
* @return String
* @throws UnsupportedEncodingException
*/
private byte[] getFailedMessagePayload(Message<MessagingException> errorMessage)
throws UnsupportedEncodingException {
if (null != errorMessage.getPayload().getFailedMessage()
&& null != errorMessage.getPayload().getFailedMessage().getPayload()) {
return errorMessage.getPayload().getFailedMessage().getPayload().toString()
.getBytes(BatchHttpClientConstants.UTF_8);
}
return "".getBytes(BatchHttpClientConstants.UTF_8);
}
public HTTPInputProperties getInputProperties() {
return inputProperties;
}
public void setInputProperties(HTTPInputProperties inputProperties) {
this.inputProperties = inputProperties;
}
}
公共类响应interceptor{
私有HTTPInputProperties-inputProperties；
私有静态最终记录器Logger=LoggerFactory.getLogger（ResponseInterceptor.class）；
/**
*截取API响应中的errorMessage并发送适当的
*将信息发送到输出通道。
*
*@param errorMessage
*@返回消息
*/
公共消息转换（消息错误消息）{
错误（“内部响应拦截器！”）；
消息响应消息=null；
试一试{
if（null！=errorMessage&&null！=errorMessage.getPayload（）
&&null！=errorMessage.getPayload（）.getCause（））{
LOGGER.error（“原因是-”+errorMessage.getPayload（）.getCause（）.getMessage（））；
if（HttpClientErrorException的errorMessage.getPayload（）.getCause（）实例）{
HttpClientErrorException clientError=（HttpClientErrorException）errorMessage.getPayload（）
.getCause（）；
LOGGER.error（“响应接收器错误”，clientError）；
List errorCodeList=getErrorCodes（inputProperties.getNonRetryErrorCodes（））；
//仅截取定义为
//流定义中的非TRYERRORCODES选项
if（null！=clientError.getStatusCode（）
&&errorCodeList.contains（clientError.getStatusCode（）.toString（））{
LOGGER.error（“响应正文中的错误”，clientError.getResponseBodyAsString（））；
debug（“找到非重试消息。不重试发送到输出通道”）；
responseMessage=MessageBuilder.withPayload（（null==clientError.getResponseBodyAsString（）| | clientError.getResponseBodyAsString（）.isEmpty（））
？getDefaultPayload（clientError.getStatusCode（）.toString（））：clientError.getResponseBodyAsString（））
.setHeader（BatchHttpClientConstants.HTTP_状态，clientError.getStatusCode（）.toString（））
.setHeader（BatchHttpClientConstants.REQUEST_对象，
getFailedMessagePayload（errorMessage））