高负载下的recvmmsg/recvfrom/recv块_C_Sockets_Centos6_Recv

高负载下的recvmmsg/recvfrom/recv块

c sockets

高负载下的recvmmsg/recvfrom/recv块,c,sockets,centos6,recv,C,Sockets,Centos6,Recv,我们在Centos 6上有一个应用程序，它在多播地址上调用recvmmsg（），一次读取1024个UDP数据包。当我们在同一个盒子上运行此应用程序的多个实例（所有实例都侦听相同的通信量）时，有时此调用将阻塞多秒，尽管套接字是非阻塞的，并传入MSG\u DONTWAIT。它在所有其他情况下工作正常，但在高负载（50MB/s）下会冻结。当应用程序阻塞时，我们在UDP通信上落后，无法恢复。进程正在使用RR调度程序作为高优先级运行，以避免来自其他进程的干扰。我们尝试在for循环中切换到recvfrom（

我们在Centos 6上有一个应用程序，它在多播地址上调用

recvmmsg（）

，一次读取1024个UDP数据包。当我们在同一个盒子上运行此应用程序的多个实例（所有实例都侦听相同的通信量）时，有时此调用将阻塞多秒，尽管套接字是非阻塞的，并传入

MSG\u DONTWAIT

。它在所有其他情况下工作正常，但在高负载（50MB/s）下会冻结。当应用程序阻塞时，我们在UDP通信上落后，无法恢复。进程正在使用RR调度程序作为高优先级运行，以避免来自其他进程的干扰。我们尝试在for循环中切换到

recvfrom（）

和

recv（）

，也得到了相同的结果

我们在内核源代码中看到的唯一可以阻止这一点的是

中队列锁上的自旋锁\u irqsave（）
。但我不知道在什么情况下这会是一个问题，或者如何防止阻塞，或者这是否真的是一个问题
我不知道下一步该去哪里，所以任何指点都将不胜感激
创建了一个非常简单的程序，可以在我们看到它的一台服务器上复制它（没有粘贴接口检索功能，但这不应该与此相关，如果需要，请告诉我）
recv（）示例：
int main(){
    int fd = socket(AF_INET,SOCK_DGRAM,0);
    int flags = fcntl(fd,F_GETFL,0);
    fcntl(fd,F_SETFL, flags | O_NONBLOCK);
    int reuse = 1;
    setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,(char*)&reuse,sizeof(reuse));
    struct sockaddr_in sockaddr;
    sockaddr.sin_port = htons(4755);
    sockaddr.sin_family = AF_INET;
    sockaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    if(bind(fd,(struct sockaddr*)&sockaddr,sizeof(sockaddr)) < 0){ 
      printf("Failed to bind.\n");
      return 1;
    }

    in_addr_t interface;
    if(!getInterface("192.168.15.255",&interface)){
      printf("Failed to get interface.\n");
      return 1;
    }
    struct ip_mreq imr;
    memset(&imr,0,sizeof(imr));
    imr.imr_multiaddr.s_addr = inet_addr("239.255.61.255");
    imr.imr_interface.s_addr = interface;
    if(!IN_MULTICAST(htonl(imr.imr_multiaddr.s_addr))){
      printf("Group not in multicast.");
      return 1;
    }
    if(setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP, (char*)&imr, sizeof(imr))    < 0){ 
      printf("Failed to add membership, errno: %d.\n",errno);
      return 1;
    }

    int epollInstance = epoll_create1(0);
    struct epoll_event* epollEvents = (struct epoll_event*)malloc(sizeof(struct epoll_event)*8192);
    epollEvents[0].events = EPOLLIN;
    epoll_ctl(epollInstance,EPOLL_CTL_ADD,fd,&epollEvents[0]);

    const int PACKETS_TO_READ = 1024;
    static char receiveBuffer[PACKETS_TO_READ][USHRT_MAX];
    static struct iovec iovecs[PACKETS_TO_READ];
    static struct mmsghdr msgs[PACKETS_TO_READ];
    static struct sockaddr_in sockFrom[PACKETS_TO_READ];
    for (int i = 0; i < PACKETS_TO_READ; i++) {
      iovecs[i].iov_base         = receiveBuffer[i];
      iovecs[i].iov_len          = USHRT_MAX;
      msgs[i].msg_hdr.msg_iov    = &iovecs[i];
      msgs[i].msg_hdr.msg_iovlen = 1;
      msgs[i].msg_hdr.msg_name = &sockFrom[i];
      msgs[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_in );
    }

    struct timeval start;
    struct timeval end;  

    while(1){
      int selected = epoll_wait(epollInstance,epollEvents,8192,10);
      if(selected > 0){ 
        gettimeofday(&start,NULL);
        int numPackets = 0;
        for(int i = 0; i < PACKETS_TO_READ; i++){
        int result = recv(fd,receiveBuffer[0],USHRT_MAX,MSG_DONTWAIT);
        if(result == EAGAIN) break;
          numPackets++;
        }
        gettimeofday(&end,NULL);
        printf("Got %d packets in %lu microseconds\n",numPackets, (end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec);
      }   
    }
}

int main(){
    int fd = socket(AF_INET,SOCK_DGRAM,0);
    int flags = fcntl(fd,F_GETFL,0);
    fcntl(fd,F_SETFL, flags | O_NONBLOCK);
    int reuse = 1;
    setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,(char*)&reuse,sizeof(reuse));
    struct sockaddr_in sockaddr;
    sockaddr.sin_port = htons(4755);
    sockaddr.sin_family = AF_INET;
    sockaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    if(bind(fd,(struct sockaddr*)&sockaddr,sizeof(sockaddr)) < 0){ 
      printf("Failed to bind.\n");
      return 1;
    }

    in_addr_t interface;
    if(!getInterface("192.168.15.255",&interface)){
      printf("Failed to get interface.\n");
      return 1;
    }
    struct ip_mreq imr;
    memset(&imr,0,sizeof(imr));
    imr.imr_multiaddr.s_addr = inet_addr("239.255.61.255");
    imr.imr_interface.s_addr = interface;
    if(!IN_MULTICAST(htonl(imr.imr_multiaddr.s_addr))){
      printf("Group not in multicast.");
      return 1;
    }
    if(setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP, (char*)&imr, sizeof(imr))    < 0){ 
      printf("Failed to add membership, errno: %d.\n",errno);
      return 1;
    }

    int epollInstance = epoll_create1(0);
    struct epoll_event* epollEvents = (struct epoll_event*)malloc(sizeof(struct epoll_event)*8192);
    epollEvents[0].events = EPOLLIN;
    epoll_ctl(epollInstance,EPOLL_CTL_ADD,fd,&epollEvents[0]);

    const int PACKETS_TO_READ = 1024;
    static char receiveBuffer[PACKETS_TO_READ][USHRT_MAX];
    static struct iovec iovecs[PACKETS_TO_READ];
    static struct mmsghdr msgs[PACKETS_TO_READ];
    static struct sockaddr_in sockFrom[PACKETS_TO_READ];
    for (int i = 0; i < PACKETS_TO_READ; i++) {
      iovecs[i].iov_base         = receiveBuffer[i];
      iovecs[i].iov_len          = USHRT_MAX;
      msgs[i].msg_hdr.msg_iov    = &iovecs[i];
      msgs[i].msg_hdr.msg_iovlen = 1;
      msgs[i].msg_hdr.msg_name = &sockFrom[i];
      msgs[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_in );
    }

    struct timeval start;
    struct timeval end;  

    while(1){
      int selected = epoll_wait(epollInstance,epollEvents,8192,10);
      if(selected > 0){ 
        gettimeofday(&start,NULL);
        int numPackets = recvmmsg(fd,msgs,PACKETS_TO_READ,MSG_DONTWAIT,0);
        gettimeofday(&end,NULL);
        printf("Got %d packets in %lu microseconds\n",numPackets, (end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec);
      }   
    }
}

intmain（）{
int fd=插座（AF INET，SOCK DGRAM，0）；
int flags=fcntl（fd，F_GETFL，0）；
fcntl（fd、F|U设置FL、标志| O|U非块）；
int=1；
setsockopt（fd，SOL_SOCKET，SO_REUSEADDR，（char*）和reuse，sizeof（reuse））；
sockaddr中的结构sockaddr_；
sockaddr.sinu端口=htons（4755）；
sockaddr.sin_family=AF_INET；
sockaddr.sin\u addr.s\u addr=htonl（INADDR\u ANY）；
if（bind（fd，（struct sockaddr*）&sockaddr，sizeof（sockaddr））<0{
printf（“绑定失败。\n”）；
返回1；
}
输入地址接口；
if（！getInterface（“192.168.15.255”和接口））{
printf（“获取接口失败。\n”）；
返回1；
}
结构ip_mreq imr；
memset（&imr，0，sizeof（imr））；
imr.imr_multiaddr.s_addr=inet_addr（“239.255.61.255”）；
imr.imr\u interface.s\u addr=接口；
if（！IN_多播（htonl（imr.imr_multiaddr.s_addr）））{
printf（“组不在多播中”）；
返回1；
}
如果（setsockopt（fd，IPPROTO_IP，IP_ADD_MEMBERSHIP，（char*）&imr，sizeof（imr））<0）{
printf（“添加成员身份失败，错误号：%d。\n”，错误号）；
返回1；
}
int epollInstance=epoll_create1（0）；
结构epoll\u事件*epollEvents=（结构epoll\u事件*）malloc（sizeof（结构epoll\u事件）*8192）；
ePolleEvents[0]。事件=EPOLLIN；
epoll_ctl（epollInstance、epoll_ctl_ADD、fd和epollEvents[0]）；
const int PACKETS_至_READ=1024；
静态字符接收缓冲区[PACKETS_TO_READ][USHRT_MAX]；
静态结构iovec iovecs[数据包读取]；
静态结构mmsghdr msgs[数据包读取]；
sockFrom[PACKETS_TO_READ]中的静态结构sockaddr_；
for（int i=0；i0）{
gettimeofday（&start，NULL）；
int numPackets=0；
for（int i=0；i

recvmmsg（）示例：
int main(){
    int fd = socket(AF_INET,SOCK_DGRAM,0);
    int flags = fcntl(fd,F_GETFL,0);
    fcntl(fd,F_SETFL, flags | O_NONBLOCK);
    int reuse = 1;
    setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,(char*)&reuse,sizeof(reuse));
    struct sockaddr_in sockaddr;
    sockaddr.sin_port = htons(4755);
    sockaddr.sin_family = AF_INET;
    sockaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    if(bind(fd,(struct sockaddr*)&sockaddr,sizeof(sockaddr)) < 0){ 
      printf("Failed to bind.\n");
      return 1;
    }

    in_addr_t interface;
    if(!getInterface("192.168.15.255",&interface)){
      printf("Failed to get interface.\n");
      return 1;
    }
    struct ip_mreq imr;
    memset(&imr,0,sizeof(imr));
    imr.imr_multiaddr.s_addr = inet_addr("239.255.61.255");
    imr.imr_interface.s_addr = interface;
    if(!IN_MULTICAST(htonl(imr.imr_multiaddr.s_addr))){
      printf("Group not in multicast.");
      return 1;
    }
    if(setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP, (char*)&imr, sizeof(imr))    < 0){ 
      printf("Failed to add membership, errno: %d.\n",errno);
      return 1;
    }

    int epollInstance = epoll_create1(0);
    struct epoll_event* epollEvents = (struct epoll_event*)malloc(sizeof(struct epoll_event)*8192);
    epollEvents[0].events = EPOLLIN;
    epoll_ctl(epollInstance,EPOLL_CTL_ADD,fd,&epollEvents[0]);

    const int PACKETS_TO_READ = 1024;
    static char receiveBuffer[PACKETS_TO_READ][USHRT_MAX];
    static struct iovec iovecs[PACKETS_TO_READ];
    static struct mmsghdr msgs[PACKETS_TO_READ];
    static struct sockaddr_in sockFrom[PACKETS_TO_READ];
    for (int i = 0; i < PACKETS_TO_READ; i++) {
      iovecs[i].iov_base         = receiveBuffer[i];
      iovecs[i].iov_len          = USHRT_MAX;
      msgs[i].msg_hdr.msg_iov    = &iovecs[i];
      msgs[i].msg_hdr.msg_iovlen = 1;
      msgs[i].msg_hdr.msg_name = &sockFrom[i];
      msgs[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_in );
    }

    struct timeval start;
    struct timeval end;  

    while(1){
      int selected = epoll_wait(epollInstance,epollEvents,8192,10);
      if(selected > 0){ 
        gettimeofday(&start,NULL);
        int numPackets = 0;
        for(int i = 0; i < PACKETS_TO_READ; i++){
        int result = recv(fd,receiveBuffer[0],USHRT_MAX,MSG_DONTWAIT);
        if(result == EAGAIN) break;
          numPackets++;
        }
        gettimeofday(&end,NULL);
        printf("Got %d packets in %lu microseconds\n",numPackets, (end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec);
      }   
    }
}

int main(){
    int fd = socket(AF_INET,SOCK_DGRAM,0);
    int flags = fcntl(fd,F_GETFL,0);
    fcntl(fd,F_SETFL, flags | O_NONBLOCK);
    int reuse = 1;
    setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,(char*)&reuse,sizeof(reuse));
    struct sockaddr_in sockaddr;
    sockaddr.sin_port = htons(4755);
    sockaddr.sin_family = AF_INET;
    sockaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    if(bind(fd,(struct sockaddr*)&sockaddr,sizeof(sockaddr)) < 0){ 
      printf("Failed to bind.\n");
      return 1;
    }

    in_addr_t interface;
    if(!getInterface("192.168.15.255",&interface)){
      printf("Failed to get interface.\n");
      return 1;
    }
    struct ip_mreq imr;
    memset(&imr,0,sizeof(imr));
    imr.imr_multiaddr.s_addr = inet_addr("239.255.61.255");
    imr.imr_interface.s_addr = interface;
    if(!IN_MULTICAST(htonl(imr.imr_multiaddr.s_addr))){
      printf("Group not in multicast.");
      return 1;
    }
    if(setsockopt(fd,IPPROTO_IP,IP_ADD_MEMBERSHIP, (char*)&imr, sizeof(imr))    < 0){ 
      printf("Failed to add membership, errno: %d.\n",errno);
      return 1;
    }

    int epollInstance = epoll_create1(0);
    struct epoll_event* epollEvents = (struct epoll_event*)malloc(sizeof(struct epoll_event)*8192);
    epollEvents[0].events = EPOLLIN;
    epoll_ctl(epollInstance,EPOLL_CTL_ADD,fd,&epollEvents[0]);

    const int PACKETS_TO_READ = 1024;
    static char receiveBuffer[PACKETS_TO_READ][USHRT_MAX];
    static struct iovec iovecs[PACKETS_TO_READ];
    static struct mmsghdr msgs[PACKETS_TO_READ];
    static struct sockaddr_in sockFrom[PACKETS_TO_READ];
    for (int i = 0; i < PACKETS_TO_READ; i++) {
      iovecs[i].iov_base         = receiveBuffer[i];
      iovecs[i].iov_len          = USHRT_MAX;
      msgs[i].msg_hdr.msg_iov    = &iovecs[i];
      msgs[i].msg_hdr.msg_iovlen = 1;
      msgs[i].msg_hdr.msg_name = &sockFrom[i];
      msgs[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_in );
    }

    struct timeval start;
    struct timeval end;  

    while(1){
      int selected = epoll_wait(epollInstance,epollEvents,8192,10);
      if(selected > 0){ 
        gettimeofday(&start,NULL);
        int numPackets = recvmmsg(fd,msgs,PACKETS_TO_READ,MSG_DONTWAIT,0);
        gettimeofday(&end,NULL);
        printf("Got %d packets in %lu microseconds\n",numPackets, (end.tv_sec - start.tv_sec) * 1000000 + end.tv_usec - start.tv_usec);
      }   
    }
}

intmain（）{
int fd=插座（AF INET，SOCK DGRAM，0）；
int flags=fcntl（fd，F_GETFL，0）；
fcntl（fd、F|U设置FL、标志| O|U非块）；
int=1；
setsockopt（fd，SOL_SOCKET，SO_REUSEADDR，（char*）和reuse，sizeof（reuse））；
sockaddr中的结构sockaddr_；
sockaddr.sinu端口=htons（4755）；
sockaddr.sin_family=AF_INET；
sockaddr.sin\u addr.s\u addr=htonl（INADDR\u ANY）；
if（bind（fd，（struct sockaddr*）&sockaddr，sizeof（sockaddr））<0{
printf（“绑定失败。\n”）；
返回1；
}
输入地址接口；
if（！getInterface（“192.168.15.255”和接口））{
printf（“获取接口失败。\n”）；
返回1；
}
结构ip_mreq imr；
memset（&imr，0，sizeof（imr））；
imr.imr_multiaddr.s_addr=inet_addr（“239.255.61.255”）；
imr.imr\u interface.s\u addr=接口；
if（！IN_多播（htonl（imr.imr_multiaddr.s_addr）））{
printf（“组不在多播中”）；
返回1；
}
如果（setsockopt（fd，IPPROTO_IP，IP_ADD_MEMBERSHIP，（char*）&imr，sizeof（imr））<0）{
printf（“添加成员身份失败，错误号：%d。\n”，错误号）；
返回1；
}
int epollInstance=epoll_create1（0）；
结构epoll\u事件*epollEvents=（结构epoll\u事件*）malloc（sizeof（结构epoll\u事件）*8192）；
ePolleEvents[0]。事件=EPOLLIN；
epoll_ctl（epollInstance、epoll_ctl_ADD、fd和epollEvents[0]）；
const int PACKETS_至_READ=1024；
静态字符接收缓冲区[PACKETS_TO_READ][USHRT_MAX]；
静态结构iovec iovecs[数据包读取]；
静态结构mmsghdr msgs[数据包读取]；
sockFrom[PACKETS_TO_READ]中的静态结构sockaddr_；
for（int i=0；i