Nginx高容量流量负载平衡
在过去的3周里,我们一直在测试Nginx作为负载平衡。 目前,我们无法成功处理超过1000个请求/秒和18K的活动连接。 当我们到达上述数字时,Nginx开始挂起,并返回超时代码。 获得响应的唯一方法是大幅减少连接数量 我必须注意的是,我的服务器每天都能处理这么多的流量,我们目前使用的是一个简单的round rubin DNS平衡 我们使用具有以下硬件的专用服务器:Nginx高容量流量负载平衡,nginx,dns,kernel,load-balancing,tomcat6,Nginx,Dns,Kernel,Load Balancing,Tomcat6,在过去的3周里,我们一直在测试Nginx作为负载平衡。 目前,我们无法成功处理超过1000个请求/秒和18K的活动连接。 当我们到达上述数字时,Nginx开始挂起,并返回超时代码。 获得响应的唯一方法是大幅减少连接数量 我必须注意的是,我的服务器每天都能处理这么多的流量,我们目前使用的是一个简单的round rubin DNS平衡 我们使用具有以下硬件的专用服务器: 英特尔至强E5620处理器 16GB内存 2T SATA硬盘驱动器 1Gb/s连接 操作系统:CentOS 5.8 我们需要平
- 英特尔至强E5620处理器
- 16GB内存
- 2T SATA硬盘驱动器
- 1Gb/s连接
- 操作系统:CentOS 5.8
用户nginx;
工人10人;
工人档案200000份;
error\u log/var/log/nginx/error.log warn;
pid/var/run/nginx.pid;
事件{
1万名工人;
使用epoll;
多重接受;
}
http{
包括/etc/nginx/mime.types;
默认_类型应用程序/八位字节流;
日志格式主“$remote\u addr-$remote\u user[$time\u local]“$request””
“$status$body\u bytes\u sent”$http\u referer”
“$http_user_agent”“$http_x_forwarded_for””;
#access\u log/var/log/nginx/access.log main;
访问/注销;
发送文件到;
tcp_nopush on;
保持生命超时65;
重置\u timedout\u连接打开;
gzip on;
gzip_comp_1级;
包括/etc/nginx/conf.d/*.conf;
}
servers.conf:
#设置上游(服务器负载平衡)
#HTTP流
上游adsbar{
最少康涅狄格州;
服务器xx.xx.xx.34最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.36最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.37最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.39最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.40最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.42最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.43最大失败次数=2次失败超时时间=15s;
}
#HTTPS流
上行adsbar ssl{
最少康涅狄格州;
服务器xx.xx.xx.34:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.36:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.37:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.39:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.40:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.42:443最大失败次数=2次失败超时时间=15s;
服务器xx.xx.xx.43:443最大失败次数=2次失败超时时间=15s;
}
#HTTP
服务器{
听xxx.xxx.xxx.xxx:8080;
服务器名称www.mycompany.com;
地点/{
代理设置头主机$Host;
#因此保留了原始HTTP主机头
代理集头X-Real-IP$remote\u addr;
#客户端的IP地址(可能是代理本身)
proxy\u set\u header X-Forwarded-For$proxy\u add\u X\u Forwarded\u For;
代理通行证http://adsbar;
}
}
#HTTPS
服务器{
听xxx.xxx.xxx.xxx:8443;
服务器名称www.mycompany.com;
ssl-on;
ssl_certificate/etc/pki/tls/certs/mycompany.crt;
#SSL证书的路径;
ssl证书密钥/etc/pki/tls/private/mycompany.key;
#SSL证书密钥的路径;
地点/{
代理设置头主机$Host;
#因此保留了原始HTTP主机头
代理集头X-Real-IP$remote\u addr;
#客户端的IP地址(可能是代理本身)
proxy\u set\u header X-Forwarded-For$proxy\u add\u X\u Forwarded\u For;
代理通行证https://adsbar-ssl;
}
}
服务器{
听xxx.xxx.xxx.xxx:61709;
位置/nginx\u状态{
stub_状态为on;
访问/注销;
允许127.0.0.1;
否认一切;
}
}
sysctl.conf:
Red Hat Linux的内核sysctl配置文件
#
#对于二进制值,
0已禁用,1已启用。参见sysctl(8)和
#sysctl.conf(5)获取更多详细信息。
#控制IP数据包转发
net.ipv4.ip_forward=0
#控制源路由验证
net.ipv4.conf.default.rp_过滤器=1
#不接受源路由
net.ipv4.conf.default.accept\u source\u route=0
#控制内核的系统请求调试功能
kernel.sysrq=1
#控制核心转储是否将PID附加到核心文件名
#用于调试多线程应用程序
kernel.core_使用_pid=1
#控制TCP syncookies的使用
net.ipv4.tcp_syncookies=1
#控制消息的最大大小(以字节为单位)
kernel.msgmnb=65536
#控制mesage队列的默认最大大小
kernel.msgmax=65536
#控制最大共享段大小(以字节为单位)
kernel.shmmax=68719476736
#控制共享内存段的最大数量(以页为单位)
kernel.shmall=4294967296
fs.file-max=120000
net.ipv4.ip_conntrack_max=131072
net.ipv4.tcp_max_syn_backlog=8196
net.ipv4.tcp\u fin\u超时=25
net.ipv4.tcp_keepalive_time=3600
net.ipv4.ip_本地_端口_范围=1024 65000
net.ipv4.tcp\u rmem=4096 25165824 25165824
net.core.rmem_max=25165824
net.core.rmem\u默认值=25165824
net.ipv4.tcp_wmem=4096 65536 25165824
net.core.wmem_max=25165824
net.core.wmem_默认值=65536
net.core.optmem_max=25165824
net.core.netdev_max_backlog=2500
net.ipv4.tcp_tw_recycle=1
net.ipv4.tcp_tw_重用=1
非常感谢任何帮助、指导和想法。nginx肯定能够处理1000个以上的请求/秒(我在使用jmeter的廉价笔记本电脑上玩时,nginx的请求/秒大约为2800个,使用两个内核的一半) 您使用的是epoll,这是最佳选项
worker_processes 4;
worker_rlimit_nofile 999999;
events {
worker_connections 32768;
use epoll;
multi_accept on;
}
net.ipv4.tcp_rmem = 4096 4096 25165824
net.ipv4.tcp_wmem = 4096 4096 25165824
fs.file-max=999999
# This number should be, at maximum, the number of CPU cores on your system.
# (since nginx doesn't benefit from more than one worker per CPU.)
worker_processes 24;
# Number of file descriptors used for Nginx. This is set in the OS with 'ulimit -n 200000'
# or using /etc/security/limits.conf
worker_rlimit_nofile 200000;
# only log critical errors
error_log /var/log/nginx/error.log crit
# Determines how many clients will be served by each worker process.
# (Max clients = worker_connections * worker_processes)
# "Max clients" is also limited by the number of socket connections available on the system (~64k)
worker_connections 4000;
# essential for linux, optmized to serve many clients with each thread
use epoll;
# Accept as many connections as possible, after nginx gets notification about a new connection.
# May flood worker_connections, if that option is set too low.
multi_accept on;
# Caches information about open FDs, freqently accessed files.
# Changing this setting, in my environment, brought performance up from 560k req/sec, to 904k req/sec.
# I recommend using some varient of these options, though not the specific values listed below.
open_file_cache max=200000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;
# Buffer log writes to speed up IO, or disable them altogether
#access_log /var/log/nginx/access.log main buffer=16k;
access_log off;
# Sendfile copies data between one FD and other from within the kernel.
# More efficient than read() + write(), since the requires transferring data to and from the user space.
sendfile on;
# Tcp_nopush causes nginx to attempt to send its HTTP response head in one packet,
# instead of using partial frames. This is useful for prepending headers before calling sendfile,
# or for throughput optimization.
tcp_nopush on;
# don't buffer data-sends (disable Nagle algorithm). Good for sending frequent small bursts of data in real time.
tcp_nodelay on;
# Timeout for keep-alive connections. Server will close connections after this time.
keepalive_timeout 30;
# Number of requests a client can make over the keep-alive connection. This is set high for testing.
keepalive_requests 100000;
# allow the server to close the connection after a client stops responding. Frees up socket-associated memory.
reset_timedout_connection on;
# send the client a "request timed out" if the body is not loaded by this time. Default 60.
client_body_timeout 10;
# If the client stops reading data, free up the stale client connection after this much time. Default 60.
send_timeout 2;
# Compression. Reduces the amount of data that needs to be transferred over the network
gzip on;
gzip_min_length 10240;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml;
gzip_disable "MSIE [1-6]\.";
# Increase system IP port limits to allow for more connections
net.ipv4.ip_local_port_range = 2000 65000
net.ipv4.tcp_window_scaling = 1
# number of packets to keep in backlog before the kernel starts dropping them
net.ipv4.tcp_max_syn_backlog = 3240000
# increase socket listen backlog
net.core.somaxconn = 3240000
net.ipv4.tcp_max_tw_buckets = 1440000
# Increase TCP buffer sizes
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.tcp_congestion_control = cubic
hash $remote_addr consistent;