Python 扭曲SSL套接字连接减速
如何扩展Twisted服务器以处理数以万计的并发SSL套接字连接 前几百个客户端的连接速度相对较快,但随着数量接近3000,它开始以每秒2个连接的速度爬行 我正在使用下面的循环进行负载测试:Python 扭曲SSL套接字连接减速,python,sockets,ssl,twisted,Python,Sockets,Ssl,Twisted,如何扩展Twisted服务器以处理数以万计的并发SSL套接字连接 前几百个客户端的连接速度相对较快,但随着数量接近3000,它开始以每秒2个连接的速度爬行 我正在使用下面的循环进行负载测试: clients = [] for i in xrange(connections): print i clients.append( ssl.wrap_socket( socket.socket(socket.AF_INET, socket.SOC
clients = []
for i in xrange(connections):
print i
clients.append(
ssl.wrap_socket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM),
ca_certs="server.crt",
cert_reqs=ssl.CERT_REQUIRED
)
)
clients[i].connect(('localhost', 9999))
cProfile:
296644049 function calls (296407530 primitive calls) in 3070.656 seconds
Ordered by: cumulative time
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.001 0.001 3070.656 3070.656 server.py:7(<module>)
1 0.000 0.000 3070.408 3070.408 server.py:148(main)
1 0.000 0.000 3070.406 3070.406 server.py:106(run)
1 0.000 0.000 3070.405 3070.405 base.py:1190(run)
1 0.047 0.047 3070.404 3070.404 base.py:1195(mainLoop)
34383 0.090 0.000 3070.263 0.089 epollreactor.py:367(doPoll)
38696 0.064 0.000 3066.883 0.079 log.py:75(callWithLogger)
38696 0.077 0.000 3066.797 0.079 log.py:70(callWithContext)
38696 0.035 0.000 3066.598 0.079 context.py:117(callWithContext)
38696 0.056 0.000 3066.556 0.079 context.py:61(callWithContext)
38695 0.093 0.000 3066.486 0.079 posixbase.py:572(_doReadOrWrite)
8599 1249.585 0.145 3019.333 0.351 protocol.py:114(getClientsDict)
37582010 1681.445 0.000 1681.445 0.000 {method 'items' of 'dict' objects}
21496 0.114 0.000 1535.798 0.071 tls.py:346(_flushReceiveBIO)
21496 0.026 0.000 1535.793 0.071 tcp.py:199(doRead)
21496 0.017 0.000 1535.718 0.071 tcp.py:218(_dataReceived)
17197 0.033 0.000 1535.701 0.089 tls.py:400(dataReceived)
8597 0.009 0.000 1531.480 0.178 policies.py:119(dataReceived)
8597 0.078 0.000 1531.471 0.178 protocol.py:65(dataReceived)
4300 0.029 0.000 1525.117 0.355 posixbase.py:242(_disconnectSelectable)
4300 0.030 0.000 1524.922 0.355 tcp.py:283(connectionLost)
4300 0.024 0.000 1524.659 0.355 tls.py:463(connectionLost)
4300 0.010 0.000 1524.492 0.355 policies.py:123(connectionLost)
4300 0.119 0.000 1524.471 0.355 protocol.py:50(connectionLost)
4299 0.027 0.000 1523.698 0.354 tcp.py:270(readConnectionLost)
4299 0.135 0.000 1520.228 0.354 protocol.py:88(handleInitialState)
74840519 31.487 0.000 44.916 0.000 __init__.py:348(__getattr__)
考虑到问题中缺少代码,我将一些代码放在一起,看看是否能感受到您所说的效果。从那个实验中,我要说的第一件事是检查并查看在脚本运行时计算机上的内存利用率 我启动了一个标准的google云计算系统(1个vCPU,3.8GB ram)(debian backports wheezy,
apt get update;apt get install python twisted
),并运行了以下(可怕的黑客)代码:
(注意:要运行此程序,我需要对客户端和服务器外壳都执行ulimit-n4096
,否则我将开始获得“太多打开的文件”,即。)
服务端
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ServerFactory, Protocol
class Echo(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
print "Currently %d open connections.\n" % len(self.factory.clients)
def connectionLost(self, reason):
self.factory.clients.remove(self)
print "Lost connection"
def dataReceived(self, data):
"""As soon as any data is received, write it back."""
self.transport.write(data)
class MyServerFactory(ServerFactory):
protocol = Echo
def __init__(self):
self.clients = []
if __name__ == '__main__':
factory = MyServerFactory()
reactor.listenSSL(8000, factory,
ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'))
reactor.run()
cli.py
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ClientFactory, Protocol
class EchoClient(Protocol):
def connectionMade(self):
print "hello, world"
# The following delay is there because as soon as the write
# happens the server will close the connection
reactor.callLater(60, self.transport.write, "hello, world!")
def dataReceived(self, data):
print "Server said:", data
self.transport.loseConnection()
class EchoClientFactory(ClientFactory):
protocol = EchoClient
def __init__(self):
self.stopping = False
def clientConnectionFailed(self, connector, reason):
print "Connection failed - reason ", reason
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
if __name__ == '__main__':
connections = 4000
factory = EchoClientFactory()
for i in xrange(connections):
# the following could certainly be done more elegantly, but I believe
# its a legit use, and given the list in finite, shouldn't be too
# resource intensive of a use... ?
reactor.callLater(i/float(400), reactor.connectSSL,'xx.xx.xx.xx', 8000, factory, ssl.ClientContextFactory())
reactor.run()
在运行和跨越2544个连接时,我的机器严重堵塞,因此很难从中收集数据,但考虑到新的ssh“返回时带有“/bin/bash:cannotallocate memory”,并且当我使用我的serv.py时,有2g的res,客户端有1.4g,我认为可以肯定地说我耗尽了ram
鉴于上面的代码只是一个快速破解,我可能有一些导致内存问题的突出bug——尽管我想我会提供这个想法,因为让你的机器交换肯定是让你的应用程序爬行的好方法。(也许你和我有同样的毛病)
(顺便说一句,对于那些更聪明的扭曲的人,我欢迎评论我做错了什么,烧了这么多内存)鉴于问题中缺少代码,我将一些放在一起,看看我是否体验到了你所说的效果。从那个实验中,我要说的第一件事是检查并查看在脚本运行时计算机上的内存利用率 我启动了一个标准的google云计算系统(1个vCPU,3.8GB ram)(debian backports wheezy,
apt get update;apt get install python twisted
),并运行了以下(可怕的黑客)代码:
(注意:要运行此程序,我需要对客户端和服务器外壳都执行ulimit-n4096
,否则我将开始获得“太多打开的文件”,即。)
服务端
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ServerFactory, Protocol
class Echo(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
print "Currently %d open connections.\n" % len(self.factory.clients)
def connectionLost(self, reason):
self.factory.clients.remove(self)
print "Lost connection"
def dataReceived(self, data):
"""As soon as any data is received, write it back."""
self.transport.write(data)
class MyServerFactory(ServerFactory):
protocol = Echo
def __init__(self):
self.clients = []
if __name__ == '__main__':
factory = MyServerFactory()
reactor.listenSSL(8000, factory,
ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'))
reactor.run()
cli.py
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ClientFactory, Protocol
class EchoClient(Protocol):
def connectionMade(self):
print "hello, world"
# The following delay is there because as soon as the write
# happens the server will close the connection
reactor.callLater(60, self.transport.write, "hello, world!")
def dataReceived(self, data):
print "Server said:", data
self.transport.loseConnection()
class EchoClientFactory(ClientFactory):
protocol = EchoClient
def __init__(self):
self.stopping = False
def clientConnectionFailed(self, connector, reason):
print "Connection failed - reason ", reason
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
if __name__ == '__main__':
connections = 4000
factory = EchoClientFactory()
for i in xrange(connections):
# the following could certainly be done more elegantly, but I believe
# its a legit use, and given the list in finite, shouldn't be too
# resource intensive of a use... ?
reactor.callLater(i/float(400), reactor.connectSSL,'xx.xx.xx.xx', 8000, factory, ssl.ClientContextFactory())
reactor.run()
在运行和跨越2544个连接时,我的机器严重堵塞,因此很难从中收集数据,但考虑到新的ssh“返回时带有“/bin/bash:cannotallocate memory”,并且当我使用我的serv.py时,有2g的res,客户端有1.4g,我认为可以肯定地说我耗尽了ram
鉴于上面的代码只是一个快速破解,我可能有一些导致内存问题的突出bug——尽管我想我会提供这个想法,因为让你的机器交换肯定是让你的应用程序爬行的好方法。(也许你和我有同样的毛病)
(顺便说一句,对于那些更聪明的扭曲的人,我欢迎评论我做错了什么,烧了这么多内存)鉴于问题中缺少代码,我将一些放在一起,看看我是否体验到了你所说的效果。从那个实验中,我要说的第一件事是检查并查看在脚本运行时计算机上的内存利用率 我启动了一个标准的google云计算系统(1个vCPU,3.8GB ram)(debian backports wheezy,
apt get update;apt get install python twisted
),并运行了以下(可怕的黑客)代码:
(注意:要运行此程序,我需要对客户端和服务器外壳都执行ulimit-n4096
,否则我将开始获得“太多打开的文件”,即。)
服务端
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ServerFactory, Protocol
class Echo(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
print "Currently %d open connections.\n" % len(self.factory.clients)
def connectionLost(self, reason):
self.factory.clients.remove(self)
print "Lost connection"
def dataReceived(self, data):
"""As soon as any data is received, write it back."""
self.transport.write(data)
class MyServerFactory(ServerFactory):
protocol = Echo
def __init__(self):
self.clients = []
if __name__ == '__main__':
factory = MyServerFactory()
reactor.listenSSL(8000, factory,
ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'))
reactor.run()
cli.py
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ClientFactory, Protocol
class EchoClient(Protocol):
def connectionMade(self):
print "hello, world"
# The following delay is there because as soon as the write
# happens the server will close the connection
reactor.callLater(60, self.transport.write, "hello, world!")
def dataReceived(self, data):
print "Server said:", data
self.transport.loseConnection()
class EchoClientFactory(ClientFactory):
protocol = EchoClient
def __init__(self):
self.stopping = False
def clientConnectionFailed(self, connector, reason):
print "Connection failed - reason ", reason
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
if __name__ == '__main__':
connections = 4000
factory = EchoClientFactory()
for i in xrange(connections):
# the following could certainly be done more elegantly, but I believe
# its a legit use, and given the list in finite, shouldn't be too
# resource intensive of a use... ?
reactor.callLater(i/float(400), reactor.connectSSL,'xx.xx.xx.xx', 8000, factory, ssl.ClientContextFactory())
reactor.run()
在运行和跨越2544个连接时,我的机器严重堵塞,因此很难从中收集数据,但考虑到新的ssh“返回时带有“/bin/bash:cannotallocate memory”,并且当我使用我的serv.py时,有2g的res,客户端有1.4g,我认为可以肯定地说我耗尽了ram
鉴于上面的代码只是一个快速破解,我可能有一些导致内存问题的突出bug——尽管我想我会提供这个想法,因为让你的机器交换肯定是让你的应用程序爬行的好方法。(也许你和我有同样的毛病)
(顺便说一句,对于那些更聪明的扭曲的人,我欢迎评论我做错了什么,烧了这么多内存)鉴于问题中缺少代码,我将一些放在一起,看看我是否体验到了你所说的效果。从那个实验中,我要说的第一件事是检查并查看在脚本运行时计算机上的内存利用率 我启动了一个标准的google云计算系统(1个vCPU,3.8GB ram)(debian backports wheezy,
apt get update;apt get install python twisted
),并运行了以下(可怕的黑客)代码:
(注意:要运行此程序,我需要对客户端和服务器外壳都执行ulimit-n4096
,否则我将开始获得“太多打开的文件”,即。)
服务端
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ServerFactory, Protocol
class Echo(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
print "Currently %d open connections.\n" % len(self.factory.clients)
def connectionLost(self, reason):
self.factory.clients.remove(self)
print "Lost connection"
def dataReceived(self, data):
"""As soon as any data is received, write it back."""
self.transport.write(data)
class MyServerFactory(ServerFactory):
protocol = Echo
def __init__(self):
self.clients = []
if __name__ == '__main__':
factory = MyServerFactory()
reactor.listenSSL(8000, factory,
ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'))
reactor.run()
cli.py
#!/usr/bin/python
from twisted.internet import ssl, reactor
from twisted.internet.protocol import ClientFactory, Protocol
class EchoClient(Protocol):
def connectionMade(self):
print "hello, world"
# The following delay is there because as soon as the write
# happens the server will close the connection
reactor.callLater(60, self.transport.write, "hello, world!")
def dataReceived(self, data):
print "Server said:", data
self.transport.loseConnection()
class EchoClientFactory(ClientFactory):
protocol = EchoClient
def __init__(self):
self.stopping = False
def clientConnectionFailed(self, connector, reason):
print "Connection failed - reason ", reason
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
if not self.stopping:
self.stopping = True
reactor.callLater(10,reactor.stop)
if __name__ == '__main__':
connections = 4000
factory = EchoClientFactory()
for i in xrange(connections):
# the following could certainly be done more elegantly, but I believe
# its a legit use, and given the list in finite, shouldn't be too
# resource intensive of a use... ?
reactor.callLater(i/float(400), reactor.connectSSL,'xx.xx.xx.xx', 8000, factory, ssl.ClientContextFactory())
reactor.run()
在运行和跨越2544个连接时,我的机器严重堵塞,因此很难从中收集数据,但考虑到新的ssh“返回时带有“/bin/bash:cannotallocate memory”,并且当我使用我的serv.py时,有2g的res,客户端有1.4g,我认为可以肯定地说我耗尽了ram
鉴于上面的代码只是一个快速破解,我可能有一些导致内存问题的突出bug——尽管我想我会提供这个想法,因为让你的机器交换肯定是让你的应用程序爬行的好方法。(也许你和我有同样的毛病)
(顺便说一句,对于那些更聪明的扭曲的人,我欢迎发表我的评论。)