Python Asyncio如何重用套接字
如何在asyncio中重用服务器的套接字?而不是为每个查询创建新连接 这是我的密码Python Asyncio如何重用套接字,python,python-3.x,python-asyncio,Python,Python 3.x,Python Asyncio,如何在asyncio中重用服务器的套接字?而不是为每个查询创建新连接 这是我的密码 async def lookup(server, port, query, sema): async with sema as sema: try: reader, writer = await asyncio.open_connection(server, port) except: return {} wr
async def lookup(server, port, query, sema):
async with sema as sema:
try:
reader, writer = await asyncio.open_connection(server, port)
except:
return {}
writer.write(query.encode("ISO-8859-1"))
await writer.drain()
data = b""
while True:
d = await reader.read(4096)
if not d:
break
data += d
writer.close()
data = data.decode("ISO-8859-1")
return data
您可以通过将读写器对存储到全局字典来创建连接缓存
# at top-level
connections = {}
然后在lookup
中,将对open\u connection
的调用替换为首先检查dict的代码:
if (server, port) in connections:
reader, writer = connections[server, port]
else:
reader, writer = await asyncio.open_connection(server, port)
connections[server, port] = reader, writer
您只需调用
asyncio.open\u连接(服务器、端口)
coroutine一次,并继续使用读写器(当然,前提是服务器不只是在其一端关闭连接)
我将在一个单独的异步上下文管理器对象中为您的连接执行此操作,并使用来管理连接,这样您就可以为许多并发任务创建和重用套接字连接。通过使用(异步)上下文管理器,Python确保在代码完成时通知连接,以便将连接释放回池:
import asyncio
import contextlib
from collections import OrderedDict
from types import TracebackType
from typing import Any, List, Optional, Tuple, Type
try: # Python 3.7
base = contextlib.AbstractAsyncContextManager
except AttributeError:
base = object # type: ignore
Server = str
Port = int
Host = Tuple[Server, Port]
class ConnectionPool(base):
def __init__(
self,
max_connections: int = 1000,
loop: Optional[asyncio.AbstractEventLoop] = None,
):
self.max_connections = max_connections
self._loop = loop or asyncio.get_event_loop()
self._connections: OrderedDict[Host, List["Connection"]] = OrderedDict()
self._semaphore = asyncio.Semaphore(max_connections)
async def connect(self, server: Server, port: Port) -> "Connection":
host = (server, port)
# enforce the connection limit, releasing connections notifies
# the semaphore to release here
await self._semaphore.acquire()
connections = self._connections.setdefault(host, [])
# find an un-used connection for this host
connection = next((conn for conn in connections if not conn.in_use), None)
if connection is None:
# disconnect the least-recently-used un-used connection to make space
# for a new connection. There will be at least one.
for conns_per_host in reversed(self._connections.values()):
for conn in conns_per_host:
if not conn.in_use:
await conn.close()
break
reader, writer = await asyncio.open_connection(server, port)
connection = Connection(self, host, reader, writer)
connections.append(connection)
connection.in_use = True
# move current host to the front as most-recently used
self._connections.move_to_end(host, False)
return connection
async def close(self):
"""Close all connections"""
connections = [c for cs in self._connections.values() for c in cs]
self._connections = OrderedDict()
for connection in connections:
await connection.close()
def _remove(self, connection):
conns_for_host = self._connections.get(connection._host)
if not conns_for_host:
return
conns_for_host[:] = [c for c in conns_for_host if c != connection]
def _notify_release(self):
self._semaphore.release()
async def __aenter__(self) -> "ConnectionPool":
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc: Optional[BaseException],
tb: Optional[TracebackType],
) -> None:
await self.close()
def __del__(self) -> None:
connections = [repr(c) for cs in self._connections.values() for c in cs]
if not connections:
return
context = {
"pool": self,
"connections": connections,
"message": "Unclosed connection pool",
}
self._loop.call_exception_handler(context)
class Connection(base):
def __init__(
self,
pool: ConnectionPool,
host: Host,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
):
self._host = host
self._pool = pool
self._reader = reader
self._writer = writer
self._closed = False
self.in_use = False
def __repr__(self):
host = f"{self._host[0]}:{self._host[1]}"
return f"Connection<{host}>"
@property
def closed(self):
return self._closed
def release(self) -> None:
self.in_use = False
self._pool._notify_release()
async def close(self) -> None:
if self._closed:
return
self._closed = True
self._writer.close()
self._pool._remove(self)
try:
await self._writer.wait_closed()
except AttributeError: # wait_closed is new in 3.7
pass
def __getattr__(self, name: str) -> Any:
"""All unknown attributes are delegated to the reader and writer"""
if self._closed or not self.in_use:
raise ValueError("Can't use a closed or unacquired connection")
if hasattr(self._reader, name):
return getattr(self._reader, name)
return getattr(self._writer, name)
async def __aenter__(self) -> "Connection":
if self._closed or not self.in_use:
raise ValueError("Can't use a closed or unacquired connection")
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc: Optional[BaseException],
tb: Optional[TracebackType],
) -> None:
self.release()
def __del__(self) -> None:
if self._closed:
return
context = {"connection": self, "message": "Unclosed connection"}
self._pool._loop.call_exception_handler(context)
请注意,声明在每次查询后连接都将关闭。如果您连接到端口43上的标准WHOIS服务,则无需重新使用套接字
在这种情况下,读取器将达到EOF(
reader.at_EOF()
为true),任何进一步的读取尝试都不会返回任何结果(reader.read(…)
将始终返回空的b'
值)。在超时后远程端终止套接字连接之前,写入写入器不会是错误的。您可以将所有需要的内容写入连接,但WHOIS服务器将忽略这些查询。您连接的是哪种服务器?标准端口43 whois协议没有任何套接字重用选项;连接完成后,服务器将立即关闭连接。这就是为什么要一直读到给出一个空的响应;这意味着读卡器连接现在已关闭。无法读取更多数据。对于保持连接打开的协议,将有一种方法可以检测消息何时完成(如CRLF组合、固定大小的消息或以长度信息开头的消息)。假设您使用的循环在reader.read()
返回空值时结束,那么您几乎可以肯定使用的是标准的whois连接。实际上,我正在连接whois服务器,这很有意义,因为我无法通过一些手动测试实现套接字重用。谢谢请纠正我,如果任何以下是错误的想法。多个task
s可以在ConnectionPool.connect的while循环中等待多个未来(fut
s)。这些未来只是作为一种等待的手段。事件循环周期性地检查正在等待的事情;当它找到一个特定的fut
时,while循环终止,fut
的task
获得连接。如果所有这些都是正确的,这是否意味着事件循环中有多个已挂起的ConnectionPool.connect
方法实例?TIA@wwii:是的,这就是使用期货的意义所在。connect()
调用将不会返回,直到可以为该任务提供一个免费连接,我们可以通过让其他任务释放连接来检测该点,并通过未来发出信号,表明可能存在可用的连接。我很好奇,为什么要保持一个显式的获取计数和一个等待者的dict。获取一个asyncio.Semaphore
,它会自动完成所有工作,这难道不会让代码变得更简单吗?@user4815162342:习惯,胜过一切;我指的是我的漏桶实现供参考,以及其他连接池。aiohttp也使用自己的服务员,但对漏桶和aiohttp池的要求要复杂一些。在这种情况下,信号灯就可以了。@user4815162342谢谢。我还没有在第8页上运行最后一次编辑,哎呀。
async def lookup(pool, server, port, query):
try:
conn = await pool.connect(server, port)
except (ValueError, OSError):
return {}
async with conn:
conn.write(query.encode("ISO-8859-1"))
await conn.drain()
data = b""
while True:
d = await conn.read(4096)
if not d:
break
data += d
data = data.decode("ISO-8859-1")
return data