Python 2.7 单独线程上的Python 2.7 qrequests.map获得响应，但grequests.send没有
我有一个python程序，它为HttpServer创建一个单独的进程来运行。然后我在一个单独的线程上或使用一个 当我在一个单独的线程上使用Python 2.7 单独线程上的Python 2.7 qrequests.map获得响应，但grequests.send没有,python-2.7,grequests,Python 2.7,Grequests,我有一个python程序，它为HttpServer创建一个单独的进程来运行。然后我在一个单独的线程上或使用一个 当我在一个单独的线程上使用grequests.map时，我会得到一个响应。但是，当我使用grequests.send时，使用threading.Pool我没有得到响应 我已经在下面粘贴了代码，演示了我遇到的问题。为什么grequests.map在一个单独的线程上可以工作，而grequests.send在线程中不能工作 使用grequest.send和threading.Pool生成以下
grequests.mapgrequests.sendthreading.Poolgrequests.mapgrequests.send线程中不能工作
使用grequest.send
和threading.Pool
生成以下输出：
[07-19 09:43:26] p99437
{/Users/mhoggan/Development/test_rest/test_request_handler.py:48} INFO - Making requst to endpoint=http://localhost:8081/test/
[07-19 09:43:26] p99437 {/Users/mhoggan/Development/test_rest/request_handler.py:64} INFO - <grequests.send was called...>
[07-19 09:43:26] p99437 {/Users/mhoggan/Development/test_rest/test_request_handler.py:54} INFO - Waiting for response...
[07-19 09:43:31] p99437 {/Users/mhoggan/Development/test_rest/test_request_handler.py:59} CRITICAL - Failed to make request with error <type 'exceptions.AssertionError'>:.
[07-19 09:43:31] p99437 {/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py:212} DEBUG - Starting new HTTP connection (1): localhost
[07-19 09:47:20] p99528 {/Users/mhoggan/Development/test_rest/test_request_handler.py:48} INFO - Making requst to endpoint=http://localhost:8081/test/
[07-19 09:47:20] p99528 {/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py:212} DEBUG - Starting new HTTP connection (1): localhost
127.0.0.1 - - [19/Jul/2017 09:47:20] "GET /test/?token=test HTTP/1.1" 200 -
[07-19 09:47:20] p99528 {/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py:400} DEBUG - http://localhost:8081 "GET /test/?token=test HTTP/1.1" 200 None
[07-19 09:47:20] p99528 {/Users/mhoggan/Development/test_rest/test_request_handler.py:25} INFO - Status code on valid request is 200 content=<html>
<head>
<title>Data.</title>
</head>
<body><p>This is a test.</p>
<p>You accessed path: /test/</p>
<p>With params: {'token': ['test']}</p>
</body>
</html>
[07-19 09:47:20] p99528 {/Users/mhoggan/Development/test_rest/request_handler.py:62} INFO - <grequests.map was called...>
[07-19 09:47:20] p99528 {/Users/mhoggan/Development/test_rest/test_request_handler.py:54} INFO - Waiting for response...
=========================================================================
import grequests
import logging
import threading
from gevent.pool import Pool
logging.basicConfig(format='[%(asctime)s] p%(process)s {%(pathname)s:'
'%(lineno)d} %(levelname)s - %(message)s',
datefmt='%m-%d %H:%M:%S',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
class RequestHandler(object):
def __init__(self, endpoint_url):
self.endpoint_url = endpoint_url
if not self.endpoint_url.startswith("http://"):
self.endpoint_url = "http://{0}".format(self.endpoint_url)
@staticmethod
def _threaded_map(
request,
response_callback,
error_callback=None,
timeout=None
):
grequests.map(
[request],
exception_handler=error_callback,
size=1
)
def request_for_test(
self,
response_callback,
error_callback=None,
timeout=None
):
header = {"Content-type": "application/json"}
payload = {'token': 'test'}
if not error_callback:
error_callback = self.request_exception
request = grequests.get(
self.endpoint_url,
headers=header,
params=payload,
timeout=timeout,
hooks={'response': [response_callback]},
)
args = (request, response_callback, error_callback, timeout,)
thread = threading.Thread(target=self._threaded_map, args=args)
thread.run()
logger.info('<grequests.map was called...>')
#grequests.send(request, pool=Pool(2), stream=False)
#logger.info('<grequests.send was called...>')
import logging
import s2sphere
import threading
import unittest2
from request_handler import RequestHandler
from test_service import (
TestService,
END_POINT_0,
HOST_NAME,
PORT,
)
logging.basicConfig(format='[%(asctime)s] p%(process)s {%(pathname)s:'
'%(lineno)d} %(levelname)s - %(message)s',
datefmt='%m-%d %H:%M:%S',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
def _handle_valid_request(response, **kwargs):
logger.info('Status code on valid request is {0} content={1}'.format(
response.status_code,
response.content
)
)
event.set()
def _handle_error(request, exception):
logger.error('Failing the tests due to request error: '
'{0} -- in request {1}'.format(exception, request))
if __name__ == '__main__':
REQUEST_TIMEOUT = 5 # In seconds
httpd = TestService()
httpd.start()
event = threading.Event()
endpoint_url = 'http://{0}:{1}{2}'.format(
HOST_NAME,
PORT,
END_POINT_0
)
rh = RequestHandler(endpoint_url=endpoint_url)
try:
logger.info('Making requst to endpoint={0}'.format(endpoint_url))
rh.request_for_test(
_handle_valid_request,
error_callback=_handle_error,
timeout=REQUEST_TIMEOUT
)
logger.info('Waiting for response...'.format(endpoint_url))
assert(event.wait(timeout=REQUEST_TIMEOUT))
except Exception as e:
logger.fatal('Failed to make request with error {0}:{1}.'.format(
type(e),
e
)
)
finally:
httpd.stop()
import multiprocessing
from BaseHTTPServer import (
BaseHTTPRequestHandler,
HTTPServer,
)
from urlparse import (
urlparse,
parse_qs
)
END_POINT_0 = '/test/'
HOST_NAME = 'localhost'
PORT = 8081
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
url_path = urlparse(self.path).path
url_query = urlparse(self.path).query
url_params= parse_qs(url_query)
if url_path == END_POINT_0 or url_path == END_POINT_1:
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(
'<html>\n'
' <head>\n'
' <title>Data.</title>\n'
' </head>\n'
' <body><p>This is a test.</p>\n'
' <p>You accessed path: %s</p>\n'
' <p>With params: %s</p>\n'
' </body>\n'
'</html>\n' % (url_path, url_params)
)
class TestService(object):
def __init__(self, server_class=HTTPServer):
self.httpd = server_class((HOST_NAME, PORT), Handler)
self.server_process = None
def _start(self):
self.server_process.daemon = True
self.httpd.serve_forever()
def start(self):
self.stop()
if not self.server_process or not self.server_process.is_alive():
self.server_process = multiprocessing.Process(
target=self._start
)
self.server_process.start()
def stop(self):
if self.server_process:
self.httpd.server_close()
self.server_process.terminate()
=========================================================================
import grequests
import logging
import threading
from gevent.pool import Pool
logging.basicConfig(format='[%(asctime)s] p%(process)s {%(pathname)s:'
'%(lineno)d} %(levelname)s - %(message)s',
datefmt='%m-%d %H:%M:%S',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
class RequestHandler(object):
def __init__(self, endpoint_url):
self.endpoint_url = endpoint_url
if not self.endpoint_url.startswith("http://"):
self.endpoint_url = "http://{0}".format(self.endpoint_url)
@staticmethod
def _threaded_map(
request,
response_callback,
error_callback=None,
timeout=None
):
grequests.map(
[request],
exception_handler=error_callback,
size=1
)
def request_for_test(
self,
response_callback,
error_callback=None,
timeout=None
):
header = {"Content-type": "application/json"}
payload = {'token': 'test'}
if not error_callback:
error_callback = self.request_exception
request = grequests.get(
self.endpoint_url,
headers=header,
params=payload,
timeout=timeout,
hooks={'response': [response_callback]},
)
args = (request, response_callback, error_callback, timeout,)
thread = threading.Thread(target=self._threaded_map, args=args)
thread.run()
logger.info('<grequests.map was called...>')
#grequests.send(request, pool=Pool(2), stream=False)
#logger.info('<grequests.send was called...>')
import logging
import s2sphere
import threading
import unittest2
from request_handler import RequestHandler
from test_service import (
TestService,
END_POINT_0,
HOST_NAME,
PORT,
)
logging.basicConfig(format='[%(asctime)s] p%(process)s {%(pathname)s:'
'%(lineno)d} %(levelname)s - %(message)s',
datefmt='%m-%d %H:%M:%S',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
def _handle_valid_request(response, **kwargs):
logger.info('Status code on valid request is {0} content={1}'.format(
response.status_code,
response.content
)
)
event.set()
def _handle_error(request, exception):
logger.error('Failing the tests due to request error: '
'{0} -- in request {1}'.format(exception, request))
if __name__ == '__main__':
REQUEST_TIMEOUT = 5 # In seconds
httpd = TestService()
httpd.start()
event = threading.Event()
endpoint_url = 'http://{0}:{1}{2}'.format(
HOST_NAME,
PORT,
END_POINT_0
)
rh = RequestHandler(endpoint_url=endpoint_url)
try:
logger.info('Making requst to endpoint={0}'.format(endpoint_url))
rh.request_for_test(
_handle_valid_request,
error_callback=_handle_error,
timeout=REQUEST_TIMEOUT
)
logger.info('Waiting for response...'.format(endpoint_url))
assert(event.wait(timeout=REQUEST_TIMEOUT))
except Exception as e:
logger.fatal('Failed to make request with error {0}:{1}.'.format(
type(e),
e
)
)
finally:
httpd.stop()
import multiprocessing
from BaseHTTPServer import (
BaseHTTPRequestHandler,
HTTPServer,
)
from urlparse import (
urlparse,
parse_qs
)
END_POINT_0 = '/test/'
HOST_NAME = 'localhost'
PORT = 8081
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
url_path = urlparse(self.path).path
url_query = urlparse(self.path).query
url_params= parse_qs(url_query)
if url_path == END_POINT_0 or url_path == END_POINT_1:
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(
'<html>\n'
' <head>\n'
' <title>Data.</title>\n'
' </head>\n'
' <body><p>This is a test.</p>\n'
' <p>You accessed path: %s</p>\n'
' <p>With params: %s</p>\n'
' </body>\n'
'</html>\n' % (url_path, url_params)
)
class TestService(object):
def __init__(self, server_class=HTTPServer):
self.httpd = server_class((HOST_NAME, PORT), Handler)
self.server_process = None
def _start(self):
self.server_process.daemon = True
self.httpd.serve_forever()
def start(self):
self.stop()
if not self.server_process or not self.server_process.is_alive():
self.server_process = multiprocessing.Process(
target=self._start
)
self.server_process.start()
def stop(self):
if self.server_process:
self.httpd.server_close()
self.server_process.terminate()
导入多处理
从BaseHTTPServer导入(
BaseHTTPRequestHandler，
HTTPServer，
)
从URL解析导入(
urlparse，
解析
)
结束点0='/test/'
主机名='localhost'
端口=8081
类处理程序（BaseHTTPRequestHandler）：
def do_获得（自我）：
url\u path=urlparse（self.path）.path
url\u query=urlparse（self.path）.query
url_参数=解析（url_查询）
如果url\u路径==结束点\u 0或url\u路径==结束点\u 1：
自我发送_响应（200）
self.send_标题（“内容类型”、“文本/html”）
self.end_头（）
self.wfile.write(
“\n”
“\n”
'数据。\n'
“\n”
“这是一个测试。\n”
“您访问了路径：%s\n”
“带有参数：%s\n”
“\n”
“\n%”（url\u路径，url\u参数）
)
类TestService（对象）：
def uuu init uuuu（self，server_class=HTTPServer）：
self.httpd=服务器类（（主机名，端口），处理程序）
self.server\u进程=无
def_启动（自）：
self.server\u process.daemon=True
self.httpd.serve_ever（）
def启动（自）：
self.stop（）
如果不是self.server\u进程或不是self.server\u进程。是否处于活动状态（）：
self.server\u process=多处理.process(
目标=自我启动
)
self.server_进程.start（）
def停止（自）：
如果self.server\u进程：
self.httpd.server_close（）
self.server_进程终止（）
原因是grequests.send在引擎盖下使用gevent，以便在调用send的同一线程上调用回调。因此，在线程上使用threading.Event来阻止调用响应回调，直到发出thread.Event信号