Python 了解提升远程断开连接（“远程端闭合连接”）；

我正在抓取twitter，试图获取twitter用户列表中的好友/用户。我正在OSX 10.13上使用tweepy和python 3.6.5。一个缩写代码块：

def get_friends_for_each_twitter_user(UserL=None, Name=None):
   .
   . # Auth keys and such
   .
   for user in UserL:  ### This is a list of USER class with the below fields ###
        ### Handle protected users ###
        if(user.protected == True):
            user.friendsL = "protected"
            continue
        screenNameL=[]
        friendIDL=[]
        friendL=[]
        friendScreenNameL=[]
        ### Get IDs of people that this user follows (i.e. 'friends') ###
        for page in tweepy.Cursor(api.friends_ids, screen_name=user.screenName).pages():
            friendIDL.extend(page)
            time.sleep(60)
        ## Loop through IDs, get user profile, keep only friends' screen name ###
        for i in range(0, len(friendIDL), 100):
            friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
        ### Keep only screen name ###
        for friend in friendL:
            friendScreenNameL.append(friend._json['screen_name'])
        user.friendsL = friendScreenNameL

当我这样做的时候，在收集了十几个用户的

朋友

（即用户遵循的配置文件）之后，我得到了以下错误：

Traceback (most recent call last):
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
    chunked=chunked)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 387, in _make_request
    six.raise_from(e, None)
  File "<string>", line 2, in raise_from
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 383, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1331, in getresponse
    response.begin()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 297, in begin
    version, status, reason = self._read_status()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 266, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 440, in send
    timeout=timeout
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 639, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/util/retry.py", line 357, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 601, in urlopen
    chunked=chunked)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 387, in _make_request
    six.raise_from(e, None)
  File "<string>", line 2, in raise_from
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/connectionpool.py", line 383, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1331, in getresponse
    response.begin()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 297, in begin
    version, status, reason = self._read_status()
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 266, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 190, in execute
    proxies=self.api.proxy)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 618, in send
    r = adapter.send(request, **kwargs)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 490, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pdb.py", line 1667, in main
    pdb._runscript(mainpyfile)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pdb.py", line 1548, in _runscript
    self.run(statement)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/bdb.py", line 434, in run
    exec(cmd, globals, locals)
  File "<string>", line 1, in <module>
  File "/Users/myusername/Code/Python/hair_prod/src/main.py", line 170, in <module>
    main()
  File "/Users/myusername/Code/Python/hair_prod/src/main.py", line 141, in main
    get_friends_for_each_twitter_user(UserL=tresemmeUserL, Name="Tresemme")
  File "src/twitter_scraper.py", line 187, in get_friends_for_each_twitter_user
    friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/api.py", line 336, in lookup_users
    return self._lookup_users(post_data=post_data)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 250, in _call
    return method.execute()
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 192, in execute
    six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2])
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/tweepy/binder.py", line 190, in execute
    proxies=self.api.proxy)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/sessions.py", line 618, in send
    r = adapter.send(request, **kwargs)
  File "/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py", line 490, in send
    raise ConnectionError(err, request=request)
tweepy.error.TweepError: Failed to send request: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
Uncaught exception. Entering post mortem debugging
Running 'cont' or 'step' will restart the program
> /Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/requests/adapters.py(490)send()
-> raise ConnectionError(err, request=request)

回溯（最近一次呼叫最后一次）：
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，urlopen中第601行
分块=分块）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，第387行，在请求中
六、从（e，无）中提高
文件“”，第2行，从
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，第383行，在请求中
httplib_response=conn.getresponse（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第1331行，在getresponse中
response.begin（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第297行，在begin中
版本、状态、原因=self.\u读取\u状态（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第266行，处于“读取”状态
升起RemoteDisconnected（“远程端关闭连接，不带”
http.client.RemoteDisconnected:远程端关闭连接，无响应
在处理上述异常期间，发生了另一个异常：
回溯（最近一次呼叫最后一次）：
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/requests/adapters.py”，第440行，在send中
超时=超时
urlopen中的文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urlib3/connectionpool.py”，第639行
_stacktrace=sys.exc_info（）[2]）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/util/retry.py”，第357行，增量
升起六个。重新升起（类型（错误），错误，_stacktrace）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site-packages/urllib3/packages/six.py”，第685行，重新登录
通过_回溯（tb）提升值
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，urlopen中第601行
分块=分块）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，第387行，在请求中
六、从（e，无）中提高
文件“”，第2行，从
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/urllib3/connectionpool.py”，第383行，在请求中
httplib_response=conn.getresponse（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第1331行，在getresponse中
response.begin（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第297行，在begin中
版本、状态、原因=self.\u读取\u状态（）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/http/client.py”，第266行，处于“读取”状态
升起RemoteDisconnected（“远程端关闭连接，不带”
urllib3.exceptions.ProtocolError:（“连接中止”，RemoteDisconnected（“远程端关闭连接，无响应）”，）
在处理上述异常期间，发生了另一个异常：
回溯（最近一次呼叫最后一次）：
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/tweepy/binder.py”，第190行，在execute中
代理服务器=self.api.proxy）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/requests/sessions.py”，请求中第508行
resp=自我发送（准备，**发送）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/requests/sessions.py”，第618行，在send中
r=适配器.send（请求，**kwargs）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/requests/adapters.py”，第490行，在send中
raise CONNECTIONERR（错误，请求=请求）
requests.exceptions.ConnectionError:（“连接已中止”）、RemoteDisconnected（“远程端关闭连接而无响应”））
在处理上述异常期间，发生了另一个异常：
回溯（最近一次呼叫最后一次）：
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/pdb.py”，第1667行，主要内容
pdb.\u运行脚本（mainpyfile）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/pdb.py”，第1548行，在运行脚本中
self.run（语句）
文件“/usr/local/ceral/python/3.6.5/Frameworks/python.framework/Versions/3.6/lib/python3.6/bdb.py”，第434行，正在运行
执行官（命令、全局命令、本地命令）
文件“”，第1行，在
文件“/Users/myusername/Code/Python/hair\u prod/src/main.py”，第170行，在
main（）
文件“/Users/myusername/Code/Python/hair_prod/src/main.py”，第141行，在main中
为每个twitter用户获取好友（UserL=tresemmeUserL，Name=“Tresemme”）
文件“src/twitter\u scraper.py”，第187行，为每个twitter用户获取好友
extend（api.lookup\u users（user\u id=friendIDL[i:i+100]））
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/tweepy/api.py”，第336行，在查找用户中
返回self.\u lookup\u用户（post\u数据=post\u数据）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/tweepy/binder.py”，第250行，在调用中
返回方法execute（）
文件“/Users/myusername/.local/virtualenvs/python3.6/lib/python3.6/site packages/tweepy/binder.py”，第192行，在execute中
6.重新发送（TweepError，TweepError（'发送请求失败：%s'%e]，sys.exc_info（）[2]）
文件“/用户/my
api = tweepy.api.API(..., retry_count=3)

def get_friends_for_each_twitter_user(UserL=None, Name=None):
    consumerKey =  #your value here
    consumerSecret = #your value here
    auth = tweepy.AppAuthHandler(consumerKey, consumerSecret)  ### Supposedly faster
    api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)  ## Now I don't have to handle rate limiting myself

    for user in UserL:
        accountStatus = 'active'
        if(user.protected == True):
            user.friendsL = "protected"
            continue
        screenNameL=[]
        friendIDL=[]
        friendL=[]
        friendScreenNameL=[]
        #### TWITTER LIMITS US #####
        try :
            for page in tweepy.Cursor(api.friends_ids, screen_name=user.screenName).pages():
                friendIDL.extend(page)
        except tweepy.TweepError as error :
            if(error.__dict__['api_code'] == 34):
                accountStatus = 'dead'
                print("...{} is dead".format(user.screenName))
                continue
            else:
                raise

        for i in range(0, len(friendIDL), 100):
            ### This handles when exception occurs (probably due to connection issues)
            ### When exception occurs, sleeps then retries. I don't notice this error
            ### when I'm running on corporate Wifi, maybe my router just sucks
            while True:
                try :
                    friendL.extend(api.lookup_users(user_ids=friendIDL[i:i+100]))
                except tweepy.TweepError as error :
                    print("...Exception for {} : api_code {}".format(user.screenName,
                          error.__dict__['api_code']))
                    time.sleep(5)
                    continue
                break