Python请求模块使用多线程引发ConnectionError

我有个棘手的问题。我有一个脚本，可以将文件异步上传到我的云文件帐户…例如，使用25个线程。我遇到的问题是，如果文件大于5MB（例如…25MB），我会从requests模块获得很多except requests.exceptions.RequestException。这已经成为一个主要问题，我真的被卡住了。我想坚持这一点，而不是扭曲或everlet，并将感谢任何帮助，请

编辑：在对代码进行修改之后，通过多线程处理，我得到了“exception requests.exceptions.ConnectionError”的提示。例如，为什么只有8次并发上传？这似乎是脚本的错误，而不是系统的错误，因为系统可以轻松处理

def do_the_uploads(file_list, file_quantity,
                   retry_list, authenticate):
    """The uploading engine"""
    value = raw_input(
        "\nPlease enter how many conncurent "
        "uploads you want at one time(example: 200)> ")
    value = int(value)
    logger.info('{} conncurent uploads will be used.'.format(value))

    confirm = raw_input(
        "\nProceed to upload files? Enter [Y/y] for yes: ").upper()
    if confirm == "Y":
        sys.stdout.write("\x1b[2J\x1b[H")
        q = Queue.Queue()

        def worker():
            while True:
                item = q.get()
                upload_file(item, file_quantity, retry_list, authenticate)
                q.task_done()

        for i in range(value):
            t = Thread(target=worker)
            t.setDaemon(True)
            t.start()

        for item in file_list:
            q.put(item)

        q.join()
        print "Finished. Cleaning up processes...",
        #Allowing the threads to cleanup
        time.sleep(4)
        print "done."

---

---

听起来你的云文件帐户限制了并发连接。

def upload_file(file_obj, file_quantity, retry_list, authenticate):
    """Uploads a file. One file per it's own thread. No batch style. This way if one upload
       fails no others are effected."""
    absolute_path_filename, filename, dir_name, token, url = file_obj
    url = url + dir_name + '/' + filename

    src_md5 = md5sum(absolute_path_filename)

    if src_md5:
        pass
    else:
        msg = (
            'Filename \"{}\" is missing md5 checksum!'
            ' This will not stop upload, but md5 check will not be checked!'.format(filename))
        logger.error(msg)

    header_collection = {
        "X-Auth-Token": token,
        "ETag": src_md5}

    print "\nUploading " + absolute_path_filename
    for i in range(5):
        try:
            with open(absolute_path_filename) as f:
                r = requests.put(url, data=f, headers=header_collection, timeout=1)
            if r.status_code == 201:
                if src_md5 == r.headers['etag']:
                    file_quantity.deduct()
                    msg = (
                        'File \"{}\" successfully'
                        ' loaded with verified md5 \"{}\"'.format(filename, src_md5))
                    logger.info(msg)
                    msg = (
                        '{} out of {} files left'.format(file_quantity.quantity, file_quantity.total))
                    logger.info(msg)
                    break
                else:
                    msg = (
                        'File \"{}\" checksum verification failed with'
                        ' \"{}\". Retrying upload'.format(filename, src_md5))
                    logger.error(msg)
                    if i is not 4:
                        continue
                    else:
                        msg = (
                            'File \"{}\" checksum verification failed with'
                            ' \"{}\". \nThis was the 5th and final try.'
                            ' File will be placed on the retry list'.format(filename, src_md5))
                        logger.error(msg)
                        retry_list.append(file_obj)

            if r.status_code == 401:
                msg = ('Token has expired for upload \"{}\"'
                       ' Renewing token...'.format(filename))
                logger.error(msg)
                authenticate.getToken()
                auth_header = {"X-Auth-Token": authenticate.token}
                continue
            else:
                msg = ('Http status code {} for file \"[]\". Retrying...'.format(filename, r.status_code))
                if i is not 4:
                    continue
                else:
                    msg = (
                        'File \"{}\" failed with HTTP code'
                        ' \"{}\". \nThis was the 5th and final try.'
                        ' File will be placed on the retry list.'.format(filename, r.status_code))
                    logger.error(msg)
                    retry_list.append(file_obj)

        except requests.exceptions.RequestException:
            if i == 4:
                msg = ('Error! Could not do API call to \"{}\"'
                       ' This was the 5th and final try.'
                       ' File will be placed on the retry list.'.format(filename))
                logger.error(msg)
                retry_list.append(file_obj)
                return
            else:
                msg = ('Error! Could not do API call to upload \"{}\". '
                       'This was the #{} attempt. Trying again...'.format(filename, i + 1))
                logger.error(msg)
            time.sleep(1)