Python线程-如何使用它一次运行多个任务？

我是Python新手，stackoverflow comunity为我提供了ruge帮助，以便将我的shellscript迁移到Python。但是我仍然在为如何实现线程化而挣扎，因为这个脚本运行在一个x结果上，运行起来会更快，例如，脚本返回120个要运行的服务器，我希望每次运行5个，并有一个队列

我希望在线程上运行的方法在下面的条件之后：（我用注释标记）

下面是extract_adapter.py文件内容：

import psycopg2
import urllib2
import base64
import sys
import re
import lxml.html as LH
import checkServer

def extractAdapter(env,family,iserver,login,password,prefix,proxyUser,proxyPass,proxyHost,service):

    print "Starting on \t"+iserver

    proxy_auth = "http://"+proxyUser+":"+proxyPass+"@"+proxyHost
    proxy_handler = urllib2.ProxyHandler({"http": proxy_auth})

    opener = urllib2.build_opener(proxy_handler)
    urllib2.install_opener(opener)
    request = urllib2.Request("http://"+iserver+"/invoke/listRegisteredAdapters")
    base64string = base64.encodestring('%s:%s' % (login, password)).replace('\n', '')
    request.add_header("Authorization", "Basic %s" % base64string)
    response = urllib2.urlopen(request)
    html = response.read()

    doc = LH.fromstring(html)
    tds = (td.text_content() for td in doc.xpath("//td[not(*)]"))

    for adapterType, adapterDescription in zip(*[tds]*2):

        proxy_auth = "http://"+proxyUser+":"+proxyPass+"@"+proxyHost
        proxy_handler = urllib2.ProxyHandler({"http": proxy_auth})
        opener = urllib2.build_opener(proxy_handler)
        opener = urllib2.build_opener()
        urllib2.install_opener(opener)
        request = urllib2.Request("http://"+iserver+service+""+adapterType)
        base64string = base64.encodestring('%s:%s' % (login, password)).replace('\n', '')
        request.add_header("Authorization", "Basic %s" % base64string)
        response = urllib2.urlopen(request)
        html2 = response.read()

        doc = LH.fromstring(html2)
        tds = (td.text_content() for td in doc.xpath("//td[not(*)]"))

        for connectionAlias,packageName,connectionFactoryType,mcfDisplayName,connectionState,hasError in zip(*[tds]*6):

            cur.execute("INSERT INTO wip.info_adapter (env,family,iserver,prefix,package,adapter_type,connection_name,status) values (%s,%s,%s,%s,%s,%s,%s,%s)",
            (env,family,iserver,prefix,packageName,adapterType,connectionAlias,connectionState))
            con.commit()

################################################################################

def extract(env):
    global cur,con
    con = None
    try:

        con = psycopg2.connect(database='xx', user='xx',password='xxx',host='localhost')
        cur = con.cursor()
        qry=" random non important query"

        cur.execute(qry)
        data = cur.fetchall()

        for result in data:

            family   = result[0]
            prefix   = result[1]
            iserver  = result[2]
            version  = result[3]
            login    = result[4]
            password = result[5]
            service  = result[6]
            proxyHost = result[7]
            proxyUser = result[8]
            proxyPass = result[9]

            parts=iserver.split(":")
            host=parts[0]
            port=parts[1]

            if checkServer.checkit(host,port):
            ##SUPOSE TO AS START THREAD 

                if version == '7' or version == '8':

                    extractAdapter(env,family,iserver,login,password,prefix,proxyUser,proxyPass,proxyHost,service)

                elif version == '60' or version == '61':
                    print "Version 6.0 and 6.1 not supported yet"
            else:
                print iserver+"is offline"
            #TO END  THREAD

    except psycopg2.DatabaseError, e:
        print 'Error %s' % e
        sys.exit(1)

    finally:

        if con:
            con.close()

这就是我在runme.py上调用extract方法的方式

import extract_adapter_thread
from datetime import datetime

startTime = datetime.now()
print"------------------------------"
extract_adapter_thread.extract('TEST')
print"------------------------------"
print(datetime.now()-startTime)

---

顺便说一下，代码运行得很好。无错误。

如果一切都是线程安全的，您可以使用

线程模块：

import threading
starttime=datetime.now()
print "-"*10
code=threading.thread(target=extract_adapter_thread.extract,args=['TEST'])
code.daemon=True
code.start()
print "-"*10
print(datetime.now()-starttime)

线程将在Python中严重阻碍非IO绑定问题的解决，这是因为。因此，您最好这样做——这是一个Queue类附带的（参见下面使用mp队列的示例）

这应该可以让您同时处理多个单独的进程（比如从120个作业中一次批处理5个作业）。请注意，进程的开销高于线程的开销，因此对于小任务，您将为在线程上使用多处理而付出代价。不过，您的任务听起来足够大，足以保证这样的成本。
我真的不知道这是否会有多大帮助，但这是我在HD中的一段代码，而且。。。来了。了解并行或顺序ping某些IP的区别是一件基本的事情（不过需要linux）。这是非常简单的，不是一个直接的答案，你的具体问题，但。。。既然你说你是Python新手，它可能会给你一些想法

#!/usr/bin/env python

import datetime
import subprocess
import threading

ipsToPing = [
    "google.com",
    "stackoverflow.com",
    "yahoo.com",
    "terra.es", 
]

def nonThreadedPinger():
    start = datetime.datetime.now()
    for ipToPing in ipsToPing:
        print "Not-threaded ping to %s" % ipToPing
        subprocess.call(["/bin/ping", "-c", "3", "-W", "1.0", ipToPing], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    end = datetime.datetime.now()
    print ("Non threaded ping of %s ips took: %s." % (len(ipsToPing), end-start))

def _threadedPingerAux(ipToPing):
    print "Threaded ping to %s" % ipToPing
    subprocess.call(["/bin/ping", "-c", "3", "-W", "1.0", ipToPing], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def threadedPinger():
    retval = dict.fromkeys(ipsToPing, -1)
    threads = list()
    start = datetime.datetime.now()
    for ipToPing in ipsToPing:
        thread = threading.Thread(target=_threadedPingerAux, args=[ipToPing])
        thread.start()
        threads.append(thread)
    for thread in threads:
        thread.join()
    end = datetime.datetime.now()
    print ("Treaded ping of %s ips took: %s" % (len(ipsToPing), end-start))


if __name__ == "__main__":
    threadedPinger()
    nonThreadedPinger()

checkServer.checkit

是快速还是慢速操作，它是重新进入的吗？另外，您可以同时从多个线程调用

cur.execute

，这样可以吗？嗨，它很快。大约。每台服务器1秒。是的，我会为每个线程调用cur.exe。我会询问同时从多个线程调用

execute

是否安全。游标对象当前将由所有线程共享，我不知道psycopg2如何处理它。不，为此我将为每个线程创建一个单独的游标。@weefwwqg3现在是2017年，我真的为我写的东西感到羞愧。真的很惭愧。脚本将在windows环境下运行。仅感谢sugestion=）是否可以将解决方案应用于呼叫提取适配器（env、family、iserver、login、密码、前缀、proxyUser、proxyPass、proxyHost、service）？由于此方法适用于每个服务器Yep，因此我建议使用多处理，下面是一个简单明了的答案：

#!/usr/bin/env python

import datetime
import subprocess
import threading

ipsToPing = [
    "google.com",
    "stackoverflow.com",
    "yahoo.com",
    "terra.es", 
]

def nonThreadedPinger():
    start = datetime.datetime.now()
    for ipToPing in ipsToPing:
        print "Not-threaded ping to %s" % ipToPing
        subprocess.call(["/bin/ping", "-c", "3", "-W", "1.0", ipToPing], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    end = datetime.datetime.now()
    print ("Non threaded ping of %s ips took: %s." % (len(ipsToPing), end-start))

def _threadedPingerAux(ipToPing):
    print "Threaded ping to %s" % ipToPing
    subprocess.call(["/bin/ping", "-c", "3", "-W", "1.0", ipToPing], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def threadedPinger():
    retval = dict.fromkeys(ipsToPing, -1)
    threads = list()
    start = datetime.datetime.now()
    for ipToPing in ipsToPing:
        thread = threading.Thread(target=_threadedPingerAux, args=[ipToPing])
        thread.start()
        threads.append(thread)
    for thread in threads:
        thread.join()
    end = datetime.datetime.now()
    print ("Treaded ping of %s ips took: %s" % (len(ipsToPing), end-start))


if __name__ == "__main__":
    threadedPinger()
    nonThreadedPinger()