Python多处理问题池.map（）

这是我的系统的一部分的示意图。

我有一个返回数据的存储过程。我需要用多个输入参数调用这个存储过程。所以我编写了python多处理脚本。但它似乎有一些问题

代码：

import cx_Oracle
import dataextract as tde
import os, sys
import time
from multiprocessing import Pool
import multiprocessing, logging
from multiprocessing.pool import ApplyResult





def __enter__():
    __db = cx_Oracle.Connection("TEST/test123@test.net:12345/test.world")
    __cursor = __db.cursor()
    return __cursor, __db

def __exit__(__cursor, __db):
    __cursor.close()
    __db.close()

def get_context_id(__cursor, context_list):
    context_id = __cursor.arrayvar(cx_Oracle.NUMBER, context_list)
    return context_id


def find_DAC_detail(input):
    cntxt_id = input [0]
    min_row_id = input[1] 
    max_row_id = input[2] 
    print "[",time.asctime(), "]", cntxt_id, min_row_id, max_row_id
    print "[",time.asctime(), "] Calculating DAC Detail...", multiprocessing.current_process().name()
    __cursor, __db = __enter__()
    query_contexts = get_context_id(__cursor, cntxt_id)

    # as it comes to all complex types we need to tell Oracle Client
    # what type to expect from an OUT parameter
    l_cur = __cursor.var(cx_Oracle.CURSOR)
    list = []
    execute_proc = "BEGIN PKG_TABLEAU_FEED.ADD_CONTEXT(:query_contexts, :min_row_id, :max_row_id, :out); END;"
    print "[",time.asctime(), "] Ready to call the stored procedure: PKG_TABLEAU_FEED.ADD_CONTEXT"
    __cursor.execute(execute_proc, (query_contexts, min_row_id, max_row_id, l_cur)) 

    for row in l_cur.getvalue():
    #          print "[",time.asctime(), "]", row
        list.append(row)

    return list



def create_extract_file_detail():
    #Step 1: Create the Extract File and open the .csv
    try:
        tdefile = tde.Extract('test_extract.tde')
    except:
        os.remove('test_extract.tde')            
        tdefile = tde.Extract('test_extract.tde')

    return tdefile


def create_table_definition():

    #Step 2: Create the tableDef
    tableDef = tde.TableDefinition()
    # DAC Data into the tde table
    tableDef.addColumn('COB_DATE', tde.Type.DATE)
    tableDef.addColumn('EXCEPTION_ID', tde.Type.INTEGER)

    return tableDef       



def generate_table(tdefile, tableDef):
    #Step 2: open the tableDef if the table exists     
    if tdefile.hasTable("Extract"):
        table = tdefile.openTable("Extract")
    else:
        # or add new table definition if the table doesn't exist
        table = tdefile.addTable('Extract',tableDef)
    return table


def generate_file(tableDef, table, detail_rowset):           

    newrow = tde.Row(tableDef)

    for line in detail_rowset:
        #Starts Here...                  
        if line[0] != None and line[0]!= '':       
            date =  line[0]
            newrow.setDate(0, date.year, date.month, date.day)

        if line[1] != None:
            newrow.setInteger(1,int(line[1]))

        table.insert(newrow)


if __name__ == '__main__':
    try:
        start = time.time()
        __cursor, __db = __enter__()

        cntxt_id = [253, 254]

        print "[",time.asctime(), "] Connection established successfully..."      



        # start 4 worker processes
        print "[",time.asctime(), "] cpu_count =" , multiprocessing.cpu_count()
        logger = multiprocessing.log_to_stderr()
        logger.setLevel(logging.DEBUG)


        #
        # Tests
        #
        PROCESSES=4
        pool = Pool(PROCESSES)
        TASKS = [(cntxt_id, i*1000 + 1, i*1000 + 1000) for i in range(4)]
        print TASKS
        detail_rowset = pool.map(find_DAC_detail, TASKS)


#        for data in detail_rowset:
#            print data
        print "[",time.asctime(), "] Data Fetched in: ", time.time() - start

        tdefile = create_extract_file_detail()
        print "[",time.asctime(), "] TDE File Validated..."
        tableDef = create_table_definition()
        table = generate_table(tdefile, tableDef)
        print "[",time.asctime(), "] Extract Table Created..."
        generate_file(tableDef, table, detail_rowset)
        print "[",time.asctime(), "] Extract File Generated..."
        print "[",time.asctime(), "] Total Time taken: ",  time.time() - start
#    except:
#        print "[",time.asctime(), "] ERROR:", sys.exc_info()[0]
    finally:                
        __exit__(__cursor, __db)
        print "[",time.asctime(), "] Exiting program..."

[Traceback (most recent call last):
  File "C:\DQWorkspace\First-Python\com\jpmorgan\cri\tableau\extract\New_DAC_Extract.py", line 349, in <module>
 Wed May 07 17:21:59 2014 ] Exiting program...
    detail_rowset = pool.map(find_DAC_detail, TASKS)
  File "C:\Python27\lib\multiprocessing\pool.py", line 225, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 522, in get
    raise self._value
TypeError: 'str' object is not callable
[INFO/MainProcess] process shutting down

解释： 我尝试使用pool.apply\u async，但它有很多细微差别。我正在尝试pool.map（）调用find_DAC_detail函数（上面的代码），但也失败了，错误如下：

错误：

import cx_Oracle
import dataextract as tde
import os, sys
import time
from multiprocessing import Pool
import multiprocessing, logging
from multiprocessing.pool import ApplyResult





def __enter__():
    __db = cx_Oracle.Connection("TEST/test123@test.net:12345/test.world")
    __cursor = __db.cursor()
    return __cursor, __db

def __exit__(__cursor, __db):
    __cursor.close()
    __db.close()

def get_context_id(__cursor, context_list):
    context_id = __cursor.arrayvar(cx_Oracle.NUMBER, context_list)
    return context_id


def find_DAC_detail(input):
    cntxt_id = input [0]
    min_row_id = input[1] 
    max_row_id = input[2] 
    print "[",time.asctime(), "]", cntxt_id, min_row_id, max_row_id
    print "[",time.asctime(), "] Calculating DAC Detail...", multiprocessing.current_process().name()
    __cursor, __db = __enter__()
    query_contexts = get_context_id(__cursor, cntxt_id)

    # as it comes to all complex types we need to tell Oracle Client
    # what type to expect from an OUT parameter
    l_cur = __cursor.var(cx_Oracle.CURSOR)
    list = []
    execute_proc = "BEGIN PKG_TABLEAU_FEED.ADD_CONTEXT(:query_contexts, :min_row_id, :max_row_id, :out); END;"
    print "[",time.asctime(), "] Ready to call the stored procedure: PKG_TABLEAU_FEED.ADD_CONTEXT"
    __cursor.execute(execute_proc, (query_contexts, min_row_id, max_row_id, l_cur)) 

    for row in l_cur.getvalue():
    #          print "[",time.asctime(), "]", row
        list.append(row)

    return list



def create_extract_file_detail():
    #Step 1: Create the Extract File and open the .csv
    try:
        tdefile = tde.Extract('test_extract.tde')
    except:
        os.remove('test_extract.tde')            
        tdefile = tde.Extract('test_extract.tde')

    return tdefile


def create_table_definition():

    #Step 2: Create the tableDef
    tableDef = tde.TableDefinition()
    # DAC Data into the tde table
    tableDef.addColumn('COB_DATE', tde.Type.DATE)
    tableDef.addColumn('EXCEPTION_ID', tde.Type.INTEGER)

    return tableDef       



def generate_table(tdefile, tableDef):
    #Step 2: open the tableDef if the table exists     
    if tdefile.hasTable("Extract"):
        table = tdefile.openTable("Extract")
    else:
        # or add new table definition if the table doesn't exist
        table = tdefile.addTable('Extract',tableDef)
    return table


def generate_file(tableDef, table, detail_rowset):           

    newrow = tde.Row(tableDef)

    for line in detail_rowset:
        #Starts Here...                  
        if line[0] != None and line[0]!= '':       
            date =  line[0]
            newrow.setDate(0, date.year, date.month, date.day)

        if line[1] != None:
            newrow.setInteger(1,int(line[1]))

        table.insert(newrow)


if __name__ == '__main__':
    try:
        start = time.time()
        __cursor, __db = __enter__()

        cntxt_id = [253, 254]

        print "[",time.asctime(), "] Connection established successfully..."      



        # start 4 worker processes
        print "[",time.asctime(), "] cpu_count =" , multiprocessing.cpu_count()
        logger = multiprocessing.log_to_stderr()
        logger.setLevel(logging.DEBUG)


        #
        # Tests
        #
        PROCESSES=4
        pool = Pool(PROCESSES)
        TASKS = [(cntxt_id, i*1000 + 1, i*1000 + 1000) for i in range(4)]
        print TASKS
        detail_rowset = pool.map(find_DAC_detail, TASKS)


#        for data in detail_rowset:
#            print data
        print "[",time.asctime(), "] Data Fetched in: ", time.time() - start

        tdefile = create_extract_file_detail()
        print "[",time.asctime(), "] TDE File Validated..."
        tableDef = create_table_definition()
        table = generate_table(tdefile, tableDef)
        print "[",time.asctime(), "] Extract Table Created..."
        generate_file(tableDef, table, detail_rowset)
        print "[",time.asctime(), "] Extract File Generated..."
        print "[",time.asctime(), "] Total Time taken: ",  time.time() - start
#    except:
#        print "[",time.asctime(), "] ERROR:", sys.exc_info()[0]
    finally:                
        __exit__(__cursor, __db)
        print "[",time.asctime(), "] Exiting program..."

[Traceback (most recent call last):
  File "C:\DQWorkspace\First-Python\com\jpmorgan\cri\tableau\extract\New_DAC_Extract.py", line 349, in <module>
 Wed May 07 17:21:59 2014 ] Exiting program...
    detail_rowset = pool.map(find_DAC_detail, TASKS)
  File "C:\Python27\lib\multiprocessing\pool.py", line 225, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 522, in get
    raise self._value
TypeError: 'str' object is not callable
[INFO/MainProcess] process shutting down

[回溯（最近一次呼叫最后一次）：
文件“C:\DQWorkspace\First Python\com\jpmorgan\cri\tableau\extract\New\u DAC\u extract.py”，第349行，在
2014年5月7日星期三17:21:59]退出计划。。。
detail\u rowset=pool.map（查找\u DAC\u detail，任务）
文件“C:\Python27\lib\multiprocessing\pool.py”，第225行，在映射中
返回self.map\u async（func，iterable，chunksize）.get（）
get中第522行的文件“C:\Python27\lib\multiprocessing\pool.py”
提升自我价值
TypeError:“str”对象不可调用
[INFO/MainProcess]进程正在关闭

---

任何指针都会有帮助。

尝试从主进程调用该函数，而不是从

Pool.map

。（即，替换

pool.map（查找\u DAC\u详细信息，任务）

。这样，回溯应该会告诉您问题的确切位置。感谢@Bakuriu似乎已经解决了问题。多处理。当前\u进程（）.name（）是windows中的一个问题。@Vivek如果您已经解决了问题，请花些时间回答并接受您的答案。这将在将来帮助其他用户。：）当然。。。稍后我将分享更多关于这个问题的概念细节。