如何将TQM与python多处理集成

我正在创建一个新的python类，在这个类中，我试图集成多处理和TQM来说明进展。我走这条路是因为我打开了非常大的>1GB的时间序列数据文件，加载到pandas中，进行分组，然后以拼花格式保存它们。处理和保存每个数据文件可能需要几分钟。多处理大大加快了处理速度。但是，我目前对该流程没有可见性，我正在尝试集成TQM

下面的代码演示了一个简单的示例。在这段代码中，TQM只显示了将进程分配给池所需的时间，而不根据实际进程进行更新

“蟒蛇

import time
import multiprocessing
from tqdm import tqdm


class test_multiprocessing(object):

    def __init__(self, *args, **kwargs):
        self.list_of_results=[]
        self.items = [0,1,2,3,4,5,6,7,8,9,10]


    def run_test(self):
        print(f'Startng test')

        for i in range(1,5,1):
            print(f'working on var1: {i}')

            p = multiprocessing.Pool()

            for j in tqdm(self.items, desc='Items', unit='items', disable=False):
                variable3=3.14159
                p.apply_async(self.worker, [i, j,variable3], callback=self.update)

            p.close()
            p.join()
            print(f'completed i = {i}')
            print(f'')

    def worker(self, var1, var2, var3):
        result=var1*var2*var3
        time.sleep(2)
        return result

    def update(self, result_to_save):
        self.list_of_results.append(result_to_save)

if __name__ == '__main__':
    test1=test_multiprocessing()
    test1.run_test()

"

---

在本例中，进度条将显示工作几乎立即完成，但实际上只需几秒钟

我发现使用concurrent.futures vs multiprocessing可以很好地解决此问题。Dan Shiebler在这方面写了一篇很好的博客，并有一个很好的例子

下面显示了该策略的植入，它解决了我前面提出的问题

import time

from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed

class test_multiprocessing(object):

    def __init__(self, *args, **kwargs):
        self.list_of_results=[]
        self.items = [0,1,2,3,4,5,6,7,8,9,10]


    def run_test(self):
        print(f'Startng test')

        for i in range(1,5,1):
            print(f'working on var1: {i}')

            variable_list=[]

            for j in self.items:
                variable3=3.14159
                variables = [i,j,variable3]
                variable_list.append(variables)

            with ThreadPoolExecutor(max_workers=1000) as pool:   # with ProcessPoolExecutor(max_workers=n_jobs) as pool:    
                futures = [pool.submit(self.worker, a) for a in variable_list]
                kwargs = {
                'total': len(futures),
                'unit': 'it',
                'unit_scale': True,
                'leave': True
                }

                #Print out the progress as tasks complete
                for f in tqdm(as_completed(futures), **kwargs):
                    pass

            out = []
            #Get the results from the futures. 
            for i, future in tqdm(enumerate(futures)):
                try:
                    self.update(future.result())
                except Exception as e:
                    print(f'We have an error: {e}')


    def worker(self, variables):
        result=variables[0]*variables[1]*variables[2]
        time.sleep(2)
        return result


    def update(self, result_to_save):
        self.list_of_results.append(result_to_save)

if __name__ == '__main__':
    test1=test_multiprocessing()
    test1.run_test()