python引入线程池进行多线程操作

import sys
from glob import glob
from os import path
import random
from redis import StrictRedis
from BICDetection import biccheck 
from concurrent.futures import ProcessPoolExecutor  #  引入进程池 然后将进程处理过的结果存入大Redis中,这样可以在多台服务器上直接跑数据。

def get_direct_fps(fp):
    with open(fp) as fd:
        fps = [path.join(root_3421, i[:8], i.split()[0]) for i in fd]
        fps.sort()
    return fps

def get_fps(fp):
    with open(fp) as fd: 
        return [i.strip() for i in fd]

folder = 'PWD'
fps = glob(path.join(folder, '*.wav'))

redis_conn = StrictRedis(host="IP", db=0)

def bic_check_wrapper(args):
    fp, idx, label = args
    if not path.exists(fp):
        print(fp)
        return
    filename = path.basename(fp)
    if redis_conn.hget(label, filename) is not None:
        return
    r = biccheck([fp])
    flag = r['flag'].iloc[0]
    redis_conn.hset(label, filename, flag)
    return flag

def cu_flag_detect(suffix, nproc=30, select_num=None):
    passed_fps = get_direct_fps('cu.log')
    new_args = [(fp, idx, 'passed_{}'.format(suffix)) for idx, fp in enumerate(passed_fps)]
    unpassed_fps = get_fps('cu_71.log')
    new_args += [(fp, idx, 'unpassed_{}'.format(suffix)) for idx, fp in enumerate(unpassed_fps)]
    random.seed(1)
    random.shuffle(new_args)
    if select_num is None:
        selected = new_args
    with ProcessPoolExecutor(int(nproc)) as executor:
        executor.map(bic_check_wrapper, selected)

关注公众号 海量干货等你
原文地址:https://www.cnblogs.com/sowhat1412/p/12734312.html