python多进程内嵌套多线程

import time
from multiprocessing import Pool
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed


def thread_func(data, pool_count, thread_count):
    # 线程函数
    time.sleep(3)
    print(f"{data}, {pool_count}, {thread_count}'
'")
    return (data, data)


def pool_func(data_list, thread_num, pool_count):
    # 进程函数
    executor = ThreadPoolExecutor(max_workers=thread_num)  # 定义线程池,设置最大线程数量
    thread_list = []  # 储存线程
    thread_count = 0
    for dt in data_list:
        thrd = executor.submit(thread_func, dt, pool_count, thread_count)  # 将线程添加到线程池
        thread_list.append(thrd)  # 将当前线程存入列表,用于后面线程完成后,获取线程返回的结果
        thread_count += 1

    result_list = []  # 用于接收线程返回的结果
    # 因为线程开启后,默认就不管了,如果需要获取线程返回的结果,需要等待线程运行完成
    for task in as_completed(thread_list):  # 等待线程全部完成
        result_list.append(task.result())  # 获取线程的结果


if __name__ == "__main__":
    thread_num = 10  # 线程最大数量
    pool_num = 4  # 进程数量

    pool = Pool(pool_num)  # 定义进程池
    batch_num = 10  # 一个进程处理的数据量
    pool_count = 0
    for batch_i in range(0, 40, batch_num):
        data_list = range(batch_i, batch_i+batch_num)
        pool.apply_async(pool_func, (data_list, thread_num, pool_count,))
        pool_count += 1
    pool.close()
    pool.join()
原文地址:https://www.cnblogs.com/jaysonteng/p/12877114.html