整个框架主要分为两个部分
    一个是分配任务模块
    另一个是处理算法模块

    分配任务模块读取文件，然后按照行数和CPU的数量进行切分
    然后自动的生成param_dict，然后把param_dict里的行数转化为对应的文件块
    最后传给进程池，调用处理算法解决。

    速度会快很多

    1. def get_taged_tokenized(name,json_data):
    2.     possessed_data = []
    3.     for idx in range(len(json_data)):
    4.         pass
    6. def multi_process_tag(name):
    7.     num_cores = int(mp.cpu_count())
    8.     print("本地计算机有: " + str(num_cores) + " 核心")
    9.     pool = mp.Pool(num_cores)
    11.     param_dict = {}
    13.     with open('./data/'+name+'.json','r',encoding='utf8')as fp:
    14.         json_data = json.load(fp)
    16.     start = 0
    17.     end =  len(json_data)
    18.     step = int((end - start)/num_cores)
    19.     print("per Task Step: ",step)
    20.     for i in range(num_cores):
    21.         param_dict['task{}'.format(i)]= json_data[start:start+step]
    22.         start = start+step
    23.     param_dict['task{}'.format(num_cores)]= json_data[start:]
    24.     start_t = datetime.datetime.now()
    25.     results = [pool.apply_async(get_taged_tokenized, args=(name, param)) for name, param in param_dict.items()]
    26.     results = [p.get() for p in results]
    27.     total = 0
    28.     my_result = []
    29.     for i in results:
    30.         for j in i:
    31.             #print(j,len(i[j]))
    32.             total += len(i[j])
    33.             my_result.extend(i[j])
    36.     print(total,len(my_result))
    37.     end_t = datetime.datetime.now()
    38.     elapsed_sec = (end_t - start_t).total_seconds()
    39.     print("多进程计算 共消耗: " + "{:.2f}".format(elapsed_sec) + " 秒")
    40.     f = open('./data/'+name+'_tokenized_taged.json','w',encoding='utf-8')
    41.     json.dump(my_result,f,ensure_ascii=False)
    42.     f.close()