# coding:utf-8 # make_msg_dict.py line_num = 0 new_block_flag = False second_flag = False linux_resource_flag = False next_predict_flag = False second_line = "" threads_num = 0 process_num = 0 torch_threads_num = 0 img_num = 0 batch_size = 0 msg_line_num = 0 linux_resource = 0 all_count = 0 min_list = [] avg_list = [] max_list = [] fps_list = [] all_dict = {} msg_dict = {} other_count = 0 gpu = 0 gpu_memory = 0 cpu = 0 cpu_memory = 0 log_file = "rt_t2_process_12-10_19-00.log" with open(log_file, 'r')as f: for i in f: line_num += 1 # 回车行 if i == " ": new_block_flag = False continue try: # 每行内容 line_msg = i.split("INFO")[1].strip() except Exception as error: continue # INFO 空行 if not line_msg: continue # 第一行 if line_msg == "predict_start": second_flag = True next_predict_flag = False msg_dict = {} min_list = [] avg_list = [] max_list = [] fps_list = [] gpu = 0 gpu_memory = 0 cpu = 0 cpu_memory = 0 continue # 取第二行 if second_flag: second_line = line_msg second_ret_list = second_line.replace(" ", '').split(",") # print(second_ret_list) judge_type = second_ret_list[0].split(":")[0] if judge_type == "process_num": threads_num = -1 process_num = int(second_ret_list[0].split(":")[1]) msg_line_num = process_num * 5 else: process_num = -1 threads_num = int(second_ret_list[0].split(":")[1]) msg_line_num = threads_num * 5 torch_threads_num = int(second_ret_list[1].split(":")[1]) img_num = int(second_ret_list[2].split(":")[1]) batch_size = int(second_ret_list[3].split(":")[1]) second_flag = False continue # 取 predict_msg if msg_line_num > 0: msg_line_num -= 1 if msg_line_num == 0: linux_resource_flag = True linux_resource = 17 # predict time if ":" in line_msg: line_msg_list = line_msg.replace(" ", '').split(":") min_list.append(float(line_msg_list[1].split("/")[0])) avg_list.append(float(line_msg_list[1].split("/")[1])) max_list.append(float(line_msg_list[1].split("/")[2])) continue if "=" in line_msg: line_msg_list = line_msg.replace(" ", '').split("=") if line_msg_list[0][-3:] == "fps": fps_list.append(float(line_msg_list[1])) continue continue # 拿到linux数据 if linux_resource > 0: linux_resource -= 1 if linux_resource == 0: next_predict_flag = True linux_resource_flag = False # print(line_msg) line_msg.strip() if line_msg.startswith("cpu utilization"): if line_msg.count("/") == 1: cpu = str(int(float(line_msg.split(" ")[2][:-1]))) + "%" continue if line_msg.startswith("memory"): if line_msg.count("/") == 1: cpu_memory = str(round(float(line_msg.split(" ")[1][:-2]), 2) * 1000) + "MB" continue if line_msg.startswith("GPU 0 memory_used"): gpu_memory = str(round(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]), 2)) + "MB" continue if line_msg.startswith("GPU 0 Utilization_Rates"): gpu = str(int(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]))) + "%" continue # 取predict下的东西 if not linux_resource_flag and next_predict_flag: # print("计数") other_count += 1 msg_dict = { "Throughput": round(sum(fps_list), 0) * batch_size, "Latency": "{} / {} / {}".format( round(min(min_list) * 1000, 2), round(sum(avg_list) / len(avg_list) * 1000, 2), round(max(max_list) * 1000, 2), ), "gpu": gpu, "gpu_memory": gpu_memory, "cpu": cpu, "cpu_memory": cpu_memory, } # 组合dict if threads_num != -1: if all_dict.get(1): if all_dict[1].get(threads_num): if all_dict[1][threads_num].get(batch_size): all_dict[1][threads_num][batch_size][torch_threads_num] = msg_dict else: all_dict[1][threads_num][batch_size] = {torch_threads_num: msg_dict} else: all_dict[1][threads_num] = {batch_size: {torch_threads_num: msg_dict}} else: all_dict[1] = {threads_num: {batch_size: {torch_threads_num: msg_dict}}} else: if all_dict.get(process_num): if all_dict[process_num].get(1): if all_dict[process_num][1].get(batch_size): all_dict[process_num][1][batch_size][torch_threads_num] = msg_dict else: all_dict[process_num][1][batch_size] = {torch_threads_num: msg_dict} else: all_dict[process_num][1] = {batch_size: {torch_threads_num: msg_dict}} else: all_dict[process_num] = {1: {batch_size: {torch_threads_num: msg_dict}}} print("other_count", other_count) print("all_dict", all_dict) # make csv
# coding:utf-8 # make_csv.py line_num = 0 new_block_flag = False second_flag = False linux_resource_flag = False next_predict_flag = False second_line = "" threads_num = 0 process_num = 0 torch_threads_num = 0 img_num = 0 batch_size = 0 msg_line_num = 0 linux_resource = 0 all_count = 0 min_list = [] avg_list = [] max_list = [] fps_list = [] all_dict = {} msg_dict = {} other_count = 0 gpu = 0 gpu_memory = 0 cpu = 0 cpu_memory = 0 import os log_dir_list = os.listdir("log_dir") for log_file in log_dir_list: all_dict = {} other_count = 0 with open("log_dir/{}".format(log_file), 'r')as f: for i in f: line_num += 1 # 回车行 if i == " ": new_block_flag = False continue try: # 每行内容 line_msg = i.split("INFO")[1].strip() except Exception as error: continue # INFO 空行 if not line_msg: continue # 第一行 if line_msg == "predict_start": second_flag = True next_predict_flag = False msg_dict = {} min_list = [] avg_list = [] max_list = [] fps_list = [] gpu = 0 gpu_memory = 0 cpu = 0 cpu_memory = 0 continue # 取第二行 if second_flag: second_line = line_msg second_ret_list = second_line.replace(" ", '').split(",") # print(second_ret_list) judge_type = second_ret_list[0].split(":")[0] if judge_type == "process_num": threads_num = -1 process_num = int(second_ret_list[0].split(":")[1]) msg_line_num = process_num * 5 else: process_num = -1 threads_num = int(second_ret_list[0].split(":")[1]) msg_line_num = threads_num * 5 torch_threads_num = int(second_ret_list[1].split(":")[1]) img_num = int(second_ret_list[2].split(":")[1]) batch_size = int(second_ret_list[3].split(":")[1]) second_flag = False continue # 取 predict_msg if msg_line_num > 0: msg_line_num -= 1 if msg_line_num == 0: linux_resource_flag = True linux_resource = 17 # predict time if ":" in line_msg: line_msg_list = line_msg.replace(" ", '').split(":") min_list.append(float(line_msg_list[1].split("/")[0])) avg_list.append(float(line_msg_list[1].split("/")[1])) max_list.append(float(line_msg_list[1].split("/")[2])) continue if "=" in line_msg: line_msg_list = line_msg.replace(" ", '').split("=") if line_msg_list[0][-3:] == "fps": fps_list.append(float(line_msg_list[1])) continue continue # 拿到linux数据 if linux_resource > 0: linux_resource -= 1 if linux_resource == 0: next_predict_flag = True linux_resource_flag = False # print(line_msg) line_msg.strip() if line_msg.startswith("cpu utilization"): if line_msg.count("/") == 1: cpu = str(int(float(line_msg.split(" ")[2][:-1]))) + "%" continue if line_msg.startswith("memory"): if line_msg.count("/") == 1: cpu_memory = str(round(float(line_msg.split(" ")[1][:-2]), 2) * 1000) + "MB" continue if line_msg.startswith("GPU 0 memory_used"): gpu_memory = str(round(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]), 2)) + "MB" continue if line_msg.startswith("GPU 0 Utilization_Rates"): gpu = str(int(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]))) + "%" continue # 取predict下的东西 if not linux_resource_flag and next_predict_flag: # print("计数") other_count += 1 msg_dict = { "Throughput": round(sum(fps_list), 0) * batch_size, "Latency": "{} / {} / {}".format( round(min(min_list) * 1000, 2), round(sum(avg_list) / len(avg_list) * 1000, 2), round(max(max_list) * 1000, 2), ), "gpu": gpu, "gpu_memory": gpu_memory, "cpu": cpu, "cpu_memory": cpu_memory, } # 组合dict if threads_num != -1: if all_dict.get(1): if all_dict[1].get(threads_num): if all_dict[1][threads_num].get(batch_size): all_dict[1][threads_num][batch_size][torch_threads_num] = msg_dict else: all_dict[1][threads_num][batch_size] = {torch_threads_num: msg_dict} else: all_dict[1][threads_num] = {batch_size: {torch_threads_num: msg_dict}} else: all_dict[1] = {threads_num: {batch_size: {torch_threads_num: msg_dict}}} else: if all_dict.get(process_num): if all_dict[process_num].get(1): if all_dict[process_num][1].get(batch_size): all_dict[process_num][1][batch_size][torch_threads_num] = msg_dict else: all_dict[process_num][1][batch_size] = {torch_threads_num: msg_dict} else: all_dict[process_num][1] = {batch_size: {torch_threads_num: msg_dict}} else: all_dict[process_num] = {1: {batch_size: {torch_threads_num: msg_dict}}} print("other_count", other_count) # print("all_dict", all_dict) # make csv cvs_str = '' all_cvs_str = '' threads_count = 0 for process, threads_num_dict in all_dict.items(): cvs_str = str(process) + "," for threads_num, batch_size_dict in threads_num_dict.items(): th_before_cvs_str = cvs_str cvs_str += str(threads_num) + "," for batch_size, torch_threads_num_dict in batch_size_dict.items(): b_before_cvs_str = cvs_str cvs_str += str(batch_size) + "," for torch_threads_num, msg_dict in torch_threads_num_dict.items(): t_before_cvs_str = cvs_str cvs_str += str(torch_threads_num) + "," msg_str = ",".join([str(i) for i in list(msg_dict.values())]) cvs_str += str(msg_str) all_cvs_str += cvs_str + " " cvs_str = t_before_cvs_str cvs_str = b_before_cvs_str cvs_str = th_before_cvs_str print(all_cvs_str.count(" ")) with open("{}.csv".format(log_file), 'w') as f: f.write(all_cvs_str) all_cvs_str = ""