文件内容重复字段查找

　　文件重复字段查找和删除，肉眼查看重复字段几行还好，几百几千行看起来就难受了，看得眼睛痛，就写个脚本来搞定吧，写就是python这么快乐，写python脚本一时爽，一直写一直爽！

from time import clock as now

all_md5=[]
total_file=0 
total_delete=0    
path = input("path: ")
start=now()
f = open(path,'rb')
i = 0
for line in f:
    i = i+1
    m = line
    if m in all_md5:
        print("重复字段：",m.decode("UTF-8"))
    else:
        all_md5.append(m)
end = now()
time_last = end - start
print('耗时：',time_last,'秒')

from time import clock as now
all_md5=[]total_file=0 total_delete=0    path = input("path: ")start=now()f = open(path,'rb')i = 0for line in f:    i = i+1    m = line    if m in all_md5:        print("重复字段：",m.decode("UTF-8"))    else:        all_md5.append(m)end = now()time_last = end - startprint('耗时：',time_last,'秒')