Python使用CRC实现图片去重

import zlib,os

def Find_Repeat_File(file_path,file_type):
    Catalogue = os.listdir(file_path)
    CatalogueDict = {}  # 查询字典,方便后期查询键值对对应参数
    for each in Catalogue:
        path = (file_path + each)
        if os.path.splitext(path)[1] == file_type:
            with open(path,"rb") as fp:
                crc32 = zlib.crc32(fp.read())
                # print("[*] 文件名: {} CRC32校验: {}".format(path,str(crc32)))
                CatalogueDict[each] = str(crc32)
    CatalogueList = []
    for value in CatalogueDict.values():
    # 该过程实现提取字典中的crc32特征组合成列表 CatalogueList
        CatalogueList.append(value)

    CountDict = {}
    for each in CatalogueList:
    # 该过程用于存储文件特征与特征重复次数,放入 CountDict
        CountDict[each] = CatalogueList.count(each)
        
    RepeatFileFeatures = []
    for key,value in CountDict.items():
    # 循环查找字典中的数据,如果value大于1就存入 RepeatFileFeatures
        if value > 1:
            print("[-] 文件特征: {} 重复次数: {}".format(key,value))
            RepeatFileFeatures.append(key)

if __name__ == "__main__":
    Find_Repeat_File("D://lyshark/",".png")

import zlib,os,argparse
from operator import itemgetter
from itertools import groupby

def Find_Repeat_File(file_path,file_type):
    Catalogue = os.listdir(file_path)
    CatalogueList = []
    for each in Catalogue:
        path = (file_path + each)
        if os.path.splitext(path)[1] == file_type:
            with open(path,"rb") as fp:
                crc32 = zlib.crc32(fp.read())
                # print("[*] 文件名: {} CRC32校验: {}".format(path,str(crc32)))
                CatalogueList.append({ "CRC32": str(crc32) , "FILE": path })
    
    # 首先排序,然后根据字典中的CRC32排序
    CatalogueList.sort(key=itemgetter("CRC32"))
    for key,value in groupby(CatalogueList,key=itemgetter("CRC32")):
        # print("[*] CRC32特征码: {}\t".format(key))
        for each in value:
            RepeatNumber = len(list(value))
            if (RepeatNumber+1) > 1:
                try:
                    print("---> 重复图片: {} 已移除".format(each.values()))
                    os.remove(str(list(each.values())[1]))
                except FileNotFoundError:
                    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-d","--dirs",dest="dirs",help="指定目录路径")
    parser.add_argument("-t","--types",dest="types",help="指定文件类型")
    args = parser.parse_args()
    # 使用方式: main.py -d "d://lyshark/" -t ".png"
    if args.dirs and args.types:
        os.chdir(args.dirs)
        try:
            for each in range(0,len(os.listdir())):
                Find_Repeat_File("./",args.types)
        except:
            pass
    else:
        parser.print_help()

import os, time
from zlib import crc32

# 计算目标CRC32
def Calculation_crc32(filename):
    try:
        with open(filename,"rb") as fp:
            crc = crc32(fp.read())
            while True:
                temp = fp.read(8196)
                if not temp:
                    break
            fp.close()
            return crc
    except Exception:
        fp.close()
        return 0
    return 0

# 开始监控目录
def MonitoringDirectory(path_to_watch):
    crc_poll = []

    before = dict([(f, None) for f in os.listdir(path_to_watch)])
    while True:
        time.sleep (0.3)

        after = dict ([(f, None) for f in os.listdir (path_to_watch)])
        added = [f for f in after if not f in before]
        removed = [f for f in before if not f in after]

        # 新增文件时触发
        if added:
            # print("新建文件: {}".format(added))
            for index in range(0,len(added)):
                # 计算文件CRC32
                AbsolutePath = path_to_watch + "/" + added[index]
                crc = Calculation_crc32(AbsolutePath)
                print(f"计算文件路径: {AbsolutePath} 文件CRC32: {crc}")

                # 只要有新文件则加入到crc_pool
                if crc != 0:
                    crc_poll.append(crc)

                print("池内数据: {}".format(crc_poll))
                crc_count = 0

                # 循环池内的所有CRC数据
                for index in range(0,len(crc_poll)):

                    # 如果当前文件CRC与池内某个一致则递增
                    if crc_poll[index] == crc:
                        crc_count = crc_count + 1

                    # 只要大于2则说明有重复的
                    if crc_count >= 2:
                        try:
                            print("存在校验值,删除文件: {}".format(AbsolutePath))
                            os.remove(AbsolutePath)
                            crc_poll.remove(crc)
                        except Exception:
                            pass
        # 删除文件时触发
        if removed:
            print("移除文件: {}".format(removed))
        before = after

if __name__ == "__main__":
    MonitoringDirectory("d://lyshark")