Ever bulk download images for a dataset and discover you have tons of duplicates? This simple Python script will use a very effective method of removing duplicate images in any given folder.
Here's how it's done:
Download script file I provided,
Right-Click and open with your favorite text editor. Modify line 6 to the destination of your images and Save.
Simply double-click the file and it will automatically detect and remove any duplicate images. An added bonus is it will always prioritize PNG files over all else(i.e. if you have a duplicate image in both JPG and PNG format it will delete the JPG).
A few notes:
This script actually takes the pixel information of the images into account, it will actually scan the images and determine duplicates.
For my paranoid friends out there, I will provide the script below, ask ChatGPT to check it so you have nothing to worry about!
import os
from PIL import Image
import imagehash
# Folder path to check for duplicates
folder_path = r"C:\Path\To\Your\Images"
# Dictionary to store image hashes
hashes = {}
# Function to compare and find duplicates, preferring PNG over JPG
def find_duplicates(folder):
for root, _, files in os.walk(folder):
for file in files:
file_path = os.path.join(root, file)
try:
# Open image and calculate its hash
with Image.open(file_path) as img:
img_hash = imagehash.average_hash(img)
# Check if the hash already exists
if img_hash in hashes:
existing_file_path = hashes[img_hash]
# Check file extensions to prefer PNG over JPG
if file_path.lower().endswith('.png') and not existing_file_path.lower().endswith('.png'):
# PNG is preferred, so remove the existing non-PNG duplicate
print(f"Removing non-PNG duplicate: {existing_file_path}")
os.remove(existing_file_path)
hashes[img_hash] = file_path # Update hash to point to the PNG version
elif not file_path.lower().endswith('.png') and existing_file_path.lower().endswith('.png'):
# Existing PNG should be kept, so remove the current duplicate
print(f"Removing non-PNG duplicate: {file_path}")
os.remove(file_path)
else:
# If both are PNG or neither, just remove the current file
print(f"Removing duplicate: {file_path}")
os.remove(file_path)
else:
hashes[img_hash] = file_path # Store the hash
except Exception as e:
print(f"Error processing {file_path}: {e}")
if name == "__main__":
find_duplicates(folder_path)