PR #98: Duplicate File Finder

Add Duplicate File Finder mini-project
Merge pull request #98 from zain-cs/duplicate-file-finder

Author: iamwatchdogs

Duplicate-File-Finder/README.md

@@ -0,0 +1,56 @@
+# Duplicate File Finder
+
+A command-line tool that scans a directory (recursively) and finds files
+that are byte-for-byte identical, based on content hashing rather than
+just filenames.
+
+## How it works
+
+1. Files are first grouped by size — files of different sizes can never
+   be duplicates, so this is a cheap way to skip most comparisons.
+2. Remaining candidates are hashed using SHA-256 (read in chunks, so
+   large files don't get loaded into memory all at once).
+3. Files that share a hash are reported as duplicates.
+
+## Usage
+
+```bash
+python duplicate_finder.py <directory>
+```
+
+Example:
+
+```bash
+python duplicate_finder.py ~/Downloads
+```
+
+Output:
+
+```
+Duplicate group (2 files, hash 3a7bd3e2ff...):
+  /home/user/Downloads/report.pdf
+  /home/user/Downloads/report (1).pdf
+
+Total duplicate groups: 1
+Space that could be reclaimed: 245.3 KB
+```
+
+### Optional: delete duplicates
+
+```bash
+python duplicate_finder.py <directory> --delete
+```
+
+This keeps the first file found in each duplicate group and asks for
+confirmation before deleting the rest.
+
+## Running tests
+
+```bash
+pip install pytest
+pytest test_duplicate_finder.py -v
+```
+
+## Requirements
+
+None — uses only the Python standard library.

Duplicate-File-Finder/duplicate_finder.py

@@ -0,0 +1,117 @@
+"""Duplicate File Finder.
+
+Scans a directory tree and reports groups of files that are byte-for-byte
+identical, based on content hashing rather than filename or size alone.
+
+Usage:
+    python duplicate_finder.py <directory> [--delete]
+
+Example:
+    python duplicate_finder.py ~/Downloads
+    python duplicate_finder.py ~/Downloads --delete
+"""
+
+import argparse
+import hashlib
+import os
+from collections import defaultdict
+
+
+def hash_file(file_path: str, chunk_size: int = 8192) -> str:
+    """Return the SHA-256 hash of a file's contents.
+
+    NOTE: files are read in chunks so large files don't get loaded into
+    memory all at once.
+    """
+    hasher = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(chunk_size), b''):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def find_duplicates(root_dir: str) -> dict[str, list[str]]:
+    """Walk root_dir and group files by content hash.
+
+    Files are first grouped by size as a cheap pre-filter before hashing,
+    since files of different sizes can never be duplicates.
+    """
+    size_groups: dict[int, list[str]] = defaultdict(list)
+
+    for dirpath, _, filenames in os.walk(root_dir):
+        for filename in filenames:
+            full_path = os.path.join(dirpath, filename)
+            try:
+                file_size = os.path.getsize(full_path)
+            except OSError:
+                continue
+            size_groups[file_size].append(full_path)
+
+    hash_groups: dict[str, list[str]] = defaultdict(list)
+    for candidates in size_groups.values():
+        if len(candidates) < 2:
+            continue
+        for file_path in candidates:
+            try:
+                file_hash = hash_file(file_path)
+            except OSError:
+                continue
+            hash_groups[file_hash].append(file_path)
+
+    return {h: paths for h, paths in hash_groups.items() if len(paths) > 1}
+
+
+def print_report(duplicates: dict[str, list[str]]) -> None:
+    """Print a human-readable summary of duplicate groups."""
+    if not duplicates:
+        print('No duplicate files found.')
+        return
+
+    total_wasted_bytes = 0
+    for file_hash, paths in duplicates.items():
+        wasted = os.path.getsize(paths[0]) * (len(paths) - 1)
+        total_wasted_bytes += wasted
+
+        print(f'\nDuplicate group ({len(paths)} files, hash {file_hash[:10]}...):')
+        for path in paths:
+            print(f'  {path}')
+
+    print(f'\nTotal duplicate groups: {len(duplicates)}')
+    print(f'Space that could be reclaimed: {total_wasted_bytes / 1024:.1f} KB')
+
+
+def delete_duplicates(duplicates: dict[str, list[str]]) -> None:
+    """Delete all but the first file in each duplicate group."""
+    for paths in duplicates.values():
+        for path in paths[1:]:
+            os.remove(path)
+            print(f'Deleted: {path}')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description='Find duplicate files by content.')
+    parser.add_argument('directory', help='Directory to scan for duplicates')
+    parser.add_argument(
+        '--delete',
+        action='store_true',
+        help='Delete duplicates, keeping only the first file found in each group',
+    )
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.directory):
+        print(f'Error: {args.directory} is not a valid directory')
+        return
+
+    duplicates = find_duplicates(args.directory)
+    print_report(duplicates)
+
+    if args.delete and duplicates:
+        confirm = input('\nDelete duplicate files listed above? [y/N]: ')
+        if confirm.lower() == 'y':
+            delete_duplicates(duplicates)
+        else:
+            print('Skipped deletion.')
+
+
+if __name__ == '__main__':
+    main()

Duplicate-File-Finder/test_duplicate_finder.py

@@ -0,0 +1,74 @@
+"""Tests for duplicate_finder.py"""
+
+import os
+import tempfile
+
+from duplicate_finder import find_duplicates, hash_file
+
+
+def test_hash_file_is_consistent():
+    with tempfile.NamedTemporaryFile(delete=False, mode='w') as f:
+        f.write('hello world')
+        path = f.name
+
+    try:
+        assert hash_file(path) == hash_file(path)
+    finally:
+        os.remove(path)
+
+
+def test_find_duplicates_detects_identical_content():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path_a = os.path.join(tmp_dir, 'a.txt')
+        path_b = os.path.join(tmp_dir, 'b.txt')
+        path_c = os.path.join(tmp_dir, 'c.txt')
+
+        with open(path_a, 'w') as f:
+            f.write('same content')
+        with open(path_b, 'w') as f:
+            f.write('same content')
+        with open(path_c, 'w') as f:
+            f.write('different content')
+
+        duplicates = find_duplicates(tmp_dir)
+
+        assert len(duplicates) == 1
+        (group,) = duplicates.values()
+        assert set(group) == {path_a, path_b}
+
+
+def test_find_duplicates_ignores_unique_files():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        with open(os.path.join(tmp_dir, 'a.txt'), 'w') as f:
+            f.write('content one')
+        with open(os.path.join(tmp_dir, 'b.txt'), 'w') as f:
+            f.write('content two')
+
+        duplicates = find_duplicates(tmp_dir)
+
+        assert duplicates == {}
+
+
+def test_find_duplicates_handles_nested_directories():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        nested_dir = os.path.join(tmp_dir, 'nested')
+        os.makedirs(nested_dir)
+
+        path_a = os.path.join(tmp_dir, 'a.txt')
+        path_b = os.path.join(nested_dir, 'b.txt')
+
+        with open(path_a, 'w') as f:
+            f.write('shared content')
+        with open(path_b, 'w') as f:
+            f.write('shared content')
+
+        duplicates = find_duplicates(tmp_dir)
+
+        assert len(duplicates) == 1
+        (group,) = duplicates.values()
+        assert set(group) == {path_a, path_b}
+
+
+def test_find_duplicates_empty_directory():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        assert find_duplicates(tmp_dir) == {}