削除する前にサイズと SHA-256 で重複ファイルを探す

from pathlib import Path

DEMO = Path("outputs") / "duplicate_demo"
FILES = {
    "archive/a_saved.txt": b"abc",
    "empty_a.txt": b"",
    "empty_b.txt": b"",
    "notes/a.txt": b"abc",
    "notes/a_copy.txt": b"abc",
    "notes/other.txt": b"xyz",
    "unique.txt": b"solo",
}

DEMO.parent.mkdir(parents=True, exist_ok=True)
DEMO.mkdir()  # Refuse to reuse an existing destination.
for relative_path, content in FILES.items():
    target = DEMO / relative_path
    target.parent.mkdir(parents=True, exist_ok=True)
    with target.open("xb") as stream:
        stream.write(content)

python create_duplicate_demo.py

python find_duplicate_files.py

import csv
import hashlib
import os
import stat
from pathlib import Path

SOURCE = Path("outputs") / "duplicate_demo"
OUTPUT_DIR = Path("outputs") / "duplicate_review"
CHUNK_BYTES = 1024 * 1024


def signature(info: os.stat_result) -> tuple[int, ...]:
    return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)


def raise_walk_error(error: OSError) -> None:
    raise error


def file_digest(path: Path, expected: os.stat_result) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        before = os.fstat(stream.fileno())
        if not stat.S_ISREG(before.st_mode) or signature(before) != signature(expected):
            raise RuntimeError(f"File changed before hashing: {path}")
        while chunk := stream.read(CHUNK_BYTES):
            digest.update(chunk)
        after = os.fstat(stream.fileno())
    if signature(after) != signature(expected) or signature(path.lstat()) != signature(expected):
        raise RuntimeError(f"File changed during hashing: {path}")
    return digest.hexdigest()


def main() -> None:
    root = SOURCE.resolve(strict=True)
    if not root.is_dir():
        raise ValueError("SOURCE must be a directory.")
    if OUTPUT_DIR.resolve().is_relative_to(root):
        raise ValueError("The report folder must be outside SOURCE.")

    OUTPUT_DIR.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT_DIR.mkdir()  # Stop if the destination already exists.

    by_size = {}
    scanned = 0
    for directory, directories, filenames in os.walk(
        root, followlinks=False, onerror=raise_walk_error
    ):
        base = Path(directory)
        directories[:] = sorted(
            name for name in directories if not (base / name).is_symlink()
        )
        for name in sorted(filenames):
            path = base / name
            info = path.lstat()
            if not stat.S_ISREG(info.st_mode):
                continue
            by_size.setdefault(info.st_size, []).append((path, info))
            scanned += 1

    groups = []
    hashed = 0
    for size, members in sorted(by_size.items()):
        if len(members) < 2:
            continue
        by_hash = {}
        for path, initial in members:
            digest = file_digest(path, initial)
            by_hash.setdefault(digest, []).append(path)
            hashed += 1
        for digest, paths in sorted(by_hash.items()):
            if len(paths) > 1:
                ordered = sorted(paths, key=lambda path: path.as_posix())
                groups.append((size, digest, ordered))

    report = OUTPUT_DIR / "duplicates.csv"
    with report.open("x", encoding="utf-8", newline="") as stream:
        writer = csv.writer(stream)
        writer.writerow([
            "group_id", "size_bytes", "sha256", "relative_path", "review_status"
        ])
        for number, (size, digest, paths) in enumerate(groups, start=1):
            for path in paths:
                writer.writerow([
                    f"G{number:03d}", size, digest,
                    path.relative_to(root).as_posix(), "UNREVIEWED"
                ])

    matched = sum(len(paths) for _, _, paths in groups)
    print(f"Scanned {scanned} regular file paths; hashed {hashed}.")
    print(f"Duplicate candidates: {len(groups)} groups, {matched} paths.")
    print(f"Review CSV: {report.as_posix()}")


if __name__ == "__main__":
    main()

Scanned 7 regular file paths; hashed 6.
Duplicate candidates: 2 groups, 5 paths.
Review CSV: outputs/duplicate_review/duplicates.csv