CSV Dedup CLI Skill

When to use

When you need to deduplicate CSV rows by business keys and emit a kept/removed summary.

Instructions

1. Save the script below as csv_dedup.py.
2. Run: python csv_dedup.py input.csv -o clean.csv -k email,user_id --summary report.txt
3. Deliver clean.csv plus the summary counts.

Script

`python #!/usr/bin/env python3 """CSV deduplication CLI — configurable key columns, summary report.""" from future import annotations

import argparse import csv import sys from pathlib import Path

def dedup(rows: list[dict], keys: list[str]) -> tuple[list[dict], int]: seen: set[tuple] = set() out: list[dict] = [] removed = 0 for row in rows: sig = tuple(row.get(k, "") for k in keys) if sig in seen: removed += 1 continue seen.add(sig) out.append(row) return out, removed

def main() -> int: p = argparse.ArgumentParser( description="Deduplicate CSV rows by configurable column keys and write a summary." ) p.add_argument("input", type=Path, help="Input CSV path") p.add_argument("-o", "--output", type=Path, required=True, help="Output CSV path") p.add_argument( "-k", "--keys", required=True, help="Comma-separated column names used as the dedup key", ) p.add_argument( "--summary", type=Path, default=None, help="Optional path for a text summary report (default: stdout)", ) args = p.parse_args() keys = [k.strip() for k in args.keys.split(",") if k.strip()] if not keys: print("error: at least one key column required", file=sys.stderr) return 2

with args.input.open(newline="", encoding="utf-8") as f:
    reader = csv.DictReader(f)
    if not reader.fieldnames:
        print("error: empty or headerless CSV", file=sys.stderr)
        return 2
    missing = [k for k in keys if k not in reader.fieldnames]
    if missing:
        print(f"error: missing columns: {missing}", file=sys.stderr)
        return 2
    rows = list(reader)

kept, removed = dedup(rows, keys)
args.output.parent.mkdir(parents=True, exist_ok=True)
with args.output.open("w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=rows[0].keys() if rows else keys)
    writer.writeheader()
    writer.writerows(kept)

report = (
    f"input_rows={len(rows)}\n"
    f"output_rows={len(kept)}\n"
    f"removed_duplicates={removed}\n"
    f"key_columns={','.join(keys)}\n"
    f"output_file={args.output}\n"
)
if args.summary:
    args.summary.write_text(report, encoding="utf-8")
print(report, end="")
return 0

if name == "main": raise SystemExit(main())

`

Direct download

https://files.catbox.moe/awevyh.py

Model

Works on any coding model (haiku+).