163 lines
4.7 KiB
Python
163 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
||
|
||
import csv
|
||
from decimal import Decimal, InvalidOperation
|
||
import sys
|
||
from pathlib import Path
|
||
import json
|
||
from datetime import datetime
|
||
|
||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||
STATE_FILE = SCRIPT_DIR / "halifax_state.json"
|
||
|
||
def load_state():
|
||
if STATE_FILE.exists():
|
||
try:
|
||
return json.loads(STATE_FILE.read_text())
|
||
except json.JSONDecodeError:
|
||
# If it ever gets corrupted, start fresh
|
||
return {"seen_keys": []}
|
||
return {"seen_keys": []}
|
||
|
||
|
||
def save_state(state):
|
||
STATE_FILE.write_text(json.dumps(state, indent=2))
|
||
|
||
|
||
def normalize_text(s: str) -> str:
|
||
# collapse whitespace + lowercase
|
||
return " ".join(s.split()).lower()
|
||
|
||
|
||
def make_key(date_str: str, payee: str, desc: str, amount: Decimal) -> str:
|
||
"""
|
||
Build a dedupe key: normalized date + payee + description + amount.
|
||
We normalize:
|
||
- date into ISO (YYYY-MM-DD)
|
||
- payee/desc into lowercased, collapsed whitespace
|
||
- amount to 2 decimal places
|
||
"""
|
||
|
||
# Halifax gives dd/mm/yyyy – convert to ISO for consistency
|
||
date_str = date_str.strip()
|
||
try:
|
||
# If it's already ISO, this will work too
|
||
try:
|
||
d = datetime.fromisoformat(date_str).date()
|
||
except ValueError:
|
||
d = datetime.strptime(date_str, "%d/%m/%Y").date()
|
||
norm_date = d.isoformat()
|
||
except Exception:
|
||
# If parsing fails for some reason, just use the raw string
|
||
norm_date = date_str
|
||
|
||
norm_payee = normalize_text(payee)
|
||
norm_desc = normalize_text(desc)
|
||
norm_amount = amount.quantize(Decimal("0.01"))
|
||
|
||
return f"{norm_date}|{norm_payee}|{norm_desc}|{norm_amount}"
|
||
|
||
|
||
def convert(in_path, out_path):
|
||
# Load state
|
||
state = load_state()
|
||
seen_keys = set(state.get("seen_keys", []))
|
||
|
||
written_count = 0
|
||
skipped_count = 0
|
||
|
||
# Read Halifax CSV and write a cleaned CSV for Actual
|
||
with open(in_path, newline="", encoding="utf-8-sig") as f_in, \
|
||
open(out_path, "w", newline="", encoding="utf-8") as f_out:
|
||
|
||
reader = csv.DictReader(f_in)
|
||
|
||
# Output columns tailored for Actual’s CSV import
|
||
fieldnames = ["Date", "Payee", "Description", "Amount"]
|
||
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
|
||
writer.writeheader()
|
||
|
||
for row in reader:
|
||
if not row:
|
||
continue
|
||
|
||
# Date in Halifax export (dd/mm/yyyy)
|
||
date = (row.get("Transaction Date") or "").strip()
|
||
if not date:
|
||
# Skip lines without a date
|
||
continue
|
||
|
||
desc = (row.get("Transaction Description") or "").strip()
|
||
payee = desc or (row.get("Transaction Type") or "").strip()
|
||
|
||
# Halifax has separate Debit / Credit columns
|
||
debit_str = (row.get("Debit Amount") or "").replace(",", "").strip()
|
||
credit_str = (row.get("Credit Amount") or "").replace(",", "").strip()
|
||
|
||
amount = None
|
||
|
||
# Debits (money going out) → negative
|
||
if debit_str:
|
||
try:
|
||
amount = -Decimal(debit_str)
|
||
except InvalidOperation:
|
||
amount = None
|
||
|
||
# Credits (money coming in) → positive
|
||
elif credit_str:
|
||
try:
|
||
amount = Decimal(credit_str)
|
||
except InvalidOperation:
|
||
amount = None
|
||
|
||
# If we still don't have an amount, skip the row
|
||
if amount is None:
|
||
continue
|
||
|
||
key = make_key(date, payee, desc, amount)
|
||
if key in seen_keys:
|
||
skipped_count += 1
|
||
continue
|
||
|
||
seen_keys.add(key)
|
||
written_count += 1
|
||
|
||
writer.writerow({
|
||
"Date": date,
|
||
"Payee": payee,
|
||
"Description": desc,
|
||
"Amount": f"{amount:.2f}",
|
||
})
|
||
|
||
# Keep the state from growing forever – remember last N keys
|
||
max_keys = 50000
|
||
seen_list = list(seen_keys)
|
||
if len(seen_list) > max_keys:
|
||
seen_list = seen_list[-max_keys:]
|
||
|
||
save_state({"seen_keys": seen_list})
|
||
|
||
print(f"Written cleaned file to {out_path}")
|
||
print(f" New transactions written: {written_count}")
|
||
print(f" Transactions skipped as duplicates: {skipped_count}")
|
||
|
||
|
||
def main():
|
||
if len(sys.argv) < 2:
|
||
print("Usage: python clean-halifax-actual.py input.csv [output.csv]")
|
||
sys.exit(1)
|
||
|
||
in_path = Path(sys.argv[1])
|
||
|
||
if len(sys.argv) >= 3:
|
||
out_path = Path(sys.argv[2])
|
||
else:
|
||
# Default to inputname_actual.csv
|
||
out_path = in_path.with_name(in_path.stem + "_actual.csv")
|
||
|
||
convert(in_path, out_path)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|