updating the scripts

legacy/halifax/clean-halifax-actual-2.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+import csv
+from decimal import Decimal, InvalidOperation
+import sys
+from pathlib import Path
+import json
+from datetime import datetime
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+STATE_FILE = SCRIPT_DIR / "halifax_state.json"
+
+def load_state():
+    if STATE_FILE.exists():
+        try:
+            return json.loads(STATE_FILE.read_text())
+        except json.JSONDecodeError:
+            # If it ever gets corrupted, start fresh
+            return {"seen_keys": []}
+    return {"seen_keys": []}
+
+
+def save_state(state):
+    STATE_FILE.write_text(json.dumps(state, indent=2))
+
+
+def normalize_text(s: str) -> str:
+    # collapse whitespace + lowercase
+    return " ".join(s.split()).lower()
+
+
+def make_key(date_str: str, payee: str, desc: str, amount: Decimal) -> str:
+    """
+    Build a dedupe key: normalized date + payee + description + amount.
+    We normalize:
+      - date into ISO (YYYY-MM-DD)
+      - payee/desc into lowercased, collapsed whitespace
+      - amount to 2 decimal places
+    """
+
+    # Halifax gives dd/mm/yyyy – convert to ISO for consistency
+    date_str = date_str.strip()
+    try:
+        # If it's already ISO, this will work too
+        try:
+            d = datetime.fromisoformat(date_str).date()
+        except ValueError:
+            d = datetime.strptime(date_str, "%d/%m/%Y").date()
+        norm_date = d.isoformat()
+    except Exception:
+        # If parsing fails for some reason, just use the raw string
+        norm_date = date_str
+
+    norm_payee = normalize_text(payee)
+    norm_desc = normalize_text(desc)
+    norm_amount = amount.quantize(Decimal("0.01"))
+
+    return f"{norm_date}|{norm_payee}|{norm_desc}|{norm_amount}"
+
+
+def convert(in_path, out_path):
+    # Load state
+    state = load_state()
+    seen_keys = set(state.get("seen_keys", []))
+
+    written_count = 0
+    skipped_count = 0
+
+    # Read Halifax CSV and write a cleaned CSV for Actual
+    with open(in_path, newline="", encoding="utf-8-sig") as f_in, \
+         open(out_path, "w", newline="", encoding="utf-8") as f_out:
+
+        reader = csv.DictReader(f_in)
+
+        # Output columns tailored for Actual’s CSV import
+        fieldnames = ["Date", "Payee", "Description", "Amount"]
+        writer = csv.DictWriter(f_out, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for row in reader:
+            if not row:
+                continue
+
+            # Date in Halifax export (dd/mm/yyyy)
+            date = (row.get("Transaction Date") or "").strip()
+            if not date:
+                # Skip lines without a date
+                continue
+
+            desc = (row.get("Transaction Description") or "").strip()
+            payee = desc or (row.get("Transaction Type") or "").strip()
+
+            # Halifax has separate Debit / Credit columns
+            debit_str = (row.get("Debit Amount") or "").replace(",", "").strip()
+            credit_str = (row.get("Credit Amount") or "").replace(",", "").strip()
+
+            amount = None
+
+            # Debits (money going out) → negative
+            if debit_str:
+                try:
+                    amount = -Decimal(debit_str)
+                except InvalidOperation:
+                    amount = None
+
+            # Credits (money coming in) → positive
+            elif credit_str:
+                try:
+                    amount = Decimal(credit_str)
+                except InvalidOperation:
+                    amount = None
+
+            # If we still don't have an amount, skip the row
+            if amount is None:
+                continue
+
+            key = make_key(date, payee, desc, amount)
+            if key in seen_keys:
+                skipped_count += 1
+                continue
+
+            seen_keys.add(key)
+            written_count += 1
+
+            writer.writerow({
+                "Date": date,
+                "Payee": payee,
+                "Description": desc,
+                "Amount": f"{amount:.2f}",
+            })
+
+    # Keep the state from growing forever – remember last N keys
+    max_keys = 50000
+    seen_list = list(seen_keys)
+    if len(seen_list) > max_keys:
+        seen_list = seen_list[-max_keys:]
+
+    save_state({"seen_keys": seen_list})
+
+    print(f"Written cleaned file to {out_path}")
+    print(f"  New transactions written: {written_count}")
+    print(f"  Transactions skipped as duplicates: {skipped_count}")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python clean-halifax-actual.py input.csv [output.csv]")
+        sys.exit(1)
+
+    in_path = Path(sys.argv[1])
+
+    if len(sys.argv) >= 3:
+        out_path = Path(sys.argv[2])
+    else:
+        # Default to inputname_actual.csv
+        out_path = in_path.with_name(in_path.stem + "_actual.csv")
+
+    convert(in_path, out_path)
+
+
+if __name__ == "__main__":
+    main()