zaine/scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updating the scripts

Browse Source
This commit is contained in:
zaine server
2026-02-11 13:31:50 +00:00
parent 3a845bf532
commit 36211d28d8
18 changed files with 700 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
legacy/halifax/clean-halifax-actual-2.py Normal file
View File

@@ -0,0 +1,162 @@
#!/usr/bin/env python3
import csv
from decimal import Decimal, InvalidOperation
import sys
from pathlib import Path
import json
from datetime import datetime
SCRIPT_DIR = Path(__file__).resolve().parent
STATE_FILE = SCRIPT_DIR / "halifax_state.json"
def load_state():
if STATE_FILE.exists():
try:
return json.loads(STATE_FILE.read_text())
except json.JSONDecodeError:
# If it ever gets corrupted, start fresh
return {"seen_keys": []}
return {"seen_keys": []}
def save_state(state):
STATE_FILE.write_text(json.dumps(state, indent=2))
def normalize_text(s: str) -> str:
# collapse whitespace + lowercase
return " ".join(s.split()).lower()
def make_key(date_str: str, payee: str, desc: str, amount: Decimal) -> str:
"""
Build a dedupe key: normalized date + payee + description + amount.
We normalize:
- date into ISO (YYYY-MM-DD)
- payee/desc into lowercased, collapsed whitespace
- amount to 2 decimal places
"""
# Halifax gives dd/mm/yyyy – convert to ISO for consistency
date_str = date_str.strip()
try:
# If it's already ISO, this will work too
try:
d = datetime.fromisoformat(date_str).date()
except ValueError:
d = datetime.strptime(date_str, "%d/%m/%Y").date()
norm_date = d.isoformat()
except Exception:
# If parsing fails for some reason, just use the raw string
norm_date = date_str
norm_payee = normalize_text(payee)
norm_desc = normalize_text(desc)
norm_amount = amount.quantize(Decimal("0.01"))
return f"{norm_date}|{norm_payee}|{norm_desc}|{norm_amount}"
def convert(in_path, out_path):
# Load state
state = load_state()
seen_keys = set(state.get("seen_keys", []))
written_count = 0
skipped_count = 0
# Read Halifax CSV and write a cleaned CSV for Actual
with open(in_path, newline="", encoding="utf-8-sig") as f_in, \
open(out_path, "w", newline="", encoding="utf-8") as f_out:
reader = csv.DictReader(f_in)
# Output columns tailored for Actual’s CSV import
fieldnames = ["Date", "Payee", "Description", "Amount"]
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
if not row:
continue
# Date in Halifax export (dd/mm/yyyy)
date = (row.get("Transaction Date") or "").strip()
if not date:
# Skip lines without a date
continue
desc = (row.get("Transaction Description") or "").strip()
payee = desc or (row.get("Transaction Type") or "").strip()
# Halifax has separate Debit / Credit columns
debit_str = (row.get("Debit Amount") or "").replace(",", "").strip()
credit_str = (row.get("Credit Amount") or "").replace(",", "").strip()
amount = None
# Debits (money going out) → negative
if debit_str:
try:
amount = -Decimal(debit_str)
except InvalidOperation:
amount = None
# Credits (money coming in) → positive
elif credit_str:
try:
amount = Decimal(credit_str)
except InvalidOperation:
amount = None
# If we still don't have an amount, skip the row
if amount is None:
continue
key = make_key(date, payee, desc, amount)
if key in seen_keys:
skipped_count += 1
continue
seen_keys.add(key)
written_count += 1
writer.writerow({
"Date": date,
"Payee": payee,
"Description": desc,
"Amount": f"{amount:.2f}",
})
# Keep the state from growing forever – remember last N keys
max_keys = 50000
seen_list = list(seen_keys)
if len(seen_list) > max_keys:
seen_list = seen_list[-max_keys:]
save_state({"seen_keys": seen_list})
print(f"Written cleaned file to {out_path}")
print(f" New transactions written: {written_count}")
print(f" Transactions skipped as duplicates: {skipped_count}")
def main():
if len(sys.argv) < 2:
print("Usage: python clean-halifax-actual.py input.csv [output.csv]")
sys.exit(1)
in_path = Path(sys.argv[1])
if len(sys.argv) >= 3:
out_path = Path(sys.argv[2])
else:
# Default to inputname_actual.csv
out_path = in_path.with_name(in_path.stem + "_actual.csv")
convert(in_path, out_path)
if __name__ == "__main__":
main()

29
legacy/halifax/halifax_state.json Normal file
View File

@@ -0,0 +1,29 @@
{
"seen_keys": [
"2025-11-03|naveed qayyum|naveed qayyum|-100.00",
"2025-11-13|mfg new john stree|mfg new john stree|-20.00",
"2025-11-28|naveed qayyum|naveed qayyum|-50.00",
"2025-11-27|mfg new john stree|mfg new john stree|-15.01",
"2025-11-27|subway @mfg new jo|subway @mfg new jo|-1.49",
"2025-11-20|n qayyum|n qayyum|10.00",
"2025-11-21|wm morrisons store|wm morrisons store|-11.65",
"2025-11-03|mrs farzana k qayy|mrs farzana k qayy|-100.00",
"2025-11-18|hockley service st|hockley service st|-30.01",
"2025-11-26|refresh vending|refresh vending|-3.00",
"2025-11-25|hockley service st|hockley service st|-35.01",
"2025-11-17|n qayyum|n qayyum|30.00",
"2025-11-28|microlise limited|microlise limited|2058.30",
"2025-11-11|sky digital|sky digital|-27.00",
"2025-11-03|ummah welfare trus|ummah welfare trus|-50.00",
"2025-11-20|www.voxi.co.uk|www.voxi.co.uk|-10.00",
"2025-11-20|mfg new john stree|mfg new john stree|-24.15",
"2025-11-05|hockley service st|hockley service st|-30.01",
"2025-11-03|amina qayyum|amina qayyum|-300.00",
"2025-11-06|the gym ltd|the gym ltd|-17.99",
"2025-11-28|mrs farzana k qayy|mrs farzana k qayy|-100.00",
"2025-11-17|naveed qayyum|naveed qayyum|-100.00",
"2025-11-24|p.o. 107 lozells r|p.o. 107 lozells r|-300.00",
"2025-11-10|morr birmingham ca|morr birmingham ca|-40.00",
"2025-11-03|lnk notemachine|lnk notemachine|-250.00"
]
}

99
legacy/halifax/new-cleaned.py Normal file
View File

@@ -0,0 +1,99 @@
#!/usr/bin/env python3
import csv
from decimal import Decimal, InvalidOperation
import sys
from pathlib import Path
import hashlib
from collections import defaultdict
def transaction_hash(date, payee, desc, amount):
"""Generate a stable hash for each transaction."""
base = f"{date}|{payee}|{desc}|{amount}"
return hashlib.md5(base.encode("utf-8")).hexdigest()[:10] # short ID
def convert(in_path, out_path):
with open(in_path, newline="", encoding="utf-8-sig") as f_in, \
open(out_path, "w", newline="", encoding="utf-8") as f_out:
reader = csv.DictReader(f_in)
# Add UniqueID field to prevent Actual from treating repeated merchants as duplicates
fieldnames = ["Date", "Payee", "Description", "Amount", "UniqueID"]
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
writer.writeheader()
seen_hashes = defaultdict(int)
skipped = 0
for row in reader:
if not row:
continue
date = (row.get("Transaction Date") or "").strip()
if not date:
continue
desc = (row.get("Transaction Description") or "").strip()
payee = desc or (row.get("Transaction Type") or "").strip()
debit_str = (row.get("Debit Amount") or "").replace(",", "").strip()
credit_str = (row.get("Credit Amount") or "").replace(",", "").strip()
amount = None
if debit_str:
try:
amount = -Decimal(debit_str)
except InvalidOperation:
continue
elif credit_str:
try:
amount = Decimal(credit_str)
except InvalidOperation:
continue
else:
continue
# Generate transaction fingerprint
unique_id = transaction_hash(date, payee, desc, amount)
# Check if we've seen this transaction hash already
seen_hashes[unique_id] += 1
if seen_hashes[unique_id] > 1:
# Optional: skip true duplicates or just warn
print(f"⚠️ Duplicate detected: {date} | {payee} | {amount}")
skipped += 1
continue
writer.writerow({
"Date": date,
"Payee": payee,
"Description": desc,
"Amount": f"{amount:.2f}",
"UniqueID": unique_id,
})
print(f"\n✅ Written cleaned file to {out_path}")
if skipped:
print(f"⚠️ Skipped {skipped} duplicate entries based on content hash.")
def main():
if len(sys.argv) < 2:
print("Usage: python clean_halifax_actual.py input.csv [output.csv]")
sys.exit(1)
in_path = Path(sys.argv[1])
if len(sys.argv) >= 3:
out_path = Path(sys.argv[2])
else:
out_path = in_path.with_name(in_path.stem + "_actual.csv")
convert(in_path, out_path)
if __name__ == "__main__":
main()