updating the scripts

This commit is contained in:
2026-02-11 13:31:50 +00:00
parent 3a845bf532
commit 36211d28d8
18 changed files with 700 additions and 148 deletions

View File

@@ -0,0 +1,97 @@
#!/usr/bin/env python3
import csv
from decimal import Decimal, InvalidOperation
from pathlib import Path
import sys
RAW_DIR = Path("~/master-folder/attachments/bank-statements/halifax/raw").expanduser()
CLEANED_DIR = Path("~/master-folder/attachments/bank-statements/halifax/cleaned").expanduser()
def convert(in_path: Path, out_path: Path):
with open(in_path, newline="", encoding="utf-8-sig") as f_in, \
open(out_path, "w", newline="", encoding="utf-8") as f_out:
reader = csv.DictReader(f_in)
fieldnames = ["Date", "Payee", "Description", "Amount"]
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
if not row:
continue
date = (row.get("Transaction Date") or "").strip()
if not date:
continue
desc = (row.get("Transaction Description") or "").strip()
payee = desc or (row.get("Transaction Type") or "").strip()
debit_str = (row.get("Debit Amount") or "").replace(",", "").strip()
credit_str = (row.get("Credit Amount") or "").replace(",", "").strip()
amount = None
if debit_str:
try:
amount = -Decimal(debit_str)
except InvalidOperation:
continue
elif credit_str:
try:
amount = Decimal(credit_str)
except InvalidOperation:
continue
if amount is None:
continue
writer.writerow({
"Date": date,
"Payee": payee,
"Description": desc,
"Amount": f"{amount:.2f}",
})
def select_file(files: list[Path]) -> Path:
print("\nSelect a Halifax CSV to clean:\n")
for i, f in enumerate(files, start=1):
print(f"[{i}] {f.name}")
try:
choice = int(input("\nEnter number: ").strip())
return files[choice - 1]
except (ValueError, IndexError):
print("Invalid selection.")
sys.exit(1)
def main():
if not RAW_DIR.exists():
print(f"Raw directory does not exist: {RAW_DIR}")
sys.exit(1)
CLEANED_DIR.mkdir(parents=True, exist_ok=True)
csv_files = sorted(RAW_DIR.glob("*.csv"))
if not csv_files:
print("No CSV files found in raw directory.")
sys.exit(0)
in_path = select_file(csv_files)
out_name = f"{in_path.stem}-cleaned.csv"
out_path = CLEANED_DIR / out_name
convert(in_path, out_path)
print(f"\n✔ Written cleaned file to:\n{out_path}\n")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,132 @@
#!/usr/bin/env python3
import csv
from decimal import Decimal, InvalidOperation
from pathlib import Path
from datetime import datetime
import sys
RAW_DIR = Path("/home/zaine/master-folder/attachments/bank-statements/nationwide/raw")
CLEANED_DIR = Path("/home/zaine/master-folder/attachments/bank-statements/nationwide/cleaned")
def clean_money(value: str) -> Decimal | None:
if not value:
return None
value = (
value.replace("£", "")
.replace(",", "")
.strip()
)
if not value:
return None
try:
return Decimal(value)
except InvalidOperation:
return None
def parse_date(value: str) -> str | None:
"""
Convert '01 Dec 2025''12/01/2025'
"""
value = value.strip()
if not value:
return None
try:
dt = datetime.strptime(value, "%d %b %Y")
return dt.strftime("%m/%d/%Y")
except ValueError:
return None
def convert(in_path: Path, out_path: Path):
with open(in_path, newline="", encoding="latin-1") as f_in, \
open(out_path, "w", newline="", encoding="utf-8") as f_out:
reader = csv.reader(f_in)
# Skip metadata lines until header
for row in reader:
if row and row[0] == "Date":
header = row
break
else:
raise RuntimeError("Could not find Nationwide CSV header row")
dict_reader = csv.DictReader(f_in, fieldnames=header)
fieldnames = ["Date", "Payee", "Description", "Amount"]
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
writer.writeheader()
for row in dict_reader:
raw_date = row.get("Date") or ""
date = parse_date(raw_date)
if not date:
continue
desc = (row.get("Description") or "").strip()
payee = desc
paid_out = clean_money(row.get("Paid out", ""))
paid_in = clean_money(row.get("Paid in", ""))
if paid_out is not None:
amount = -paid_out
elif paid_in is not None:
amount = paid_in
else:
continue
writer.writerow({
"Date": date,
"Payee": payee,
"Description": desc,
"Amount": f"{amount:.2f}",
})
def select_file(files: list[Path]) -> Path:
print("\nSelect a Nationwide CSV to clean:\n")
for i, f in enumerate(files, start=1):
print(f"[{i}] {f.name}")
try:
choice = int(input("\nEnter number: ").strip())
return files[choice - 1]
except (ValueError, IndexError):
print("Invalid selection.")
sys.exit(1)
def main():
if not RAW_DIR.exists():
print(f"Raw directory does not exist: {RAW_DIR}")
sys.exit(1)
CLEANED_DIR.mkdir(parents=True, exist_ok=True)
csv_files = sorted(RAW_DIR.glob("*.csv"))
if not csv_files:
print("No CSV files found in raw directory.")
sys.exit(0)
in_path = select_file(csv_files)
out_name = f"{in_path.stem}-cleaned.csv"
out_path = CLEANED_DIR / out_name
convert(in_path, out_path)
print(f"\n✔ Written cleaned file to:\n{out_path}\n")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,104 @@
#!/usr/bin/env python3
import csv
from decimal import Decimal, InvalidOperation
import sys
from pathlib import Path
from datetime import datetime
def clean_money(value: str) -> Decimal | None:
if not value:
return None
value = (
value.replace("£", "")
.replace(",", "")
.strip()
)
if not value:
return None
try:
return Decimal(value)
except InvalidOperation:
return None
def parse_date(value: str) -> str | None:
"""
Convert '01 Dec 2025''12/01/2025'
"""
value = value.strip()
if not value:
return None
try:
dt = datetime.strptime(value, "%d %b %Y")
return dt.strftime("%m/%d/%Y")
except ValueError:
return None
def convert(in_path: Path, out_path: Path):
with open(in_path, newline="", encoding="latin-1") as f_in, \
open(out_path, "w", newline="", encoding="utf-8") as f_out:
reader = csv.reader(f_in)
# Skip metadata lines until we hit the header
for row in reader:
if row and row[0] == "Date":
header = row
break
else:
raise RuntimeError("Could not find Nationwide CSV header row")
dict_reader = csv.DictReader(f_in, fieldnames=header)
fieldnames = ["Date", "Payee", "Description", "Amount"]
writer = csv.DictWriter(f_out, fieldnames=fieldnames)
writer.writeheader()
for row in dict_reader:
raw_date = row.get("Date") or ""
date = parse_date(raw_date)
if not date:
continue
desc = (row.get("Description") or "").strip()
payee = desc
paid_out = clean_money(row.get("Paid out", ""))
paid_in = clean_money(row.get("Paid in", ""))
if paid_out is not None:
amount = -paid_out
elif paid_in is not None:
amount = paid_in
else:
continue
writer.writerow({
"Date": date,
"Payee": payee,
"Description": desc,
"Amount": f"{amount:.2f}",
})
def main():
if len(sys.argv) < 2:
print("Usage: python nationwide-cleaner.py input.csv [output.csv]")
sys.exit(1)
in_path = Path(sys.argv[1])
out_path = Path(sys.argv[2]) if len(sys.argv) >= 3 else in_path.with_name(
in_path.stem + "_actual.csv"
)
convert(in_path, out_path)
print(f"Written cleaned file to {out_path}")
if __name__ == "__main__":
main()

View File

@@ -1,21 +0,0 @@
#!/usr/bin/env python3
import os, random, requests, discord
from discord import app_commands
from discord.ext import commands
from datetime import datetime
from pathlib import Path
NIKAH_DAY = datetime(2026, 08, 02)
WALIMAH_DAY = datetime(2026, 08, 09)
docstring = """
This script sends daily countdown reminders via a Discord webhook about
the number of days remaining until two significant events: Nikah Day and
Walimah Day. It calculates the days left for each event and formats a
message to be sent to a specified Discord channel.
"""
# -----------------------------
# CONFIG
# -----------------------------
DISCORD_WEBHOOK_URL = "https://discordapp.com/api/webhooks/146000414

View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python3
import requests
from datetime import datetime
docstring = """
This script sends a reminder message via a Discord webhook every month to import the latest bank statements into actual.
"""
DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1465643876953100433/P5AIBRJobwkryQesUXBsJx6W91VseUbcPZbiOvGTie5FvUdtiOtP2XPx_jlcaXdr56X1"
if __name__ == "__main__":
actual_link = "https://actual.zainezq.com/"
halifax_link = "https://www.halifax-online.co.uk/personal/logon/login.jsp"
message = f"Reminder to import your latest bank statements into Actual!\n\n"
message += f"🔗 [Actual App]({actual_link})\n"
message += f"🔗 [Halifax Online Banking]({halifax_link})\n"
payload = {
"content": message
}
r = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
r.raise_for_status()

View File

@@ -0,0 +1,26 @@
#!/usr/bin/env python3
import requests
from datetime import datetime
docstring = """
This script sends a reminder message via a Discord webhook every month to import the latest bank statements into actual.
"""
DISCORD_WEBHOOK_URL = "https://discord.com/api/webhooks/1461100056580067684/BAxF4ndrugLJX8L6WyUtbzOtqoTR0GyxMWhg6jzkdCy1CC8yLfDb7e5erz_FoMUHmviw"
if __name__ == "__main__":
actual_link = "https://actual.zainezq.com/"
halifax_link = "https://www.halifax-online.co.uk/personal/logon/login.jsp"
message = f"Reminder to import your latest bank statements into Actual!\n\n"
message += f"🔗 [Actual App]({actual_link})\n"
message += f"🔗 [Halifax Online Banking]({halifax_link})\n"
payload = {
"content": message
}
r = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
r.raise_for_status()

View File

@@ -0,0 +1,303 @@
#!/usr/bin/env python3
import os, random, requests, discord
from discord import app_commands
from discord.ext import commands
from datetime import datetime
from pathlib import Path
docstring = """
This script connects to a Miniflux RSS feed reader instance and a Discord bot.
It allows users to fetch random unread articles from specified RSS feeds
via Discord commands, mark them as read in Miniflux, and append their details
to an Org-mode file for personal knowledge management.
These are the available commands:
- /rss [feed]: Fetch a random unread article from the specified feed.
- /rss-any: Fetch a random unread article from any feed.
- /next: Fetch another article from the same feed.
- /done: Mark the current article as read and save it to the Org file.
"""
# -----------------------------
# CONFIG
# -----------------------------
MINIFLUX_URL = "https://miniflux.zainezq.com"
MINIFLUX_TOKEN = "Q_tDLF7Ht-aPjub2Xsbn6JCKM1WbNUEDfjHPYAIzT0I="
DISCORD_TOKEN = "MTQ1OTYwNzg2Mjg1Nzk1NzY0OQ.GBTRdl.iqfwD3kMN277z7rhm6lG5EgwSyqyPoCF1t8NVI"
ORG_RSS_FILE = "/home/zaine/master-folder/org_files/org_roam/20260111204228-rss_articles.org"
GUILD_ID = 1459545736961327116
HEADERS = {
"X-Auth-Token": MINIFLUX_TOKEN
}
intents = discord.Intents.default()
bot = commands.Bot(command_prefix="!", intents=intents)
tree = bot.tree
current_state = {}
def append_to_org(entry):
org_path = Path(ORG_RSS_FILE)
org_path.parent.mkdir(parents=True, exist_ok=True)
title = entry["title"].replace("\n", " ").strip()
url = entry["url"]
feed = entry["feed"]["title"]
entry_id = entry["id"]
date_read = datetime.now().strftime("%Y-%m-%d %H:%M")
org_block = f"""* [[{url}][{title}]]
:PROPERTIES:
:FEED: {feed}
:MINIFLUX_ID: {entry_id}
:READ_AT: {date_read}
:END:
"""
with org_path.open("a", encoding="utf-8") as f:
f.write(org_block)
def get_feeds():
r = requests.get(f"{MINIFLUX_URL}/v1/feeds", headers=HEADERS, timeout=10)
r.raise_for_status()
return r.json()
def get_unread_entries(feed_id=None, limit=100):
params = {"status": "unread", "limit": limit}
if feed_id:
params["feed_id"] = feed_id
r = requests.get(
f"{MINIFLUX_URL}/v1/entries",
headers=HEADERS,
params=params,
timeout=10
)
r.raise_for_status()
return r.json()["entries"]
def mark_as_read(entry_id):
r = requests.put(
f"{MINIFLUX_URL}/v1/entries",
headers=HEADERS,
json={"entry_ids": [entry_id], "status": "read"},
timeout=10
)
r.raise_for_status()
async def send_random_entry(
interaction: discord.Interaction,
feed_id: int,
):
entries = get_unread_entries(feed_id=feed_id)
if not entries:
await interaction.followup.send(
"🎉 No unread articles in this feed!"
)
return
entry = random.choice(entries)
current_state[interaction.channel_id] = {
"entry": entry,
"feed_id": feed_id
}
embed = discord.Embed(
title=entry["title"],
url=entry["url"],
description=(entry.get("summary") or "")[:500],
color=0x3498DB
)
embed.set_footer(
text=f"{entry['feed']['title']} • /next or /done"
)
await interaction.followup.send(embed=embed)
async def feed_autocomplete(
interaction: discord.Interaction,
current: str,
):
feeds = get_feeds()
return [
app_commands.Choice(name=f["title"], value=str(f["id"]))
for f in feeds
if current.lower() in f["title"].lower()
][:25]
async def entry_autocomplete(
interaction: discord.Interaction,
current: str,
):
if len(current) < 3:
return []
matches = []
offset = 0
limit = 100
while len(matches) < 25:
r = requests.get(
f"{MINIFLUX_URL}/v1/entries",
headers=HEADERS,
params={
"status": "unread",
"limit": limit,
"offset": offset,
},
timeout=10
)
r.raise_for_status()
entries = r.json()["entries"]
if not entries:
break # no more unread entries
for e in entries:
if current.lower() in e["title"].lower():
matches.append(
app_commands.Choice(
name=e["title"][:100],
value=str(e["id"])
)
)
if len(matches) >= 25:
break
offset += limit
return matches
def get_entry(entry_id: int):
r = requests.get(
f"{MINIFLUX_URL}/v1/entries/{entry_id}",
headers=HEADERS,
timeout=10
)
r.raise_for_status()
return r.json()
async def send_entry_embed(interaction, entry):
current_state[interaction.channel_id] = {
"entry": entry,
"feed_id": entry["feed"]["id"]
}
embed = discord.Embed(
title=entry["title"],
url=entry["url"],
description=(entry.get("summary") or "")[:500],
color=0x3498DB
)
embed.set_footer(
text=f"{entry['feed']['title']} • /next or /done"
)
await interaction.followup.send(embed=embed)
@tree.command(name="rss", description="Get a random unread RSS article")
@app_commands.describe(feed="Which feed to read from")
@app_commands.autocomplete(feed=feed_autocomplete)
async def rss(interaction: discord.Interaction, feed: str):
await interaction.response.defer()
await send_random_entry(interaction, feed_id=int(feed))
@tree.command(name="rss-any", description="Get a random unread RSS article from any feed")
async def rss_any(interaction: discord.Interaction):
await interaction.response.defer()
await send_random_entry(interaction, feed_id=None)
@tree.command(
name="rss-find",
description="Search unread RSS articles by title"
)
@app_commands.describe(title="Start typing the article title")
@app_commands.autocomplete(title=entry_autocomplete)
async def rss_find(interaction: discord.Interaction, title: str):
await interaction.response.defer()
entry_id = int(title)
entry = get_entry(entry_id)
await send_entry_embed(interaction, entry)
@tree.command(name="next", description="Get another article from the same feed")
async def next_article(interaction: discord.Interaction):
state = current_state.get(interaction.channel_id)
if not state:
await interaction.response.send_message(
"⚠️ No active feed. Use /rss first.",
ephemeral=True
)
return
await interaction.response.defer()
await send_random_entry(
interaction,
feed_id=state.get("feed_id")
)
@tree.command(name="done", description="Mark the current article as read and save to Org")
async def done(interaction: discord.Interaction):
state = current_state.get(interaction.channel_id)
if not state:
await interaction.response.send_message(
"⚠️ No article to mark as read.",
ephemeral=True
)
return
if "entry" not in state:
await interaction.response.send_message(
"⚠️ This article was queued before the last update. Please use /rss again.",
ephemeral=True
)
del current_state[interaction.channel_id]
return
entry = state["entry"]
mark_as_read(entry["id"])
append_to_org(entry)
del current_state[interaction.channel_id]
await interaction.response.send_message(
"✅ Article marked as read and added to Org"
)
@bot.event
async def on_ready():
guild = discord.Object(id=GUILD_ID)
tree.copy_global_to(guild=guild)
await tree.sync(guild=guild)
print(f"✅ Logged in as {bot.user} (guild-only sync)")
if __name__ == "__main__":
if not all([MINIFLUX_URL, MINIFLUX_TOKEN, DISCORD_TOKEN]):
raise RuntimeError("Missing environment variables")
bot.run(DISCORD_TOKEN)

View File

@@ -86,6 +86,16 @@ def format_ts(ts):
except ValueError:
return ts
def delete_and_recreate_log_files():
log_files = [
CALIBRE_LOG,
ROAM_LOG,
WEB_LOG
]
for log_file in log_files:
if log_file.exists():
log_file.unlink()
log_file.touch()
# -----------------------------
# MAIN
@@ -128,6 +138,7 @@ def main():
}
send_discord_embed(embed)
delete_and_recreate_log_files()
if __name__ == "__main__":

View File

@@ -1,126 +0,0 @@
#!/usr/bin/env python3
import json
import requests
from pathlib import Path
from datetime import datetime
# -----------------------------
# CONFIG
# -----------------------------
LOG_DIR = Path("/home/zaine/logs")
CALIBRE_LOG = LOG_DIR / "org-books-calibre.log"
ROAM_LOG = LOG_DIR / "org-roam.log"
WEB_LOG = LOG_DIR / "org-web.log"
DISCORD_WEBHOOK_URL = "https://discordapp.com/api/webhooks/1460004143870513464/kRSTsNMXU8AL3Pnj-t1EUKxrLb6cBKROdMVyowX02P46CPmayiB-RHi4UH5g6tCZQf5l"
# -----------------------------
# HELPERS
# -----------------------------
def send_discord_embed(embed: dict):
payload = {
"embeds": [embed]
}
r = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
r.raise_for_status()
def parse_calibre_log(path: Path):
successes = 0
failures = 0
last_ts = None
if not path.exists():
return successes, failures, last_ts
for line in path.read_text().splitlines():
try:
entry = json.loads(line)
status = entry.get("status")
ts = entry.get("timestamp")
if status == "success":
successes += 1
else:
failures += 1
if ts:
last_ts = ts
except json.JSONDecodeError:
failures += 1
return successes, failures, last_ts
def parse_roam_log(path: Path):
if not path.exists():
return False
return "✅ Full rebuild complete" in path.read_text()
def parse_web_log(path: Path):
if not path.exists():
return False
text = path.read_text()
return (
"Successfully installed" in text
and "JSON saved to" in text
)
def format_ts(ts):
if not ts:
return "Unknown"
try:
return datetime.fromisoformat(ts).strftime("%Y-%m-%d %H:%M")
except ValueError:
return ts
# -----------------------------
# MAIN
# -----------------------------
def main():
cal_ok, cal_fail, cal_last = parse_calibre_log(CALIBRE_LOG)
roam_ok = parse_roam_log(ROAM_LOG)
web_ok = parse_web_log(WEB_LOG)
all_ok = (cal_fail == 0) and roam_ok and web_ok
embed = {
"title": "Org Automation Status",
"color": 0x2ECC71 if all_ok else 0xE74C3C,
"fields": [
{
"name": "Calibre → Org",
"value": (
f"✅ Successes: **{cal_ok}**\n"
f"❌ Failures: **{cal_fail}**\n"
f"⏱ Last run: `{format_ts(cal_last)}`"
),
"inline": False
},
{
"name": "Org-Roam",
"value": "✅ Successful rebuild" if roam_ok else "❌ No success marker found",
"inline": True
},
{
"name": "Org-Web",
"value": "✅ Build + index complete" if web_ok else "❌ Build or index incomplete",
"inline": True
}
],
"footer": {
"text": "Generated automatically from ~/logs"
},
"timestamp": datetime.utcnow().isoformat()
}
send_discord_embed(embed)
if __name__ == "__main__":
main()