Updates
This commit is contained in:
97
bash-scripts/system-clean.sh
Executable file
97
bash-scripts/system-clean.sh
Executable file
@@ -0,0 +1,97 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# Colours
|
||||||
|
# ============================
|
||||||
|
RED="\e[31m"
|
||||||
|
GREEN="\e[32m"
|
||||||
|
YELLOW="\e[33m"
|
||||||
|
BLUE="\e[34m"
|
||||||
|
MAGENTA="\e[35m"
|
||||||
|
CYAN="\e[36m"
|
||||||
|
BOLD="\e[1m"
|
||||||
|
RESET="\e[0m"
|
||||||
|
|
||||||
|
line() {
|
||||||
|
echo -e "${CYAN}────────────────────────────────────────────────────────${RESET}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# DATE HEADER
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${MAGENTA}=== System Clean Run: $(date '+%Y-%m-%d %H:%M:%S') ===${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
echo -e "${BOLD}${GREEN}=== Ubuntu System Cleanup ===${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 1. Clean APT cache
|
||||||
|
# ============================
|
||||||
|
echo -y >/dev/null 2>&1
|
||||||
|
sudo apt autoclean -y >/dev/null 2>&1
|
||||||
|
sudo apt clean -y >/dev/null 2>&1
|
||||||
|
echo -e "${GREEN}✓ APT cache cleaned.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 2. Journal logs cleanup
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[2/8] Cleaning journal logs (keeping 100MB)...${RESET}"
|
||||||
|
echo -e "${CYAN}Current journal disk usage:${RESET}"
|
||||||
|
journalctl --disk-usage || true
|
||||||
|
sudo journalctl --vacuum-size=100M >/dev/null 2>&1
|
||||||
|
echo -e "${GREEN}✓ Journal logs vacuumed.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# Remove old logs
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[3/8] Removing rotated logs...${RESET}"
|
||||||
|
sudo find /var/log -type f -name "*.gz" -delete
|
||||||
|
sudo find /var/log -type f -name "*.1" -delete
|
||||||
|
echo -e "${GREEN}✓ Old log archives removed.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 4. Clear /tmp and /var/tmp
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[4/8] Clearing temp directories...${RESET}"
|
||||||
|
sudo rm -rf /tmp/* /var/tmp/*
|
||||||
|
echo -e "${GREEN}✓ Temporary files cleared.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 5. Free RAM (cache)
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[5/8] Dropping filesystem caches (safe)...${RESET}"
|
||||||
|
sync
|
||||||
|
echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
|
||||||
|
echo -e "${GREEN}✓ RAM caches dropped.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 6. Reset failed services
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[6/8] Resetting failed systemd units...${RESET}"
|
||||||
|
sudo systemctl reset-failed >/dev/null 2>&1
|
||||||
|
echo -e "${GREEN}✓ Failed units reset.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 7. Disk usage snapshot
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[7/8] Largest directories under / ...${RESET}"
|
||||||
|
du -h --max-depth=1 / 2>/dev/null | sort -hr | head -n 10
|
||||||
|
echo -e "${GREEN}✓ Disk usage shown.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
# ============================
|
||||||
|
# 8. CPU usage snapshot
|
||||||
|
# ============================
|
||||||
|
echo -e "${BOLD}${BLUE}[8/8] Top CPU-consuming processes...${RESET}"
|
||||||
|
ps -eo pid,ppid,cmd,%mem,%cpu --sort=-%cpu | head
|
||||||
|
echo -e "${GREEN}✓ CPU snapshot shown.${RESET}"
|
||||||
|
line
|
||||||
|
|
||||||
|
echo -e "${BOLD}${GREEN}=== Cleanup Complete! ===${RESET}"
|
||||||
0
bash-scripts/system-clean.sh~
Normal file
0
bash-scripts/system-clean.sh~
Normal file
0
cleaners/clean-halifax-actual.py
Normal file → Executable file
0
cleaners/clean-halifax-actual.py
Normal file → Executable file
0
cleaners/clean-halifax-actual.py~
Normal file → Executable file
0
cleaners/clean-halifax-actual.py~
Normal file → Executable file
0
cleaners/nationwide-cleaner.py
Normal file → Executable file
0
cleaners/nationwide-cleaner.py
Normal file → Executable file
0
cleaners/nationwide-cleaner.py~
Normal file → Executable file
0
cleaners/nationwide-cleaner.py~
Normal file → Executable file
377
cookbook_org_sync.py
Normal file
377
cookbook_org_sync.py
Normal file
@@ -0,0 +1,377 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
from urllib.error import URLError
|
||||||
|
import base64
|
||||||
|
|
||||||
|
docstring = """
|
||||||
|
This script fetches recipes from the Nextcloud Cookbook API, merges them
|
||||||
|
with any existing notes already written in the output Org file (cook dates,
|
||||||
|
tips, etc.), and regenerates the file while preserving the org-roam header
|
||||||
|
and all hand-written notes.
|
||||||
|
|
||||||
|
It runs on a cron schedule (e.g. nightly) and emits a JSON summary line on
|
||||||
|
stdout on success, or on stderr on failure, matching the same format used by
|
||||||
|
calibre_org_sync.
|
||||||
|
|
||||||
|
JSON summary format:
|
||||||
|
{"script": "cookbook_org_sync", "status": "success", "duration_ms": 42,
|
||||||
|
"output": "/home/zaine/master-folder/org_files/org_roam/20260307233451-recipes_cookbook.org",
|
||||||
|
"timestamp": "2026-03-08T00:00:02.123456"}
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Configuration – edit these values
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
NEXTCLOUD_BASE_URL = "https://nextcloud.zainezq.com"
|
||||||
|
NEXTCLOUD_USERNAME = os.environ.get("NEXTCLOUD_USER", "zaine") # or hard-code
|
||||||
|
NEXTCLOUD_PASSWORD = os.environ.get("NEXTCLOUD_PASS", "Shakkal123!") # or hard-code
|
||||||
|
|
||||||
|
RECIPES_ENDPOINT = "/apps/cookbook/api/v1/recipes"
|
||||||
|
|
||||||
|
OUTPUT_ORG = Path(
|
||||||
|
"/home/zaine/master-folder/org_files/org_roam/"
|
||||||
|
"20260307233451-recipes_cookbook.org"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# API fetch
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def fetch_recipes():
|
||||||
|
url = NEXTCLOUD_BASE_URL.rstrip("/") + RECIPES_ENDPOINT
|
||||||
|
|
||||||
|
# Support either a pre-encoded token (NEXTCLOUD_TOKEN) or user:pass
|
||||||
|
token = os.environ.get("NEXTCLOUD_TOKEN", "")
|
||||||
|
if token:
|
||||||
|
auth_header = f"Basic {token}"
|
||||||
|
else:
|
||||||
|
credentials = base64.b64encode(
|
||||||
|
f"{NEXTCLOUD_USERNAME}:{NEXTCLOUD_PASSWORD}".encode()
|
||||||
|
).decode()
|
||||||
|
auth_header = f"Basic {credentials}"
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Authorization": auth_header,
|
||||||
|
"Accept": "application/json",
|
||||||
|
"OCS-APIRequest": "true",
|
||||||
|
"User-Agent": (
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/122.0.0.0 Safari/537.36"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
if os.environ.get("COOKBOOK_DEBUG"):
|
||||||
|
print(f"[debug] GET {url}", file=sys.stderr)
|
||||||
|
print(f"[debug] Headers: { {k: v for k, v in headers.items()} }", file=sys.stderr)
|
||||||
|
|
||||||
|
req = Request(url, headers=headers)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urlopen(req, timeout=30) as resp:
|
||||||
|
raw = resp.read()
|
||||||
|
except URLError as exc:
|
||||||
|
# Capture HTTP error body for easier debugging
|
||||||
|
body = ""
|
||||||
|
if hasattr(exc, "read"):
|
||||||
|
try:
|
||||||
|
body = exc.read().decode(errors="replace")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise RuntimeError(
|
||||||
|
f"HTTP error from Nextcloud API: {exc} — body: {body!r}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RuntimeError(f"Invalid JSON from API: {exc}") from exc
|
||||||
|
|
||||||
|
if not isinstance(data, list):
|
||||||
|
raise RuntimeError(f"Unexpected API response shape: {type(data)}")
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Organise recipes by category
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def group_by_category(recipes):
|
||||||
|
"""Returns {category: [recipe, ...]} sorted alphabetically."""
|
||||||
|
grouped = {}
|
||||||
|
for recipe in recipes:
|
||||||
|
cat = (recipe.get("category") or "Uncategorised").strip()
|
||||||
|
grouped.setdefault(cat, []).append(recipe)
|
||||||
|
|
||||||
|
for cat in grouped:
|
||||||
|
grouped[cat].sort(key=lambda r: r.get("name", "").lower())
|
||||||
|
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Preserve existing hand-written notes from the current file
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def parse_existing_notes(path):
|
||||||
|
"""
|
||||||
|
Walks the current org file and collects the PROPERTIES drawer content
|
||||||
|
for each recipe heading (matched by RECIPE_ID property).
|
||||||
|
|
||||||
|
Returns {recipe_id_str: {"cooked_dates": [...], "tips": str, "extra_props": [...]}}
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
return {}
|
||||||
|
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
notes = {}
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(lines):
|
||||||
|
line = lines[i]
|
||||||
|
|
||||||
|
# Detect a level-2 recipe heading "** Some Title :keywords:"
|
||||||
|
if line.startswith("** "):
|
||||||
|
recipe_id = None
|
||||||
|
cooked_dates = []
|
||||||
|
tips_lines = []
|
||||||
|
extra_props = []
|
||||||
|
|
||||||
|
# Look for the PROPERTIES drawer immediately following
|
||||||
|
j = i + 1
|
||||||
|
in_props = False
|
||||||
|
while j < len(lines):
|
||||||
|
l = lines[j].strip()
|
||||||
|
if l == ":PROPERTIES:":
|
||||||
|
in_props = True
|
||||||
|
j += 1
|
||||||
|
continue
|
||||||
|
if in_props:
|
||||||
|
if l == ":END:":
|
||||||
|
in_props = False
|
||||||
|
j += 1
|
||||||
|
break
|
||||||
|
if l.startswith(":RECIPE_ID:"):
|
||||||
|
recipe_id = l.split(":", 2)[2].strip()
|
||||||
|
elif l.startswith(":COOKED:"):
|
||||||
|
# may be repeated
|
||||||
|
val = l.split(":", 2)[2].strip()
|
||||||
|
if val:
|
||||||
|
cooked_dates.append(val)
|
||||||
|
elif l.startswith(":TIPS:"):
|
||||||
|
tips_lines.append(l.split(":", 2)[2].strip())
|
||||||
|
elif l and not l.startswith(":CATEGORY:") \
|
||||||
|
and not l.startswith(":KEYWORDS:") \
|
||||||
|
and not l.startswith(":CREATED:") \
|
||||||
|
and not l.startswith(":MODIFIED:") \
|
||||||
|
and not l.startswith(":URL:"):
|
||||||
|
# Preserve unknown/custom properties
|
||||||
|
extra_props.append(l)
|
||||||
|
j += 1
|
||||||
|
continue
|
||||||
|
break # stop if we hit something that is not the drawer
|
||||||
|
|
||||||
|
if recipe_id:
|
||||||
|
notes[recipe_id] = {
|
||||||
|
"cooked_dates": cooked_dates,
|
||||||
|
"tips": " ".join(tips_lines),
|
||||||
|
"extra_props": extra_props,
|
||||||
|
}
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
return notes
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Org header handling
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def read_org_roam_header(path):
|
||||||
|
"""
|
||||||
|
Returns the org-roam PROPERTIES block (up to and including :END:).
|
||||||
|
Raises RuntimeError if the file exists but has no such block.
|
||||||
|
Returns None if the file does not exist.
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
return None
|
||||||
|
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
for i, line in enumerate(lines):
|
||||||
|
if line.strip() == ":END:":
|
||||||
|
return "\n".join(lines[: i + 1]).rstrip()
|
||||||
|
|
||||||
|
raise RuntimeError("No org-roam PROPERTIES drawer found in output file")
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Org body generation
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def emit_org_body(grouped, existing_notes):
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
lines.append("#+title: Recipes Cookbook")
|
||||||
|
lines.append("#+AUTHOR: Auto-generated")
|
||||||
|
lines.append("#+filetags: :recipes:cooking:org:index:")
|
||||||
|
lines.append("#+STARTUP: content")
|
||||||
|
lines.append(f"#+PROPERTY: GENERATED_AT {datetime.now().isoformat()}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
"# How to add notes: under any recipe heading, add :COOKED: <date> "
|
||||||
|
"to record when you made it, and :TIPS: your tip text to add notes."
|
||||||
|
)
|
||||||
|
lines.append(
|
||||||
|
"# Multiple :COOKED: lines are supported. "
|
||||||
|
"These are preserved across regenerations."
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for category in sorted(grouped.keys(), key=str.lower):
|
||||||
|
lines.append(f"* {category}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for recipe in grouped[category]:
|
||||||
|
rid = str(recipe.get("recipe_id") or recipe.get("id", ""))
|
||||||
|
name = recipe.get("name", "Unnamed Recipe")
|
||||||
|
keywords_raw = recipe.get("keywords", "") or ""
|
||||||
|
keywords = [k.strip() for k in keywords_raw.split(",") if k.strip()]
|
||||||
|
date_created = recipe.get("dateCreated", "")
|
||||||
|
date_modified = recipe.get("dateModified", "")
|
||||||
|
recipe_url = (
|
||||||
|
f"{NEXTCLOUD_BASE_URL.rstrip('/')}"
|
||||||
|
f"/apps/cookbook/webapp/recipes/{rid}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# org tag string from keywords (spaces → underscores)
|
||||||
|
tag_str = ":".join(k.replace(" ", "_") for k in keywords)
|
||||||
|
tag_block = f":{tag_str}:" if tag_str else ""
|
||||||
|
|
||||||
|
lines.append(f"** {name} {tag_block}".rstrip())
|
||||||
|
lines.append(":PROPERTIES:")
|
||||||
|
lines.append(f":RECIPE_ID: {rid}")
|
||||||
|
lines.append(f":CATEGORY: {category}")
|
||||||
|
if keywords:
|
||||||
|
lines.append(f":KEYWORDS: {keywords_raw}")
|
||||||
|
lines.append(f":CREATED: {date_created}")
|
||||||
|
lines.append(f":MODIFIED: {date_modified}")
|
||||||
|
lines.append(f":URL: {recipe_url}")
|
||||||
|
|
||||||
|
# Re-inject preserved notes
|
||||||
|
note = existing_notes.get(rid, {})
|
||||||
|
for cd in note.get("cooked_dates", []):
|
||||||
|
lines.append(f":COOKED: {cd}")
|
||||||
|
if note.get("tips"):
|
||||||
|
lines.append(f":TIPS: {note['tips']}")
|
||||||
|
for ep in note.get("extra_props", []):
|
||||||
|
lines.append(ep)
|
||||||
|
|
||||||
|
lines.append(":END:")
|
||||||
|
|
||||||
|
# ── Visible metadata (exported to HTML) ──────────────
|
||||||
|
cooked = note.get("cooked_dates", [])
|
||||||
|
cooked_str = ", ".join(cooked) if cooked else "/not yet made/"
|
||||||
|
|
||||||
|
lines.append("#+ATTR_HTML: :class recipe-meta")
|
||||||
|
lines.append("| | |")
|
||||||
|
lines.append("|---|---|")
|
||||||
|
lines.append(f"| *Category* | {category} |")
|
||||||
|
if keywords:
|
||||||
|
lines.append(f"| *Keywords* | {keywords_raw} |")
|
||||||
|
lines.append(f"| *Created* | {date_created[:10] if date_created else '—'} |")
|
||||||
|
lines.append(f"| *Modified* | {date_modified[:10] if date_modified else '—'} |")
|
||||||
|
lines.append(f"| *Cooked* | {cooked_str} |")
|
||||||
|
lines.append(f"| *Link* | [[{recipe_url}][Open in Nextcloud]] |")
|
||||||
|
|
||||||
|
if note.get("tips"):
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"*Tips:* {note['tips']}")
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Atomic write
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def atomic_write(path, content):
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
"w", delete=False, dir=str(path.parent), encoding="utf-8"
|
||||||
|
) as tmp:
|
||||||
|
tmp.write(content)
|
||||||
|
temp_name = tmp.name
|
||||||
|
os.replace(temp_name, path)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Main
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not NEXTCLOUD_USERNAME or not NEXTCLOUD_PASSWORD:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Nextcloud credentials not set. "
|
||||||
|
"Export NEXTCLOUD_USER and NEXTCLOUD_PASS environment variables."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 1. Read existing org-roam header (must already exist)
|
||||||
|
header = read_org_roam_header(OUTPUT_ORG)
|
||||||
|
if header is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"{OUTPUT_ORG} does not exist. "
|
||||||
|
"Create it first with the org-roam ID block."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Scrape any hand-written notes from the current file
|
||||||
|
existing_notes = parse_existing_notes(OUTPUT_ORG)
|
||||||
|
|
||||||
|
# 3. Fetch recipes from Nextcloud
|
||||||
|
recipes = fetch_recipes()
|
||||||
|
|
||||||
|
# 4. Group by category
|
||||||
|
grouped = group_by_category(recipes)
|
||||||
|
|
||||||
|
# 5. Generate new body and combine with header
|
||||||
|
body = emit_org_body(grouped, existing_notes)
|
||||||
|
content = header + "\n\n" + body
|
||||||
|
|
||||||
|
# 6. Write atomically
|
||||||
|
atomic_write(OUTPUT_ORG, content)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
duration_ms = int((time.time() - start) * 1000)
|
||||||
|
print(json.dumps({
|
||||||
|
"script": "cookbook_org_sync",
|
||||||
|
"status": "success",
|
||||||
|
"duration_ms": duration_ms,
|
||||||
|
"output": str(OUTPUT_ORG),
|
||||||
|
"timestamp": datetime.now().isoformat(),
|
||||||
|
}))
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
duration_ms = int((time.time() - start) * 1000)
|
||||||
|
print(json.dumps({
|
||||||
|
"script": "cookbook_org_sync",
|
||||||
|
"status": "failure",
|
||||||
|
"duration_ms": duration_ms,
|
||||||
|
"error": str(exc),
|
||||||
|
}), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
338
cookbook_org_sync.py~
Normal file
338
cookbook_org_sync.py~
Normal file
@@ -0,0 +1,338 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
from urllib.error import URLError
|
||||||
|
import base64
|
||||||
|
|
||||||
|
docstring = """
|
||||||
|
This script fetches recipes from the Nextcloud Cookbook API, merges them
|
||||||
|
with any existing notes already written in the output Org file (cook dates,
|
||||||
|
tips, etc.), and regenerates the file while preserving the org-roam header
|
||||||
|
and all hand-written notes.
|
||||||
|
|
||||||
|
It runs on a cron schedule (e.g. nightly) and emits a JSON summary line on
|
||||||
|
stdout on success, or on stderr on failure, matching the same format used by
|
||||||
|
calibre_org_sync.
|
||||||
|
|
||||||
|
JSON summary format:
|
||||||
|
{"script": "cookbook_org_sync", "status": "success", "duration_ms": 42,
|
||||||
|
"output": "/home/zaine/master-folder/org_files/org_roam/20260307233451-recipes_cookbook.org",
|
||||||
|
"timestamp": "2026-03-08T00:00:02.123456"}
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Configuration – edit these values
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
NEXTCLOUD_BASE_URL = "https://nextcloud.zainezq.com"
|
||||||
|
NEXTCLOUD_USERNAME = os.environ.get("NEXTCLOUD_USER", "") # or hard-code
|
||||||
|
NEXTCLOUD_PASSWORD = os.environ.get("NEXTCLOUD_PASS", "") # or hard-code
|
||||||
|
|
||||||
|
RECIPES_ENDPOINT = "/apps/cookbook/api/v1/recipes"
|
||||||
|
|
||||||
|
OUTPUT_ORG = Path(
|
||||||
|
"/home/zaine/master-folder/org_files/org_roam/"
|
||||||
|
"20260307233451-recipes_cookbook.org"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# API fetch
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def fetch_recipes():
|
||||||
|
url = NEXTCLOUD_BASE_URL.rstrip("/") + RECIPES_ENDPOINT
|
||||||
|
credentials = base64.b64encode(
|
||||||
|
f"{NEXTCLOUD_USERNAME}:{NEXTCLOUD_PASSWORD}".encode()
|
||||||
|
).decode()
|
||||||
|
|
||||||
|
req = Request(url, headers={
|
||||||
|
"Authorization": f"Basic {credentials}",
|
||||||
|
"Accept": "application/json",
|
||||||
|
"OCS-APIRequest": "true",
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urlopen(req, timeout=30) as resp:
|
||||||
|
raw = resp.read()
|
||||||
|
except URLError as exc:
|
||||||
|
raise RuntimeError(f"Failed to reach Nextcloud API: {exc}") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RuntimeError(f"Invalid JSON from API: {exc}") from exc
|
||||||
|
|
||||||
|
if not isinstance(data, list):
|
||||||
|
raise RuntimeError(f"Unexpected API response shape: {type(data)}")
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Organise recipes by category
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def group_by_category(recipes):
|
||||||
|
"""Returns {category: [recipe, ...]} sorted alphabetically."""
|
||||||
|
grouped = {}
|
||||||
|
for recipe in recipes:
|
||||||
|
cat = (recipe.get("category") or "Uncategorised").strip()
|
||||||
|
grouped.setdefault(cat, []).append(recipe)
|
||||||
|
|
||||||
|
for cat in grouped:
|
||||||
|
grouped[cat].sort(key=lambda r: r.get("name", "").lower())
|
||||||
|
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Preserve existing hand-written notes from the current file
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def parse_existing_notes(path):
|
||||||
|
"""
|
||||||
|
Walks the current org file and collects the PROPERTIES drawer content
|
||||||
|
for each recipe heading (matched by RECIPE_ID property).
|
||||||
|
|
||||||
|
Returns {recipe_id_str: {"cooked_dates": [...], "tips": str, "extra_props": [...]}}
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
return {}
|
||||||
|
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
notes = {}
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(lines):
|
||||||
|
line = lines[i]
|
||||||
|
|
||||||
|
# Detect a level-2 recipe heading "** Some Title :keywords:"
|
||||||
|
if line.startswith("** "):
|
||||||
|
recipe_id = None
|
||||||
|
cooked_dates = []
|
||||||
|
tips_lines = []
|
||||||
|
extra_props = []
|
||||||
|
|
||||||
|
# Look for the PROPERTIES drawer immediately following
|
||||||
|
j = i + 1
|
||||||
|
in_props = False
|
||||||
|
while j < len(lines):
|
||||||
|
l = lines[j].strip()
|
||||||
|
if l == ":PROPERTIES:":
|
||||||
|
in_props = True
|
||||||
|
j += 1
|
||||||
|
continue
|
||||||
|
if in_props:
|
||||||
|
if l == ":END:":
|
||||||
|
in_props = False
|
||||||
|
j += 1
|
||||||
|
break
|
||||||
|
if l.startswith(":RECIPE_ID:"):
|
||||||
|
recipe_id = l.split(":", 2)[2].strip()
|
||||||
|
elif l.startswith(":COOKED:"):
|
||||||
|
# may be repeated
|
||||||
|
val = l.split(":", 2)[2].strip()
|
||||||
|
if val:
|
||||||
|
cooked_dates.append(val)
|
||||||
|
elif l.startswith(":TIPS:"):
|
||||||
|
tips_lines.append(l.split(":", 2)[2].strip())
|
||||||
|
elif l and not l.startswith(":CATEGORY:") \
|
||||||
|
and not l.startswith(":KEYWORDS:") \
|
||||||
|
and not l.startswith(":CREATED:") \
|
||||||
|
and not l.startswith(":MODIFIED:") \
|
||||||
|
and not l.startswith(":URL:"):
|
||||||
|
# Preserve unknown/custom properties
|
||||||
|
extra_props.append(l)
|
||||||
|
j += 1
|
||||||
|
continue
|
||||||
|
break # stop if we hit something that is not the drawer
|
||||||
|
|
||||||
|
if recipe_id:
|
||||||
|
notes[recipe_id] = {
|
||||||
|
"cooked_dates": cooked_dates,
|
||||||
|
"tips": " ".join(tips_lines),
|
||||||
|
"extra_props": extra_props,
|
||||||
|
}
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
return notes
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Org header handling
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def read_org_roam_header(path):
|
||||||
|
"""
|
||||||
|
Returns the org-roam PROPERTIES block (up to and including :END:).
|
||||||
|
Raises RuntimeError if the file exists but has no such block.
|
||||||
|
Returns None if the file does not exist.
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
return None
|
||||||
|
|
||||||
|
lines = path.read_text(encoding="utf-8").splitlines()
|
||||||
|
for i, line in enumerate(lines):
|
||||||
|
if line.strip() == ":END:":
|
||||||
|
return "\n".join(lines[: i + 1]).rstrip()
|
||||||
|
|
||||||
|
raise RuntimeError("No org-roam PROPERTIES drawer found in output file")
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Org body generation
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def emit_org_body(grouped, existing_notes):
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
lines.append("#+title: Recipes Cookbook")
|
||||||
|
lines.append("#+AUTHOR: Auto-generated")
|
||||||
|
lines.append("#+filetags: :recipes:cooking:org:index:")
|
||||||
|
lines.append("#+STARTUP: content")
|
||||||
|
lines.append(f"#+PROPERTY: GENERATED_AT {datetime.now().isoformat()}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
"# How to add notes: under any recipe heading, add :COOKED: <date> "
|
||||||
|
"to record when you made it, and :TIPS: your tip text to add notes."
|
||||||
|
)
|
||||||
|
lines.append(
|
||||||
|
"# Multiple :COOKED: lines are supported. "
|
||||||
|
"These are preserved across regenerations."
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for category in sorted(grouped.keys(), key=str.lower):
|
||||||
|
lines.append(f"* {category}")
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
for recipe in grouped[category]:
|
||||||
|
rid = str(recipe.get("recipe_id") or recipe.get("id", ""))
|
||||||
|
name = recipe.get("name", "Unnamed Recipe")
|
||||||
|
keywords_raw = recipe.get("keywords", "") or ""
|
||||||
|
keywords = [k.strip() for k in keywords_raw.split(",") if k.strip()]
|
||||||
|
date_created = recipe.get("dateCreated", "")
|
||||||
|
date_modified = recipe.get("dateModified", "")
|
||||||
|
recipe_url = (
|
||||||
|
f"{NEXTCLOUD_BASE_URL.rstrip('/')}"
|
||||||
|
f"/apps/cookbook/webapp/recipes/{rid}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# org tag string from keywords (spaces → underscores)
|
||||||
|
tag_str = ":".join(k.replace(" ", "_") for k in keywords)
|
||||||
|
tag_block = f":{tag_str}:" if tag_str else ""
|
||||||
|
|
||||||
|
lines.append(f"** {name} {tag_block}".rstrip())
|
||||||
|
lines.append(":PROPERTIES:")
|
||||||
|
lines.append(f":RECIPE_ID: {rid}")
|
||||||
|
lines.append(f":CATEGORY: {category}")
|
||||||
|
if keywords:
|
||||||
|
lines.append(f":KEYWORDS: {keywords_raw}")
|
||||||
|
lines.append(f":CREATED: {date_created}")
|
||||||
|
lines.append(f":MODIFIED: {date_modified}")
|
||||||
|
lines.append(f":URL: {recipe_url}")
|
||||||
|
|
||||||
|
# Re-inject preserved notes
|
||||||
|
note = existing_notes.get(rid, {})
|
||||||
|
for cd in note.get("cooked_dates", []):
|
||||||
|
lines.append(f":COOKED: {cd}")
|
||||||
|
if note.get("tips"):
|
||||||
|
lines.append(f":TIPS: {note['tips']}")
|
||||||
|
for ep in note.get("extra_props", []):
|
||||||
|
lines.append(ep)
|
||||||
|
|
||||||
|
lines.append(":END:")
|
||||||
|
|
||||||
|
# Placeholder reminder if never cooked
|
||||||
|
if not note.get("cooked_dates"):
|
||||||
|
lines.append(
|
||||||
|
"# Add :COOKED: <yyyy-mm-dd> in the drawer above "
|
||||||
|
"when you first make this."
|
||||||
|
)
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Atomic write
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def atomic_write(path, content):
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
"w", delete=False, dir=str(path.parent), encoding="utf-8"
|
||||||
|
) as tmp:
|
||||||
|
tmp.write(content)
|
||||||
|
temp_name = tmp.name
|
||||||
|
os.replace(temp_name, path)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Main
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not NEXTCLOUD_USERNAME or not NEXTCLOUD_PASSWORD:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Nextcloud credentials not set. "
|
||||||
|
"Export NEXTCLOUD_USER and NEXTCLOUD_PASS environment variables."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 1. Read existing org-roam header (must already exist)
|
||||||
|
header = read_org_roam_header(OUTPUT_ORG)
|
||||||
|
if header is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"{OUTPUT_ORG} does not exist. "
|
||||||
|
"Create it first with the org-roam ID block."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Scrape any hand-written notes from the current file
|
||||||
|
existing_notes = parse_existing_notes(OUTPUT_ORG)
|
||||||
|
|
||||||
|
# 3. Fetch recipes from Nextcloud
|
||||||
|
recipes = fetch_recipes()
|
||||||
|
|
||||||
|
# 4. Group by category
|
||||||
|
grouped = group_by_category(recipes)
|
||||||
|
|
||||||
|
# 5. Generate new body and combine with header
|
||||||
|
body = emit_org_body(grouped, existing_notes)
|
||||||
|
content = header + "\n\n" + body
|
||||||
|
|
||||||
|
# 6. Write atomically
|
||||||
|
atomic_write(OUTPUT_ORG, content)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
start = time.time()
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
duration_ms = int((time.time() - start) * 1000)
|
||||||
|
print(json.dumps({
|
||||||
|
"script": "cookbook_org_sync",
|
||||||
|
"status": "success",
|
||||||
|
"duration_ms": duration_ms,
|
||||||
|
"output": str(OUTPUT_ORG),
|
||||||
|
"timestamp": datetime.now().isoformat(),
|
||||||
|
}))
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
duration_ms = int((time.time() - start) * 1000)
|
||||||
|
print(json.dumps({
|
||||||
|
"script": "cookbook_org_sync",
|
||||||
|
"status": "failure",
|
||||||
|
"duration_ms": duration_ms,
|
||||||
|
"error": str(exc),
|
||||||
|
}), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
0
cover_to_pdf.py
Normal file → Executable file
0
cover_to_pdf.py
Normal file → Executable file
0
discord-integrations/actual-reminder.py
Normal file → Executable file
0
discord-integrations/actual-reminder.py
Normal file → Executable file
0
discord-integrations/actual-reminder.py~
Normal file → Executable file
0
discord-integrations/actual-reminder.py~
Normal file → Executable file
0
discord-integrations/countdown.py
Normal file → Executable file
0
discord-integrations/countdown.py
Normal file → Executable file
0
discord-integrations/monitor.ps1
Normal file → Executable file
0
discord-integrations/monitor.ps1
Normal file → Executable file
0
discord-integrations/monitor.ps1~
Normal file → Executable file
0
discord-integrations/monitor.ps1~
Normal file → Executable file
0
legacy/crawler.py
Normal file → Executable file
0
legacy/crawler.py
Normal file → Executable file
0
legacy/fetch.py
Normal file → Executable file
0
legacy/fetch.py
Normal file → Executable file
0
legacy/gen-hash/generate-hash.js
Normal file → Executable file
0
legacy/gen-hash/generate-hash.js
Normal file → Executable file
0
legacy/gen-hash/package-lock.json
generated
Normal file → Executable file
0
legacy/gen-hash/package-lock.json
generated
Normal file → Executable file
0
legacy/gen-hash/package.json
Normal file → Executable file
0
legacy/gen-hash/package.json
Normal file → Executable file
0
legacy/halifax/clean-halifax-actual-2.py
Normal file → Executable file
0
legacy/halifax/clean-halifax-actual-2.py
Normal file → Executable file
0
legacy/halifax/halifax_state.json
Normal file → Executable file
0
legacy/halifax/halifax_state.json
Normal file → Executable file
0
legacy/halifax/new-cleaned.py
Normal file → Executable file
0
legacy/halifax/new-cleaned.py
Normal file → Executable file
0
legacy/insert_services.py
Normal file → Executable file
0
legacy/insert_services.py
Normal file → Executable file
0
legacy/move_rss.py
Normal file → Executable file
0
legacy/move_rss.py
Normal file → Executable file
0
legacy/website_pages.xlsx
Normal file → Executable file
0
legacy/website_pages.xlsx
Normal file → Executable file
Reference in New Issue
Block a user