Cleaning up

search-index.py

@@ -0,0 +1,112 @@
+import json
+import re
+from pathlib import Path
+
+def get_org_id_and_title(file_path):
+    """Extract ID and TITLE from an Org file."""
+    id_pattern = re.compile(r"^:ID:\s*(.+)", re.MULTILINE)
+    title_pattern = re.compile(r"^#\+title:\s*(.+)", re.IGNORECASE | re.MULTILINE)
+
+    content = file_path.read_text(encoding="utf-8", errors="ignore")
+
+    id_match = id_pattern.search(content)
+    title_match = title_pattern.search(content)
+
+    org_id = id_match.group(1).strip() if id_match else None
+    title = title_match.group(1).strip() if title_match else "Untitled"
+
+    return org_id, title
+
+
+def generate_search_index(org_dir, out_file):
+    """Generate search-index.json with title, url, and id."""
+    print("🔍 Generating search-index.json...")
+
+    org_dir = Path(org_dir).expanduser()
+    out_file = Path(out_file).expanduser()
+
+    # Recursively find all .org files (excluding Emacs lockfiles like .#foo.org)
+    org_files = [
+        f for f in org_dir.rglob("*.org")
+        if not re.search(r"/\.#[^/]+\.org$", str(f))
+    ]
+
+    index = []
+    for file_path in org_files:
+        org_id, title = get_org_id_and_title(file_path)
+        if org_id:
+            index.append({
+                "title": title,
+                "url": f"/{file_path.stem}.html",
+                "id": org_id
+            })
+
+    # De-duplicate by ID
+    seen_ids = set()
+    unique_index = []
+    for entry in index:
+        if entry["id"] not in seen_ids:
+            unique_index.append(entry)
+            seen_ids.add(entry["id"])
+
+    # Ensure output directory exists
+    out_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write JSON
+    with open(out_file, "w", encoding="utf-8") as f:
+        json.dump(unique_index, f, ensure_ascii=False, indent=2)
+
+    print(f"✅ search-index.json generated ({len(unique_index)} entries).")
+
+
+def generate_all_files_json(search_index_file, out_file):
+    """Generate all-files.json from search-index.json."""
+    search_index_file = Path(search_index_file).expanduser()
+    out_file = Path(out_file).expanduser()
+
+    # Load data
+    with open(search_index_file, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    print(f"🔎 Loaded {len(data)} entries from {search_index_file}")
+
+    # Sort alphabetically
+    data.sort(key=lambda x: x.get("title", "").lower())
+
+    # Group by first letter
+    grouped = {}
+    for entry in data:
+        title = entry.get("title", "")
+        if not title:
+            continue
+        letter = title[0].upper()
+        grouped.setdefault(letter, []).append({
+            "id": entry["id"],
+            "title": title,
+            "url": entry["url"]
+        })
+
+    # Output structure
+    output = {
+        "title": "All Files",
+        "description": "A complete alphabetical index of all published pages.",
+        "groups": grouped,
+    }
+
+    with open(out_file, "w", encoding="utf-8") as f:
+        json.dump(output, f, ensure_ascii=False, indent=2)
+
+    print(f"✅ all-files.json generated → {out_file}")
+
+
+def main():
+    base_dir = Path.home() / "master-folder" / "org_files" / "org_roam"
+    search_index = base_dir / "output" / "search-index.json"
+    all_files_json = base_dir / "all-files.json"
+
+    generate_search_index(base_dir, search_index)
+    generate_all_files_json(search_index, all_files_json)
+
+
+if __name__ == "__main__":
+    main()