org_roam/search-index.py

import json
import re
from pathlib import Path

# ================================
# REGEX PATTERNS
# ================================

ID_PATTERN = re.compile(r"^:ID:\s*(.+)", re.MULTILINE)
TITLE_PATTERN = re.compile(r"^#\+title:\s*(.+)", re.IGNORECASE | re.MULTILINE)

# Org links:
# [[id:...]]
# [[file:...]]
# [[./...]]
LINK_PATTERN = re.compile(r"\[\[(.*?)\]\]")


# ================================
# PARSING FUNCTIONS
# ================================

def parse_org_file(file_path):
    """Extract ID, title, and links from an Org file."""
    content = file_path.read_text(encoding="utf-8", errors="ignore")

    id_match = ID_PATTERN.search(content)
    title_match = TITLE_PATTERN.search(content)

    org_id = id_match.group(1).strip() if id_match else None
    title = title_match.group(1).strip() if title_match else "Untitled"

    links = extract_links(content)

    return org_id, title, links


def extract_links(content):
    """Extract all internal Org links."""
    raw_links = LINK_PATTERN.findall(content)

    links = []

    for link in raw_links:
        # Remove description part: [[link][desc]]
        if "][" in link:
            link = link.split("][")[0]

        # ID links
        if link.startswith("id:"):
            links.append(link.replace("id:", "").strip())

        # File links
        elif link.startswith("file:"):
            path = link.replace("file:", "").strip()
            links.append(path)

        # Relative links
        elif link.endswith(".org"):
            links.append(link.strip())

    return links


# ================================
# MAIN GENERATOR
# ================================

def generate_search_index(org_dir, out_file):
    print("Generating search-index.json with links...")

    org_dir = Path(org_dir).expanduser()
    out_file = Path(out_file).expanduser()

    org_files = [
        f for f in org_dir.rglob("*.org")
        if not re.search(r"/\.#[^/]+\.org$", str(f))
    ]

    # First pass: collect all files + IDs
    file_map = {}  # id -> url
    path_map = {}  # relative path -> url
    temp_entries = []

    for file_path in org_files:
        org_id, title, links = parse_org_file(file_path)
        relative = file_path.relative_to(org_dir).with_suffix('')
        url = f"/{relative}.html"

        if org_id:
            file_map[org_id] = url

        path_map[str(relative) + ".org"] = url

        temp_entries.append({
            "title": title,
            "url": url,
            "id": org_id,
            "raw_links": links
        })

    # Second pass: resolve links → URLs
    index = []

    for entry in temp_entries:
        resolved_links = []

        for link in entry["raw_links"]:
            # ID link
            if link in file_map:
                resolved_links.append(file_map[link])

            # File link
            elif link in path_map:
                resolved_links.append(path_map[link])

        index.append({
            "title": entry["title"],
            "url": entry["url"],
            "id": entry["id"],
            "links": list(set(resolved_links))  # remove duplicates
        })

    # De-duplicate by ID
    seen_ids = set()
    unique_index = []
    for entry in index:
        if entry["id"] and entry["id"] not in seen_ids:
            unique_index.append(entry)
            seen_ids.add(entry["id"])

    out_file.parent.mkdir(parents=True, exist_ok=True)

    with open(out_file, "w", encoding="utf-8") as f:
        json.dump(unique_index, f, ensure_ascii=False, indent=2)

    print(f"search-index.json generated ({len(unique_index)} entries).")


# ================================
# ALL FILES JSON (UNCHANGED)
# ================================

def generate_all_files_json(search_index_file, out_file):
    search_index_file = Path(search_index_file).expanduser()
    out_file = Path(out_file).expanduser()

    with open(search_index_file, "r", encoding="utf-8") as f:
        data = json.load(f)

    print(f"Loaded {len(data)} entries from {search_index_file}")

    data.sort(key=lambda x: x.get("title", "").lower())

    grouped = {}
    for entry in data:
        title = entry.get("title", "")
        if not title:
            continue
        letter = title[0].upper()
        grouped.setdefault(letter, []).append({
            "id": entry["id"],
            "title": title,
            "url": entry["url"]
        })

    output = {
        "title": "All Files",
        "description": "A complete alphabetical index of all published pages.",
        "groups": grouped,
    }

    with open(out_file, "w", encoding="utf-8") as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    print(f"all-files.json generated → {out_file}")


# ================================
# ENTRY POINT
# ================================

def main():
    base_dir = Path.home() / "master-folder" / "org_files" / "org_roam"
    search_index = base_dir / "output" / "search-index.json"
    all_files_json = base_dir / "all-files.json"

    generate_search_index(base_dir, search_index)
    generate_all_files_json(search_index, all_files_json)


if __name__ == "__main__":
    main()