Make Resource Loader server-backed and drop local metadata cache

2026-08-20 14:19:00 +01:00
parent 7a60d33610
commit 4653a2caae
8 changed files with 78 additions and 58 deletions
--- a/README.md
+++ b/README.md
@@ -281,7 +281,7 @@ These APIs are not implemented here. Production routing must serve `output/` and

 The hidden-details feature uses browser `localStorage`/`sessionStorage` for visit and discovery state and may call Open-Meteo after explicit geolocation interaction. Its friendly content source is `assets/content/hidden-details.json`; `assets/scripts/features/hidden-details.js` is generated by the external hidden-narrative authoring workflow. Treat both files as a pair and do not hand-edit only the generated JavaScript.

-The Resource Loader at `/rl/` loads and saves metadata through the same-origin `/api/resource-loader` backend, with ETags protecting concurrent edits. Generated cover thumbnails use `/api/resource-loader/thumbnails/:id`. IndexedDB under `zxh-resource-loader` remains an offline cache and retains imported PDF binaries only on the device where they were added. JSON exports contain metadata and reading sessions, not PDF binaries. Its page code lives in `assets/scripts/pages/resource-loader.js`; the separately testable schema/progress/sync operations live in `resource-loader-model.js` and run with `make test-js`.
+The Resource Loader at `/rl/` uses the same-origin `/api/resource-loader` backend as its sole metadata source, with ETags protecting concurrent edits. There is no browser metadata fallback. Generated cover thumbnails use `/api/resource-loader/thumbnails/:id` and are removed from local storage after upload. IndexedDB under `zxh-resource-loader` retains only imported PDF binaries on the device where they were added. JSON exports contain metadata and reading sessions, not PDF binaries. Its page code lives in `assets/scripts/pages/resource-loader.js`; the separately testable schema/progress/sync operations live in `resource-loader-model.js` and run with `make test-js`.

 ## Authoring service

--- a/assets/scripts/pages/resource-loader.js
+++ b/assets/scripts/pages/resource-loader.js
@@ -6,8 +6,7 @@
  if (!root || !Model) return;

  const DB_NAME = "zxh-resource-loader";
-  const DB_VERSION = 1;
-  const META_STORE = "metadata";
+  const DB_VERSION = 2;
  const ATTACHMENT_STORE = "attachments";
  const API_URL = "/api/resource-loader";
  const $ = (selector) => document.querySelector(selector);
@@ -67,7 +66,7 @@
      const request = indexedDB.open(DB_NAME, DB_VERSION);
      request.onupgradeneeded = () => {
        const db = request.result;
-        if (!db.objectStoreNames.contains(META_STORE)) db.createObjectStore(META_STORE);
+        if (db.objectStoreNames.contains("metadata")) db.deleteObjectStore("metadata");
        if (!db.objectStoreNames.contains(ATTACHMENT_STORE)) db.createObjectStore(ATTACHMENT_STORE, { keyPath: "id" });
      };
      request.onsuccess = () => resolve(request.result);
@@ -86,11 +85,21 @@
    });
  }

-  function getMetadata(key) { return request(META_STORE, "readonly", (store) => store.get(key)); }
-  function setMetadata(key, value) { return request(META_STORE, "readwrite", (store) => store.put(value, key)); }
  function getAttachment(id) { return id ? request(ATTACHMENT_STORE, "readonly", (store) => store.get(id)) : Promise.resolve(null); }
  function putAttachment(record) { return request(ATTACHMENT_STORE, "readwrite", (store) => store.put(record)); }
  function deleteAttachment(id) { return id ? request(ATTACHMENT_STORE, "readwrite", (store) => store.delete(id)) : Promise.resolve(); }
+  function deleteStoredThumbnails() {
+    return request(ATTACHMENT_STORE, "readwrite", (store) => {
+      const cursorRequest = store.openCursor();
+      cursorRequest.onsuccess = () => {
+        const cursor = cursorRequest.result;
+        if (!cursor) return;
+        if (cursor.value && cursor.value.kind === "thumbnail") cursor.delete();
+        cursor.continue();
+      };
+      return cursorRequest;
+    });
+  }

  function setSyncStatus(status, label) {
    if (!els.syncStatus) return;
@@ -120,8 +129,10 @@
        body: attachment.blob,
      });
      if (!response.ok) throw new Error(`Thumbnail upload failed (${response.status}).`);
+      await deleteAttachment(id);
      state.uploadedThumbnails.add(id);
    }));
+    await deleteStoredThumbnails();
  }

  async function pushServerLibrary(library, allowMerge) {
@@ -138,7 +149,6 @@
      const remote = await fetchServerLibrary();
      const merged = Model.mergeForSync(remote, library);
      state.library = merged;
-      await setMetadata("library", merged);
      if (state.eventsBound) render();
      toast("Changes from another device were merged.");
      return pushServerLibrary(merged, false);
@@ -150,35 +160,34 @@
  }

  async function saveLibrary() {
-    await setMetadata("library", Model.normaliseLibrary(state.library));
    state.syncQueue = state.syncQueue.catch(() => {}).then(async () => {
      try {
        await pushServerLibrary(Model.normaliseLibrary(state.library), true);
+        return true;
      } catch (error) {
-        console.warn("Resource Loader server sync is unavailable.", error);
-        setSyncStatus("offline", "offline · saved here");
+        console.error("Resource Loader could not save to the server.", error);
+        setSyncStatus("error", "server unavailable");
+        toast("The change was not saved. Reloading the shared server library.", true);
+        try {
+          const remote = await fetchServerLibrary();
+          if (remote) {
+            state.library = remote;
+            if (state.eventsBound) render();
+          }
+        } catch (reloadError) {
+          console.warn("The authoritative server library could not be reloaded.", reloadError);
+        }
+        return false;
      }
    });
    return state.syncQueue;
  }

  async function loadLibrary() {
-    const saved = await getMetadata("library");
-    try {
-      const remote = await fetchServerLibrary();
-      if (remote) {
-        setSyncStatus("synced", "synced to server");
-        await setMetadata("library", remote);
-        return remote;
-      }
-    } catch (error) {
-      console.warn("Resource Loader server is unavailable; using this browser's cache.", error);
-      setSyncStatus("offline", "offline · using cache");
-    }
-    if (saved) return Model.normaliseLibrary(saved);
-    const response = await fetch("/assets/content/resource-loader.json", { cache: "no-store" });
-    if (response.ok) return Model.normaliseLibrary(await response.json());
-    return Model.createSeed();
+    const remote = await fetchServerLibrary();
+    if (!remote) throw new Error("The server Resource Loader library has not been initialised.");
+    setSyncStatus("synced", "synced to server");
+    return remote;
  }

  function currentVault() {
@@ -297,7 +306,7 @@
    let blob = attachment && attachment.blob;
    if (!blob) {
      try {
-        const response = await fetch(`${API_URL}/thumbnails/${encodeURIComponent(id)}`, { credentials: "same-origin" });
+        const response = await fetch(`${API_URL}/thumbnails/${encodeURIComponent(id)}`, { cache: "no-store", credentials: "same-origin" });
        if (response.ok) blob = await response.blob();
      } catch (error) {
        console.warn(`Thumbnail ${id} is unavailable.`, error);
@@ -438,7 +447,7 @@
      if (!confirm("Delete this session?")) return;
      resource.sessions = resource.sessions.filter((item) => item.id !== button.dataset.deleteSession);
      resource.updatedAt = Model.now();
-      await saveLibrary();
+      if (!await saveLibrary()) return;
      renderSessions(resource);
      render();
      toast("Session deleted.");
@@ -512,7 +521,7 @@
    if (existing) vault.resources[vault.resources.findIndex((item) => item.id === existing.id)] = resource;
    else vault.resources.push(resource);
    vault.updatedAt = Model.now();
-    await saveLibrary();
+    if (!await saveLibrary()) return;
    state.pendingImport = null;
    closeDialog(els.resourceDialog);
    render();
@@ -522,10 +531,11 @@
  async function removeResource() {
    const resource = currentResource($("#rl-resource-id").value);
    if (!resource || !confirm(`Delete “${resource.title}”? This also removes its retained PDF.`)) return;
-    await Promise.all([deleteAttachment(resource.attachmentId), deleteAttachment(resource.thumbnailId)]);
+    const attachmentIds = [resource.attachmentId, resource.thumbnailId];
    currentVault().resources = currentVault().resources.filter((item) => item.id !== resource.id);
    currentVault().updatedAt = Model.now();
-    await saveLibrary();
+    if (!await saveLibrary()) return;
+    await Promise.all(attachmentIds.map(deleteAttachment));
    closeDialog(els.resourceDialog);
    render();
    toast("Resource deleted.");
@@ -557,7 +567,7 @@
    resource.updatedAt = Model.now();
    if (Model.progressFor(resource).percent >= 100) resource.status = "completed";
    else if (["backlog", "shelf"].includes(resource.status)) resource.status = "active";
-    await saveLibrary();
+    if (!await saveLibrary()) return;
    closeDialog(els.sessionDialog);
    renderSessions(resource);
    render();
@@ -610,7 +620,7 @@
      const vault = state.library.vaults.find((item) => item.id === action.split(":")[1]);
      vault.colour = $("#rl-name-colour").value;
    }
-    await saveLibrary();
+    if (!await saveLibrary()) return;
    closeDialog(els.nameDialog);
    render();
    renderVaultManager();
@@ -630,7 +640,7 @@
    const folder = currentVault().folders.find((item) => item.id === folderId);
    if (!folder || !confirm(`Delete “${folder.name}”? Its resources will move to Unfiled.`)) return;
    state.library = Model.deleteFolder(state.library, state.vaultId, folderId);
-    await saveLibrary();
+    if (!await saveLibrary()) return;
    closeDialog(els.nameDialog);
    render();
    toast("Folder deleted; resources moved to Unfiled.");
@@ -657,11 +667,12 @@
    $("#rl-vault-list").querySelectorAll("[data-delete-vault]").forEach((button) => button.addEventListener("click", async () => {
      const vault = state.library.vaults.find((item) => item.id === button.dataset.deleteVault);
      if (!confirm(`Delete the “${vault.name}” vault and all of its records? Retained PDFs will also be removed.`)) return;
-      await Promise.all(vault.resources.flatMap((resource) => [deleteAttachment(resource.attachmentId), deleteAttachment(resource.thumbnailId)]));
+      const attachmentIds = vault.resources.flatMap((resource) => [resource.attachmentId, resource.thumbnailId]);
      state.library = Model.deleteVault(state.library, vault.id);
      state.vaultId = state.library.vaults[0].id;
      state.folderId = "all";
-      await saveLibrary();
+      if (!await saveLibrary()) return;
+      await Promise.all(attachmentIds.map(deleteAttachment));
      renderVaultManager(); render(); toast("Vault deleted.");
    }));
  }
@@ -697,11 +708,11 @@
  async function applyImport(mode) {
    if (!state.importData) return;
    if (mode === "replace" && !confirm("Replace every vault and resource? Retained PDF files will also be cleared.")) return;
-    if (mode === "replace") await request(ATTACHMENT_STORE, "readwrite", (store) => store.clear());
    state.library = mode === "merge" ? Model.mergeLibraries(state.library, state.importData) : state.importData;
    state.vaultId = state.library.vaults[0] && state.library.vaults[0].id;
    state.folderId = "all";
-    await saveLibrary();
+    if (!await saveLibrary()) return;
+    if (mode === "replace") await request(ATTACHMENT_STORE, "readwrite", (store) => store.clear());
    state.importData = null;
    closeDialog(els.importDialog);
    render();
@@ -805,7 +816,7 @@
  }

  async function clearData() {
-    if (!confirm("Clear this browser's offline cache and retained PDFs? Server metadata and thumbnails will remain available.")) return;
+    if (!confirm("Remove retained PDFs from this browser? Shared server metadata and thumbnails will remain available.")) return;
    closeDialog(els.storageDialog);
    state.db.close();
    await new Promise((resolve, reject) => {
@@ -851,7 +862,6 @@
      state.pdfUrls.forEach((url) => URL.revokeObjectURL(url));
      state.thumbnailUrls.forEach((url) => URL.revokeObjectURL(url));
    });
-    window.addEventListener("online", () => saveLibrary());
  }

  async function initialise() {
@@ -859,14 +869,17 @@
    try {
      state.db = await openDatabase();
      state.library = await loadLibrary();
-      if (!state.library.vaults.length) state.library = Model.createSeed();
      state.vaultId = state.library.vaults[0].id;
-      await saveLibrary();
      bindEvents();
      render();
+      uploadLocalThumbnails(state.library).catch((error) => {
+        console.warn("Some generated covers could not be uploaded yet.", error);
+        toast("The library is synced, but a generated cover could not be uploaded.", true);
+      });
    } catch (error) {
      console.error(error);
-      root.innerHTML = `<div class="rl-empty"><span class="rl-empty__mark">!</span><h3>Library storage is unavailable</h3><p>${escapeHtml(error.message || "This browser could not open IndexedDB.")}</p></div>`;
+      setSyncStatus("error", "server unavailable");
+      root.innerHTML = `<div class="rl-empty"><span class="rl-empty__mark">!</span><h3>Server library is unavailable</h3><p>${escapeHtml(error.message || "The shared Resource Loader library could not be loaded.")}</p></div>`;
    } finally {
      hideLoading();
    }
--- a/assets/styles/pages/resource-loader.css
+++ b/assets/styles/pages/resource-loader.css
@@ -160,6 +160,7 @@
 .rl-summary__privacy span { color: #4a8a5b; font-size: .8rem; }
 .rl-summary__privacy[data-state="syncing"] span { color: #a87831; }
 .rl-summary__privacy[data-state="offline"] span { color: #a13f35; }
+.rl-summary__privacy[data-state="error"] span { color: #a13f35; }

 .rl-command-bar {
  display: flex;
--- a/build-logs/gitea-build-monitor.log
+++ b/build-logs/gitea-build-monitor.log
@@ -944,3 +944,9 @@ at <ScriptBlock>, /home/zaine/master-folder/org-platform/org_web/build-logs/gite
 2026-08-20T11:34:33.3081787+01:00 [INFO] Running authoring server tests with /home/zaine/master-folder/org-platform/authoring-service/.venv/bin/python.
 2026-08-20T11:34:33.4006115+01:00 [INFO] Authoring server tests passed: ran=0, failures=0, errors=0, skipped=0, exit=0
 2026-08-20T11:34:33.7249113+01:00 [INFO] Sent authoring server test notification.
+2026-08-20T14:04:36.9758174+01:00 [INFO] Prepared current build status for zaine/org_web: severity=Info, status=success
+2026-08-20T14:04:36.9831625+01:00 [INFO] Fetching recent Gitea Actions runs for zaine/org_web.
+2026-08-20T14:04:37.8280029+01:00 [INFO] Sent build status notification with 1 embed(s).
+2026-08-20T14:04:37.8512952+01:00 [INFO] Running authoring server tests with /home/zaine/master-folder/org-platform/authoring-service/.venv/bin/python.
+2026-08-20T14:04:37.9343738+01:00 [INFO] Authoring server tests passed: ran=0, failures=0, errors=0, skipped=0, exit=0
+2026-08-20T14:04:38.2713260+01:00 [INFO] Sent authoring server test notification.
--- a/posts/posts-list.org
+++ b/posts/posts-list.org
@@ -4,7 +4,7 @@
 See the categories: @@html:<a href="../home/categories.html">Categories</a>@@

 * Posts:
- [[file:career/career-list.org][Career List]] @@html:<span class="post-date">20-08-2026 11:34</span>@@ 
+- [[file:career/career-list.org][Career List]] @@html:<span class="post-date">20-08-2026 14:04</span>@@ 
 - [[file:career/cross-site-scripting-xss.org][Cross-Site Scripting (XSS)]] @@html:<span class="post-date">01-06-2026 10:21</span>@@ @@html:<a href="/tags/learning.html"><span class="post-tag">learning</span></a>@@ @@html:<a href="/tags/notes.html"><span class="post-tag">notes</span></a>@@
 - [[file:career/ha-dr.org][High Availability, Disaster Recovery and Business Continuity]] @@html:<span class="post-date">11-03-2026 17:18</span>@@ @@html:<a href="/tags/learning.html"><span class="post-tag">learning</span></a>@@ @@html:<a href="/tags/notes.html"><span class="post-tag">notes</span></a>@@
 - [[file:career/javascript.org][Understands the Javascript language]] @@html:<span class="post-date">11-03-2026 16:52</span>@@ @@html:<a href="/tags/learning.html"><span class="post-tag">learning</span></a>@@ @@html:<a href="/tags/notes.html"><span class="post-tag">notes</span></a>@@
--- a/rl/index.org
+++ b/rl/index.org
@@ -165,8 +165,8 @@
 <dialog class="rl-dialog" id="rl-storage-dialog">
  <div class="rl-dialog__shell rl-dialog__shell--small">
    <header class="rl-dialog__head"><div><p class="rl-eyebrow">This browser</p><h2>Local storage</h2></div><button type="button" class="rl-close" data-close-dialog aria-label="Close">×</button></header>
-    <div class="rl-dialog__body"><p id="rl-storage-summary">Calculating storage use…</p><p class="rl-help">Metadata and thumbnails sync to the server. PDFs remain only in the browser where they were imported.</p><button type="button" id="rl-request-persistence">Request persistent storage</button></div>
-    <footer class="rl-dialog__foot"><button type="button" class="rl-danger" id="rl-clear-data">Clear this browser's cache &amp; PDFs</button><button type="button" data-close-dialog>Close</button></footer>
+    <div class="rl-dialog__body"><p id="rl-storage-summary">Calculating storage use…</p><p class="rl-help">Metadata and thumbnails are read from the shared server library. Only PDF binaries remain in the browser where they were imported.</p><button type="button" id="rl-request-persistence">Request persistent storage</button></div>
+    <footer class="rl-dialog__foot"><button type="button" class="rl-danger" id="rl-clear-data">Clear PDFs from this browser</button><button type="button" data-close-dialog>Close</button></footer>
  </div>
 </dialog>

--- a/rl/resource-loader-2026-08-20.json
+++ b/rl/resource-loader-2026-08-20.json
--- a/sitemap.org
+++ b/sitemap.org
@@ -80,17 +80,17 @@ flowchart TD
  n30 --> n37
  n38["Tag: life"]
  n30 --> n38
-  n39["Tag: update"]
+  n39["Tag: education"]
  n30 --> n39
-  n40["Tag: education"]
+  n40["Tag: update"]
  n30 --> n40
  n41["Tag: insights"]
  n30 --> n41
-  n42["Tag: reading"]
+  n42["Tag: emacs"]
  n30 --> n42
-  n43["Tag: emacs"]
+  n43["Tag: maths"]
  n30 --> n43
-  n44["Tag: maths"]
+  n44["Tag: reading"]
  n30 --> n44
  n45{{"posts"}}
  root --> n45
@@ -212,12 +212,12 @@ flowchart TD
  click n36 "tags/review.html" "Tag: review"
  click n37 "tags/website.html" "Tag: website"
  click n38 "tags/life.html" "Tag: life"
-  click n39 "tags/update.html" "Tag: update"
-  click n40 "tags/education.html" "Tag: education"
+  click n39 "tags/education.html" "Tag: education"
+  click n40 "tags/update.html" "Tag: update"
  click n41 "tags/insights.html" "Tag: insights"
-  click n42 "tags/reading.html" "Tag: reading"
-  click n43 "tags/emacs.html" "Tag: emacs"
-  click n44 "tags/maths.html" "Tag: maths"
+  click n42 "tags/emacs.html" "Tag: emacs"
+  click n43 "tags/maths.html" "Tag: maths"
+  click n44 "tags/reading.html" "Tag: reading"
  click n46 "posts/posts-intro.html" "Posts Introduction"
  click n47 "posts/posts-list.html" "Posts List"
  click n49 "posts/career/solid-principles.html" "SOLID Principles"
@@ -302,12 +302,12 @@ flowchart TD
  - [[file:tags/review.org][Tag: review]]
  - [[file:tags/website.org][Tag: website]]
  - [[file:tags/life.org][Tag: life]]
-  - [[file:tags/update.org][Tag: update]]
  - [[file:tags/education.org][Tag: education]]
+  - [[file:tags/update.org][Tag: update]]
  - [[file:tags/insights.org][Tag: insights]]
-  - [[file:tags/reading.org][Tag: reading]]
  - [[file:tags/emacs.org][Tag: emacs]]
  - [[file:tags/maths.org][Tag: maths]]
+  - [[file:tags/reading.org][Tag: reading]]
 - posts
  - [[file:posts/posts-intro.org][Posts Introduction]]
  - [[file:posts/posts-list.org][Posts List]]