From 2a6117142c79f622e5ddb48d4f113b1905dd6e0b Mon Sep 17 00:00:00 2001 From: saberzero1 Date: Fri, 3 Apr 2026 14:39:30 +0200 Subject: [PATCH] feat: parallel git command handling --- quartz/cli/plugin-git-handlers.js | 502 ++++++++++++++++++------------ 1 file changed, 297 insertions(+), 205 deletions(-) diff --git a/quartz/cli/plugin-git-handlers.js b/quartz/cli/plugin-git-handlers.js index f1a16f8..a62d0cd 100644 --- a/quartz/cli/plugin-git-handlers.js +++ b/quartz/cli/plugin-git-handlers.js @@ -44,6 +44,26 @@ function cloneWithSubdir({ url, ref, subdir, pluginDir }) { } } +async function cloneWithSubdirAsync({ url, ref, subdir, pluginDir }) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "quartz-plugin-")) + try { + if (ref) { + await execAsync(`git clone --depth 1 --branch ${ref} "${url}" "${tmpDir}"`) + } else { + await execAsync(`git clone --depth 1 "${url}" "${tmpDir}"`) + } + const subdirPath = path.join(tmpDir, subdir) + if (!fs.existsSync(subdirPath)) { + throw new Error(`Subdirectory "${subdir}" not found in cloned repository`) + } + fs.cpSync(subdirPath, pluginDir, { recursive: true }) + const { stdout } = await execAsync("git rev-parse HEAD", { cwd: tmpDir }) + return stdout.trim() + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } +} + function buildPlugin(pluginDir, name) { try { const skipBuild = !needsBuild(pluginDir) @@ -492,6 +512,8 @@ export async function handlePluginInstallUnified({ let failed = 0 let lockfileChanged = false + // Handle existing dirs and local symlinks (fast), collect remote clones + const remoteEntries = [] for (const entry of missing) { try { const { name, url, ref, local, subdir } = parseGitSource(entry.source) @@ -549,46 +571,8 @@ export async function handlePluginInstallUnified({ installed.push({ name, pluginDir }) lockfileChanged = true console.log(styleText("green", `✓ Linked ${name} (local)`)) - } else if (subdir) { - console.log(styleText("cyan", `→ Cloning ${name} from ${url} (subdir: ${subdir})...`)) - fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) - const commit = cloneWithSubdir({ url, ref, subdir, pluginDir }) - lockfile.plugins[name] = { - source: entry.source, - resolved: url, - commit, - ...(ref && { ref }), - subdir, - installedAt: new Date().toISOString(), - } - installed.push({ name, pluginDir }) - lockfileChanged = true - console.log( - styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`), - ) } else { - console.log(styleText("cyan", `→ Cloning ${name} from ${url}...`)) - - if (ref) { - execSync(`git clone --depth 1 --branch ${ref} "${url}" "${pluginDir}"`, { - stdio: "ignore", - }) - } else { - execSync(`git clone --depth 1 "${url}" "${pluginDir}"`, { stdio: "ignore" }) - } - - const commit = getGitCommit(pluginDir) - lockfile.plugins[name] = { - source: entry.source, - resolved: url, - commit, - ...(ref && { ref }), - installedAt: new Date().toISOString(), - } - - installed.push({ name, pluginDir }) - lockfileChanged = true - console.log(styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)}`)) + remoteEntries.push({ entry, name, url, ref, subdir, pluginDir }) } } catch (error) { console.log(styleText("red", `✗ Failed to resolve ${formatSource(entry.source)}: ${error}`)) @@ -596,6 +580,61 @@ export async function handlePluginInstallUnified({ } } + // Clone remote plugins in parallel + if (remoteEntries.length > 0) { + const concurrency = Math.max(1, os.cpus().length) + await runParallel( + remoteEntries, + concurrency, + async ({ entry, name, url, ref, subdir, pluginDir }) => { + try { + if (subdir) { + console.log(styleText("cyan", `→ Cloning ${name} from ${url} (subdir: ${subdir})...`)) + fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) + const commit = await cloneWithSubdirAsync({ url, ref, subdir, pluginDir }) + lockfile.plugins[name] = { + source: entry.source, + resolved: url, + commit, + ...(ref && { ref }), + subdir, + installedAt: new Date().toISOString(), + } + installed.push({ name, pluginDir }) + lockfileChanged = true + console.log( + styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`), + ) + } else { + console.log(styleText("cyan", `→ Cloning ${name} from ${url}...`)) + + const branchArg = ref ? ` --branch ${ref}` : "" + await execAsync(`git clone --depth 1${branchArg} "${url}" "${pluginDir}"`) + + const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir }) + const commit = stdout.trim() + lockfile.plugins[name] = { + source: entry.source, + resolved: url, + commit, + ...(ref && { ref }), + installedAt: new Date().toISOString(), + } + + installed.push({ name, pluginDir }) + lockfileChanged = true + console.log(styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)}`)) + } + } catch (error) { + console.log( + styleText("red", `✗ Failed to resolve ${formatSource(entry.source)}: ${error}`), + ) + failed++ + } + }, + ) + } + if (installed.length > 0) { console.log() console.log(styleText("cyan", "→ Building plugins...")) @@ -692,6 +731,8 @@ export async function handlePluginInstallUnified({ nameFilter ? nameFilter.has(name) : true, ) + // Handle local symlinks and collect remote plugins to clone + const remotePlugins = [] for (const [name, entry] of entries) { const pluginDir = path.join(PLUGINS_DIR, name) @@ -719,36 +760,47 @@ export async function handlePluginInstallUnified({ continue } - try { - if (entry.subdir) { - console.log( - styleText( - "cyan", - `→ ${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)} (subdir: ${entry.subdir})...`, - ), - ) - fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) - cloneWithSubdir({ url: entry.resolved, ref: entry.ref, subdir: entry.subdir, pluginDir }) - } else { - console.log( - styleText( - "cyan", - `→ ${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)}...`, - ), - ) - const branchArg = entry.ref ? ` --branch ${entry.ref}` : "" - execSync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`, { - stdio: "ignore", - }) - execSync(`git checkout ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" }) + remotePlugins.push({ name, entry, pluginDir }) + } + + // Clone remote plugins in parallel + if (remotePlugins.length > 0) { + const concurrency = Math.max(1, os.cpus().length) + await runParallel(remotePlugins, concurrency, async ({ name, entry, pluginDir }) => { + try { + if (entry.subdir) { + console.log( + styleText( + "cyan", + `→ ${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)} (subdir: ${entry.subdir})...`, + ), + ) + fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) + await cloneWithSubdirAsync({ + url: entry.resolved, + ref: entry.ref, + subdir: entry.subdir, + pluginDir, + }) + } else { + console.log( + styleText( + "cyan", + `→ ${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)}...`, + ), + ) + const branchArg = entry.ref ? ` --branch ${entry.ref}` : "" + await execAsync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`) + await execAsync(`git checkout ${entry.commit}`, { cwd: pluginDir }) + } + console.log(styleText("green", `✓ ${name} restored`)) + restoredPlugins.push({ name, pluginDir }) + installed++ + } catch { + console.log(styleText("red", `✗ ${name}: failed to restore`)) + failed++ } - console.log(styleText("green", `✓ ${name} restored`)) - restoredPlugins.push({ name, pluginDir }) - installed++ - } catch { - console.log(styleText("red", `✗ ${name}: failed to restore`)) - failed++ - } + }) } if (restoredPlugins.length > 0) { @@ -787,6 +839,8 @@ export async function handlePluginInstallUnified({ const updatedPlugins = [] let lockfileChanged = false + // Phase 1: Validate and categorize plugins (fast, sequential) + const validPlugins = [] for (const name of pluginsToUpdate) { const entry = lockfile.plugins[name] if (!entry) { @@ -808,59 +862,68 @@ export async function handlePluginInstallUnified({ continue } - try { - console.log(styleText("cyan", `→ Updating ${name}...`)) - - if (entry.subdir) { - fs.rmSync(pluginDir, { recursive: true }) - fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) - const newCommit = cloneWithSubdir({ - url: entry.resolved, - ref: entry.ref, - subdir: entry.subdir, - pluginDir, - }) - if (needsBuild(pluginDir)) { - updatedPlugins.push({ name, pluginDir }) - } - if (newCommit !== entry.commit) { - entry.commit = newCommit - entry.installedAt = new Date().toISOString() - lockfileChanged = true - console.log( - styleText( - "green", - `✓ Updated ${name} to ${newCommit.slice(0, 7)} (subdir: ${entry.subdir})`, - ), - ) - } else { - console.log(styleText("gray", `✓ ${name} rebuilt (subdir: ${entry.subdir})`)) - } - } else { - const fetchRef = entry.ref || "" - const resetTarget = entry.ref ? `origin/${entry.ref}` : "origin/HEAD" - execSync(`git fetch --depth 1 origin${fetchRef ? " " + fetchRef : ""}`, { - cwd: pluginDir, - stdio: "ignore", - }) - execSync(`git reset --hard ${resetTarget}`, { cwd: pluginDir, stdio: "ignore" }) - - const newCommit = getGitCommit(pluginDir) - if (newCommit !== entry.commit) { - entry.commit = newCommit - entry.installedAt = new Date().toISOString() - updatedPlugins.push({ name, pluginDir }) - lockfileChanged = true - console.log(styleText("green", `✓ Updated ${name} to ${newCommit.slice(0, 7)}`)) - } else { - console.log(styleText("gray", `✓ ${name} already up to date`)) - } - } - } catch (error) { - console.log(styleText("red", `✗ Failed to update ${name}: ${error}`)) - } + validPlugins.push({ name, pluginDir, entry }) } + // Phase 2: Fetch/update plugins in parallel + if (validPlugins.length > 0) { + const concurrency = Math.max(1, os.cpus().length) + await runParallel(validPlugins, concurrency, async ({ name, pluginDir, entry }) => { + try { + console.log(styleText("cyan", `→ Updating ${name}...`)) + + if (entry.subdir) { + fs.rmSync(pluginDir, { recursive: true }) + fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) + const newCommit = await cloneWithSubdirAsync({ + url: entry.resolved, + ref: entry.ref, + subdir: entry.subdir, + pluginDir, + }) + if (needsBuild(pluginDir)) { + updatedPlugins.push({ name, pluginDir }) + } + if (newCommit !== entry.commit) { + entry.commit = newCommit + entry.installedAt = new Date().toISOString() + lockfileChanged = true + console.log( + styleText( + "green", + `✓ Updated ${name} to ${newCommit.slice(0, 7)} (subdir: ${entry.subdir})`, + ), + ) + } else { + console.log(styleText("gray", `✓ ${name} rebuilt (subdir: ${entry.subdir})`)) + } + } else { + const fetchRef = entry.ref || "" + const resetTarget = entry.ref ? `origin/${entry.ref}` : "origin/HEAD" + await execAsync(`git fetch --depth 1 origin${fetchRef ? " " + fetchRef : ""}`, { + cwd: pluginDir, + }) + await execAsync(`git reset --hard ${resetTarget}`, { cwd: pluginDir }) + + const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir }) + const newCommit = stdout.trim() + if (newCommit !== entry.commit) { + entry.commit = newCommit + entry.installedAt = new Date().toISOString() + updatedPlugins.push({ name, pluginDir }) + lockfileChanged = true + console.log(styleText("green", `✓ Updated ${name} to ${newCommit.slice(0, 7)}`)) + } else { + console.log(styleText("gray", `✓ ${name} already up to date`)) + } + } + } catch (error) { + console.log(styleText("red", `✗ Failed to update ${name}: ${error}`)) + } + }) + } + + // Phase 3: Build updated plugins in parallel if (updatedPlugins.length > 0) { console.log() console.log(styleText("cyan", "→ Rebuilding updated plugins...")) @@ -898,6 +961,8 @@ export async function handlePluginInstallUnified({ let failed = 0 const pluginsToBuild = [] + // Handle local plugins and collect entries needing git operations + const gitEntries = [] for (const [name, entry] of entries) { const pluginDir = path.join(PLUGINS_DIR, name) @@ -931,70 +996,79 @@ export async function handlePluginInstallUnified({ } if (fs.existsSync(pluginDir)) { - try { - if (entry.subdir) { - if (!needsBuild(pluginDir)) { - console.log( - styleText( - "gray", - ` ✓ ${name}@${entry.commit.slice(0, 7)} already installed (subdir)`, - ), - ) - installed++ - continue - } - } else { - const currentCommit = getGitCommit(pluginDir) - if (currentCommit === entry.commit && !needsBuild(pluginDir)) { - console.log( - styleText("gray", ` ✓ ${name}@${entry.commit.slice(0, 7)} already installed`), - ) - installed++ - continue - } - if (currentCommit !== entry.commit) { - console.log( - styleText("cyan", ` → ${name}: updating to ${entry.commit.slice(0, 7)}...`), - ) - const fetchRef = entry.ref ? ` ${entry.ref}` : "" - execSync(`git fetch --depth 1 origin${fetchRef}`, { cwd: pluginDir, stdio: "ignore" }) - execSync(`git reset --hard ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" }) - } + if (entry.subdir) { + if (!needsBuild(pluginDir)) { + console.log( + styleText("gray", ` ✓ ${name}@${entry.commit.slice(0, 7)} already installed (subdir)`), + ) + installed++ + continue } pluginsToBuild.push({ name, pluginDir }) installed++ - } catch { - console.log(styleText("red", ` ✗ ${name}: failed to update`)) - failed++ + } else { + const currentCommit = getGitCommit(pluginDir) + if (currentCommit === entry.commit && !needsBuild(pluginDir)) { + console.log( + styleText("gray", ` ✓ ${name}@${entry.commit.slice(0, 7)} already installed`), + ) + installed++ + continue + } + if (currentCommit !== entry.commit) { + gitEntries.push({ name, entry, pluginDir, action: "update" }) + } else { + pluginsToBuild.push({ name, pluginDir }) + installed++ + } } } else { + gitEntries.push({ name, entry, pluginDir, action: "clone" }) + } + } + + // Run git fetch/clone operations in parallel + if (gitEntries.length > 0) { + const concurrency = Math.max(1, os.cpus().length) + await runParallel(gitEntries, concurrency, async ({ name, entry, pluginDir, action }) => { try { - if (entry.subdir) { - console.log(styleText("cyan", ` → ${name}: cloning (subdir: ${entry.subdir})...`)) - fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) - cloneWithSubdir({ url: entry.resolved, ref: entry.ref, subdir: entry.subdir, pluginDir }) + if (action === "update") { + console.log(styleText("cyan", ` → ${name}: updating to ${entry.commit.slice(0, 7)}...`)) + const fetchRef = entry.ref ? ` ${entry.ref}` : "" + await execAsync(`git fetch --depth 1 origin${fetchRef}`, { cwd: pluginDir }) + await execAsync(`git reset --hard ${entry.commit}`, { cwd: pluginDir }) + pluginsToBuild.push({ name, pluginDir }) + installed++ } else { - console.log(styleText("cyan", ` → ${name}: cloning...`)) - const branchArg = entry.ref ? ` --branch ${entry.ref}` : "" - execSync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`, { - stdio: "ignore", - }) - if (entry.commit !== "unknown") { - execSync(`git fetch --depth 1 origin ${entry.commit}`, { - cwd: pluginDir, - stdio: "ignore", + if (entry.subdir) { + console.log(styleText("cyan", ` → ${name}: cloning (subdir: ${entry.subdir})...`)) + fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) + await cloneWithSubdirAsync({ + url: entry.resolved, + ref: entry.ref, + subdir: entry.subdir, + pluginDir, }) - execSync(`git checkout ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" }) + } else { + console.log(styleText("cyan", ` → ${name}: cloning...`)) + const branchArg = entry.ref ? ` --branch ${entry.ref}` : "" + await execAsync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`) + if (entry.commit !== "unknown") { + await execAsync(`git fetch --depth 1 origin ${entry.commit}`, { cwd: pluginDir }) + await execAsync(`git checkout ${entry.commit}`, { cwd: pluginDir }) + } } + console.log(styleText("green", ` ✓ ${name}@${entry.commit.slice(0, 7)}`)) + pluginsToBuild.push({ name, pluginDir }) + installed++ } - console.log(styleText("green", ` ✓ ${name}@${entry.commit.slice(0, 7)}`)) - pluginsToBuild.push({ name, pluginDir }) - installed++ } catch { - console.log(styleText("red", ` ✗ ${name}: failed to clone`)) + console.log( + styleText("red", ` ✗ ${name}: failed to ${action === "update" ? "update" : "clone"}`), + ) failed++ } - } + }) } if (pluginsToBuild.length > 0) { @@ -1052,6 +1126,8 @@ export async function handlePluginAdd( const addedPlugins = [] + // Handle local plugins and collect remote sources to clone + const remoteSources = [] for (const source of sources) { try { const parsed = parseGitSource(source) @@ -1093,48 +1169,64 @@ export async function handlePluginAdd( } addedPlugins.push({ name, pluginDir, source, configSource }) console.log(styleText("green", `✓ Added ${name} (local symlink)`)) - } else if (subdir) { - console.log(styleText("cyan", `→ Adding ${name} from ${url} (subdir: ${subdir})...`)) - fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) - const commit = cloneWithSubdir({ url, ref, subdir, pluginDir }) - lockfile.plugins[name] = { - source, - resolved: url, - commit, - ...(ref && { ref }), - subdir, - installedAt: new Date().toISOString(), - } - addedPlugins.push({ name, pluginDir, source, configSource }) - console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`)) } else { - console.log(styleText("cyan", `→ Adding ${name} from ${url}...`)) - - if (ref) { - execSync(`git clone --depth 1 --branch ${ref} "${url}" "${pluginDir}"`, { - stdio: "ignore", - }) - } else { - execSync(`git clone --depth 1 "${url}" "${pluginDir}"`, { stdio: "ignore" }) - } - - const commit = getGitCommit(pluginDir) - lockfile.plugins[name] = { - source, - resolved: url, - commit, - ...(ref && { ref }), - installedAt: new Date().toISOString(), - } - - addedPlugins.push({ name, pluginDir, source, configSource }) - console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)}`)) + remoteSources.push({ source, name, url, ref, subdir, pluginDir, configSource }) } } catch (error) { console.log(styleText("red", `✗ Failed to add ${formatSource(source)}: ${error}`)) } } + // Clone remote plugins in parallel + if (remoteSources.length > 0) { + const concurrency = Math.max(1, os.cpus().length) + await runParallel( + remoteSources, + concurrency, + async ({ source, name, url, ref, subdir, pluginDir, configSource }) => { + try { + if (subdir) { + console.log(styleText("cyan", `→ Adding ${name} from ${url} (subdir: ${subdir})...`)) + fs.mkdirSync(path.dirname(pluginDir), { recursive: true }) + const commit = await cloneWithSubdirAsync({ url, ref, subdir, pluginDir }) + lockfile.plugins[name] = { + source, + resolved: url, + commit, + ...(ref && { ref }), + subdir, + installedAt: new Date().toISOString(), + } + addedPlugins.push({ name, pluginDir, source, configSource }) + console.log( + styleText("green", `✓ Added ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`), + ) + } else { + console.log(styleText("cyan", `→ Adding ${name} from ${url}...`)) + + const branchArg = ref ? ` --branch ${ref}` : "" + await execAsync(`git clone --depth 1${branchArg} "${url}" "${pluginDir}"`) + + const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir }) + const commit = stdout.trim() + lockfile.plugins[name] = { + source, + resolved: url, + commit, + ...(ref && { ref }), + installedAt: new Date().toISOString(), + } + + addedPlugins.push({ name, pluginDir, source, configSource }) + console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)}`)) + } + } catch (error) { + console.log(styleText("red", `✗ Failed to add ${formatSource(source)}: ${error}`)) + } + }, + ) + } + if (addedPlugins.length > 0) { console.log() console.log(styleText("cyan", "→ Building plugins..."))