feat: parallel git command handling

This commit is contained in:
saberzero1
2026-04-03 14:39:30 +02:00
parent 52e4257bda
commit 2a6117142c

View File

@@ -44,6 +44,26 @@ function cloneWithSubdir({ url, ref, subdir, pluginDir }) {
}
}
async function cloneWithSubdirAsync({ url, ref, subdir, pluginDir }) {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "quartz-plugin-"))
try {
if (ref) {
await execAsync(`git clone --depth 1 --branch ${ref} "${url}" "${tmpDir}"`)
} else {
await execAsync(`git clone --depth 1 "${url}" "${tmpDir}"`)
}
const subdirPath = path.join(tmpDir, subdir)
if (!fs.existsSync(subdirPath)) {
throw new Error(`Subdirectory "${subdir}" not found in cloned repository`)
}
fs.cpSync(subdirPath, pluginDir, { recursive: true })
const { stdout } = await execAsync("git rev-parse HEAD", { cwd: tmpDir })
return stdout.trim()
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true })
}
}
function buildPlugin(pluginDir, name) {
try {
const skipBuild = !needsBuild(pluginDir)
@@ -492,6 +512,8 @@ export async function handlePluginInstallUnified({
let failed = 0
let lockfileChanged = false
// Handle existing dirs and local symlinks (fast), collect remote clones
const remoteEntries = []
for (const entry of missing) {
try {
const { name, url, ref, local, subdir } = parseGitSource(entry.source)
@@ -549,46 +571,8 @@ export async function handlePluginInstallUnified({
installed.push({ name, pluginDir })
lockfileChanged = true
console.log(styleText("green", `✓ Linked ${name} (local)`))
} else if (subdir) {
console.log(styleText("cyan", `→ Cloning ${name} from ${url} (subdir: ${subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const commit = cloneWithSubdir({ url, ref, subdir, pluginDir })
lockfile.plugins[name] = {
source: entry.source,
resolved: url,
commit,
...(ref && { ref }),
subdir,
installedAt: new Date().toISOString(),
}
installed.push({ name, pluginDir })
lockfileChanged = true
console.log(
styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`),
)
} else {
console.log(styleText("cyan", `→ Cloning ${name} from ${url}...`))
if (ref) {
execSync(`git clone --depth 1 --branch ${ref} "${url}" "${pluginDir}"`, {
stdio: "ignore",
})
} else {
execSync(`git clone --depth 1 "${url}" "${pluginDir}"`, { stdio: "ignore" })
}
const commit = getGitCommit(pluginDir)
lockfile.plugins[name] = {
source: entry.source,
resolved: url,
commit,
...(ref && { ref }),
installedAt: new Date().toISOString(),
}
installed.push({ name, pluginDir })
lockfileChanged = true
console.log(styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)}`))
remoteEntries.push({ entry, name, url, ref, subdir, pluginDir })
}
} catch (error) {
console.log(styleText("red", `✗ Failed to resolve ${formatSource(entry.source)}: ${error}`))
@@ -596,6 +580,61 @@ export async function handlePluginInstallUnified({
}
}
// Clone remote plugins in parallel
if (remoteEntries.length > 0) {
const concurrency = Math.max(1, os.cpus().length)
await runParallel(
remoteEntries,
concurrency,
async ({ entry, name, url, ref, subdir, pluginDir }) => {
try {
if (subdir) {
console.log(styleText("cyan", `→ Cloning ${name} from ${url} (subdir: ${subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const commit = await cloneWithSubdirAsync({ url, ref, subdir, pluginDir })
lockfile.plugins[name] = {
source: entry.source,
resolved: url,
commit,
...(ref && { ref }),
subdir,
installedAt: new Date().toISOString(),
}
installed.push({ name, pluginDir })
lockfileChanged = true
console.log(
styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`),
)
} else {
console.log(styleText("cyan", `→ Cloning ${name} from ${url}...`))
const branchArg = ref ? ` --branch ${ref}` : ""
await execAsync(`git clone --depth 1${branchArg} "${url}" "${pluginDir}"`)
const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir })
const commit = stdout.trim()
lockfile.plugins[name] = {
source: entry.source,
resolved: url,
commit,
...(ref && { ref }),
installedAt: new Date().toISOString(),
}
installed.push({ name, pluginDir })
lockfileChanged = true
console.log(styleText("green", `✓ Cloned ${name}@${commit.slice(0, 7)}`))
}
} catch (error) {
console.log(
styleText("red", `✗ Failed to resolve ${formatSource(entry.source)}: ${error}`),
)
failed++
}
},
)
}
if (installed.length > 0) {
console.log()
console.log(styleText("cyan", "→ Building plugins..."))
@@ -692,6 +731,8 @@ export async function handlePluginInstallUnified({
nameFilter ? nameFilter.has(name) : true,
)
// Handle local symlinks and collect remote plugins to clone
const remotePlugins = []
for (const [name, entry] of entries) {
const pluginDir = path.join(PLUGINS_DIR, name)
@@ -719,36 +760,47 @@ export async function handlePluginInstallUnified({
continue
}
try {
if (entry.subdir) {
console.log(
styleText(
"cyan",
`${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)} (subdir: ${entry.subdir})...`,
),
)
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
cloneWithSubdir({ url: entry.resolved, ref: entry.ref, subdir: entry.subdir, pluginDir })
} else {
console.log(
styleText(
"cyan",
`${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)}...`,
),
)
const branchArg = entry.ref ? ` --branch ${entry.ref}` : ""
execSync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`, {
stdio: "ignore",
})
execSync(`git checkout ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" })
remotePlugins.push({ name, entry, pluginDir })
}
// Clone remote plugins in parallel
if (remotePlugins.length > 0) {
const concurrency = Math.max(1, os.cpus().length)
await runParallel(remotePlugins, concurrency, async ({ name, entry, pluginDir }) => {
try {
if (entry.subdir) {
console.log(
styleText(
"cyan",
`${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)} (subdir: ${entry.subdir})...`,
),
)
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
await cloneWithSubdirAsync({
url: entry.resolved,
ref: entry.ref,
subdir: entry.subdir,
pluginDir,
})
} else {
console.log(
styleText(
"cyan",
`${name}: cloning ${entry.resolved}@${entry.commit.slice(0, 7)}...`,
),
)
const branchArg = entry.ref ? ` --branch ${entry.ref}` : ""
await execAsync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`)
await execAsync(`git checkout ${entry.commit}`, { cwd: pluginDir })
}
console.log(styleText("green", `${name} restored`))
restoredPlugins.push({ name, pluginDir })
installed++
} catch {
console.log(styleText("red", `${name}: failed to restore`))
failed++
}
console.log(styleText("green", `${name} restored`))
restoredPlugins.push({ name, pluginDir })
installed++
} catch {
console.log(styleText("red", `${name}: failed to restore`))
failed++
}
})
}
if (restoredPlugins.length > 0) {
@@ -787,6 +839,8 @@ export async function handlePluginInstallUnified({
const updatedPlugins = []
let lockfileChanged = false
// Phase 1: Validate and categorize plugins (fast, sequential)
const validPlugins = []
for (const name of pluginsToUpdate) {
const entry = lockfile.plugins[name]
if (!entry) {
@@ -808,59 +862,68 @@ export async function handlePluginInstallUnified({
continue
}
try {
console.log(styleText("cyan", `→ Updating ${name}...`))
if (entry.subdir) {
fs.rmSync(pluginDir, { recursive: true })
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const newCommit = cloneWithSubdir({
url: entry.resolved,
ref: entry.ref,
subdir: entry.subdir,
pluginDir,
})
if (needsBuild(pluginDir)) {
updatedPlugins.push({ name, pluginDir })
}
if (newCommit !== entry.commit) {
entry.commit = newCommit
entry.installedAt = new Date().toISOString()
lockfileChanged = true
console.log(
styleText(
"green",
`✓ Updated ${name} to ${newCommit.slice(0, 7)} (subdir: ${entry.subdir})`,
),
)
} else {
console.log(styleText("gray", `${name} rebuilt (subdir: ${entry.subdir})`))
}
} else {
const fetchRef = entry.ref || ""
const resetTarget = entry.ref ? `origin/${entry.ref}` : "origin/HEAD"
execSync(`git fetch --depth 1 origin${fetchRef ? " " + fetchRef : ""}`, {
cwd: pluginDir,
stdio: "ignore",
})
execSync(`git reset --hard ${resetTarget}`, { cwd: pluginDir, stdio: "ignore" })
const newCommit = getGitCommit(pluginDir)
if (newCommit !== entry.commit) {
entry.commit = newCommit
entry.installedAt = new Date().toISOString()
updatedPlugins.push({ name, pluginDir })
lockfileChanged = true
console.log(styleText("green", `✓ Updated ${name} to ${newCommit.slice(0, 7)}`))
} else {
console.log(styleText("gray", `${name} already up to date`))
}
}
} catch (error) {
console.log(styleText("red", `✗ Failed to update ${name}: ${error}`))
}
validPlugins.push({ name, pluginDir, entry })
}
// Phase 2: Fetch/update plugins in parallel
if (validPlugins.length > 0) {
const concurrency = Math.max(1, os.cpus().length)
await runParallel(validPlugins, concurrency, async ({ name, pluginDir, entry }) => {
try {
console.log(styleText("cyan", `→ Updating ${name}...`))
if (entry.subdir) {
fs.rmSync(pluginDir, { recursive: true })
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const newCommit = await cloneWithSubdirAsync({
url: entry.resolved,
ref: entry.ref,
subdir: entry.subdir,
pluginDir,
})
if (needsBuild(pluginDir)) {
updatedPlugins.push({ name, pluginDir })
}
if (newCommit !== entry.commit) {
entry.commit = newCommit
entry.installedAt = new Date().toISOString()
lockfileChanged = true
console.log(
styleText(
"green",
`✓ Updated ${name} to ${newCommit.slice(0, 7)} (subdir: ${entry.subdir})`,
),
)
} else {
console.log(styleText("gray", `${name} rebuilt (subdir: ${entry.subdir})`))
}
} else {
const fetchRef = entry.ref || ""
const resetTarget = entry.ref ? `origin/${entry.ref}` : "origin/HEAD"
await execAsync(`git fetch --depth 1 origin${fetchRef ? " " + fetchRef : ""}`, {
cwd: pluginDir,
})
await execAsync(`git reset --hard ${resetTarget}`, { cwd: pluginDir })
const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir })
const newCommit = stdout.trim()
if (newCommit !== entry.commit) {
entry.commit = newCommit
entry.installedAt = new Date().toISOString()
updatedPlugins.push({ name, pluginDir })
lockfileChanged = true
console.log(styleText("green", `✓ Updated ${name} to ${newCommit.slice(0, 7)}`))
} else {
console.log(styleText("gray", `${name} already up to date`))
}
}
} catch (error) {
console.log(styleText("red", `✗ Failed to update ${name}: ${error}`))
}
})
}
// Phase 3: Build updated plugins in parallel
if (updatedPlugins.length > 0) {
console.log()
console.log(styleText("cyan", "→ Rebuilding updated plugins..."))
@@ -898,6 +961,8 @@ export async function handlePluginInstallUnified({
let failed = 0
const pluginsToBuild = []
// Handle local plugins and collect entries needing git operations
const gitEntries = []
for (const [name, entry] of entries) {
const pluginDir = path.join(PLUGINS_DIR, name)
@@ -931,70 +996,79 @@ export async function handlePluginInstallUnified({
}
if (fs.existsSync(pluginDir)) {
try {
if (entry.subdir) {
if (!needsBuild(pluginDir)) {
console.log(
styleText(
"gray",
`${name}@${entry.commit.slice(0, 7)} already installed (subdir)`,
),
)
installed++
continue
}
} else {
const currentCommit = getGitCommit(pluginDir)
if (currentCommit === entry.commit && !needsBuild(pluginDir)) {
console.log(
styleText("gray", `${name}@${entry.commit.slice(0, 7)} already installed`),
)
installed++
continue
}
if (currentCommit !== entry.commit) {
console.log(
styleText("cyan", `${name}: updating to ${entry.commit.slice(0, 7)}...`),
)
const fetchRef = entry.ref ? ` ${entry.ref}` : ""
execSync(`git fetch --depth 1 origin${fetchRef}`, { cwd: pluginDir, stdio: "ignore" })
execSync(`git reset --hard ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" })
}
if (entry.subdir) {
if (!needsBuild(pluginDir)) {
console.log(
styleText("gray", `${name}@${entry.commit.slice(0, 7)} already installed (subdir)`),
)
installed++
continue
}
pluginsToBuild.push({ name, pluginDir })
installed++
} catch {
console.log(styleText("red", `${name}: failed to update`))
failed++
} else {
const currentCommit = getGitCommit(pluginDir)
if (currentCommit === entry.commit && !needsBuild(pluginDir)) {
console.log(
styleText("gray", `${name}@${entry.commit.slice(0, 7)} already installed`),
)
installed++
continue
}
if (currentCommit !== entry.commit) {
gitEntries.push({ name, entry, pluginDir, action: "update" })
} else {
pluginsToBuild.push({ name, pluginDir })
installed++
}
}
} else {
gitEntries.push({ name, entry, pluginDir, action: "clone" })
}
}
// Run git fetch/clone operations in parallel
if (gitEntries.length > 0) {
const concurrency = Math.max(1, os.cpus().length)
await runParallel(gitEntries, concurrency, async ({ name, entry, pluginDir, action }) => {
try {
if (entry.subdir) {
console.log(styleText("cyan", `${name}: cloning (subdir: ${entry.subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
cloneWithSubdir({ url: entry.resolved, ref: entry.ref, subdir: entry.subdir, pluginDir })
if (action === "update") {
console.log(styleText("cyan", `${name}: updating to ${entry.commit.slice(0, 7)}...`))
const fetchRef = entry.ref ? ` ${entry.ref}` : ""
await execAsync(`git fetch --depth 1 origin${fetchRef}`, { cwd: pluginDir })
await execAsync(`git reset --hard ${entry.commit}`, { cwd: pluginDir })
pluginsToBuild.push({ name, pluginDir })
installed++
} else {
console.log(styleText("cyan", `${name}: cloning...`))
const branchArg = entry.ref ? ` --branch ${entry.ref}` : ""
execSync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`, {
stdio: "ignore",
})
if (entry.commit !== "unknown") {
execSync(`git fetch --depth 1 origin ${entry.commit}`, {
cwd: pluginDir,
stdio: "ignore",
if (entry.subdir) {
console.log(styleText("cyan", `${name}: cloning (subdir: ${entry.subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
await cloneWithSubdirAsync({
url: entry.resolved,
ref: entry.ref,
subdir: entry.subdir,
pluginDir,
})
execSync(`git checkout ${entry.commit}`, { cwd: pluginDir, stdio: "ignore" })
} else {
console.log(styleText("cyan", `${name}: cloning...`))
const branchArg = entry.ref ? ` --branch ${entry.ref}` : ""
await execAsync(`git clone --depth 1${branchArg} "${entry.resolved}" "${pluginDir}"`)
if (entry.commit !== "unknown") {
await execAsync(`git fetch --depth 1 origin ${entry.commit}`, { cwd: pluginDir })
await execAsync(`git checkout ${entry.commit}`, { cwd: pluginDir })
}
}
console.log(styleText("green", `${name}@${entry.commit.slice(0, 7)}`))
pluginsToBuild.push({ name, pluginDir })
installed++
}
console.log(styleText("green", `${name}@${entry.commit.slice(0, 7)}`))
pluginsToBuild.push({ name, pluginDir })
installed++
} catch {
console.log(styleText("red", `${name}: failed to clone`))
console.log(
styleText("red", `${name}: failed to ${action === "update" ? "update" : "clone"}`),
)
failed++
}
}
})
}
if (pluginsToBuild.length > 0) {
@@ -1052,6 +1126,8 @@ export async function handlePluginAdd(
const addedPlugins = []
// Handle local plugins and collect remote sources to clone
const remoteSources = []
for (const source of sources) {
try {
const parsed = parseGitSource(source)
@@ -1093,48 +1169,64 @@ export async function handlePluginAdd(
}
addedPlugins.push({ name, pluginDir, source, configSource })
console.log(styleText("green", `✓ Added ${name} (local symlink)`))
} else if (subdir) {
console.log(styleText("cyan", `→ Adding ${name} from ${url} (subdir: ${subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const commit = cloneWithSubdir({ url, ref, subdir, pluginDir })
lockfile.plugins[name] = {
source,
resolved: url,
commit,
...(ref && { ref }),
subdir,
installedAt: new Date().toISOString(),
}
addedPlugins.push({ name, pluginDir, source, configSource })
console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`))
} else {
console.log(styleText("cyan", `→ Adding ${name} from ${url}...`))
if (ref) {
execSync(`git clone --depth 1 --branch ${ref} "${url}" "${pluginDir}"`, {
stdio: "ignore",
})
} else {
execSync(`git clone --depth 1 "${url}" "${pluginDir}"`, { stdio: "ignore" })
}
const commit = getGitCommit(pluginDir)
lockfile.plugins[name] = {
source,
resolved: url,
commit,
...(ref && { ref }),
installedAt: new Date().toISOString(),
}
addedPlugins.push({ name, pluginDir, source, configSource })
console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)}`))
remoteSources.push({ source, name, url, ref, subdir, pluginDir, configSource })
}
} catch (error) {
console.log(styleText("red", `✗ Failed to add ${formatSource(source)}: ${error}`))
}
}
// Clone remote plugins in parallel
if (remoteSources.length > 0) {
const concurrency = Math.max(1, os.cpus().length)
await runParallel(
remoteSources,
concurrency,
async ({ source, name, url, ref, subdir, pluginDir, configSource }) => {
try {
if (subdir) {
console.log(styleText("cyan", `→ Adding ${name} from ${url} (subdir: ${subdir})...`))
fs.mkdirSync(path.dirname(pluginDir), { recursive: true })
const commit = await cloneWithSubdirAsync({ url, ref, subdir, pluginDir })
lockfile.plugins[name] = {
source,
resolved: url,
commit,
...(ref && { ref }),
subdir,
installedAt: new Date().toISOString(),
}
addedPlugins.push({ name, pluginDir, source, configSource })
console.log(
styleText("green", `✓ Added ${name}@${commit.slice(0, 7)} (subdir: ${subdir})`),
)
} else {
console.log(styleText("cyan", `→ Adding ${name} from ${url}...`))
const branchArg = ref ? ` --branch ${ref}` : ""
await execAsync(`git clone --depth 1${branchArg} "${url}" "${pluginDir}"`)
const { stdout } = await execAsync("git rev-parse HEAD", { cwd: pluginDir })
const commit = stdout.trim()
lockfile.plugins[name] = {
source,
resolved: url,
commit,
...(ref && { ref }),
installedAt: new Date().toISOString(),
}
addedPlugins.push({ name, pluginDir, source, configSource })
console.log(styleText("green", `✓ Added ${name}@${commit.slice(0, 7)}`))
}
} catch (error) {
console.log(styleText("red", `✗ Failed to add ${formatSource(source)}: ${error}`))
}
},
)
}
if (addedPlugins.length > 0) {
console.log()
console.log(styleText("cyan", "→ Building plugins..."))