diff --git a/.claude/skills/architecture/SKILL.md b/.claude/skills/architecture/SKILL.md index 67f80b7..c0b05bf 100644 --- a/.claude/skills/architecture/SKILL.md +++ b/.claude/skills/architecture/SKILL.md @@ -62,7 +62,8 @@ The runtime is selected per branch behind the `BuildRunner` interface: `Dispatch ## Watcher -`Watcher` replaces the legacy blocking main loop with a non-blocking fixed-delay poll cycle: fetch origin, enqueue due branches (changed local, recent new origin, due auto-build slots) via `BuildExecutor`, then prune results, artifacts, and stale worktrees. Branches with `branches..requirePullRequest` are enqueued only while their head commit matches a pull-request head, detected without an API token by listing `refs/pull/*/head` via `git ls-remote` (lazily, at most once per poll cycle); manual `build` commands bypass this gate, and `watcher.pullRequestGate: false` disables it globally for plain-git origins without pull-request refs. Nothing is scheduled until `Watcher.start()` is called explicitly (server/watch mode) — CLI commands and tests never start the loop. "Already built" is tracked via the result repository, not by moving local branch refs. Auto-build slot state lives in `.git/gittally/auto-builds.json`; watcher health is exposed via `Watcher.state()`. +`Watcher` replaces the legacy blocking main loop with a non-blocking fixed-delay poll cycle: fetch origin, enqueue due branches (changed local, recent new origin, due auto-build slots) via `BuildExecutor`, then prune results, artifacts, and stale worktrees. Branches with `branches..requirePullRequest` are enqueued only while their head commit matches a pull-request head, detected without an API token by listing `refs/pull/*/head` via `git ls-remote` (lazily, at most once per poll cycle); manual `build` commands bypass this gate, and `watcher.pullRequestGate: false` disables it globally for plain-git origins without pull-request refs. Nothing is scheduled until `Watcher.start()` is called explicitly (server/watch mode) — CLI commands and tests never start the loop. "Already built" is tracked via the result repository, not by moving local branch refs. +After the enqueue decision — and only after it, because a local ref lagging behind origin *is* the change signal — the cycle fast-forwards the primary checkout's local branch refs to their origin counterparts (`watcher.fastForwardLocalRefs`, `GitService.fastForwardLocalBranches`), so build tools reading the shared `.git` from a worktree see the refs they expect; diverged or ahead branches are never touched. Auto-build slot state lives in `.git/gittally/auto-builds.json`; watcher health is exposed via `Watcher.state()`. ## System Metrics diff --git a/docs/configuration.md b/docs/configuration.md index bc118fd..928fb53 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -122,6 +122,10 @@ watcher: # Set false for a plain git origin without pull-request refs (no Gitea/GitHub); # gated branches then build on new commits like any other branch. pullRequestGate: true + # At the end of each poll cycle, fast-forward the primary checkout's local branch refs + # to their origin counterparts (see notes below). Fast-forward only — a diverged or + # ahead local branch is never touched. Set false to leave refs/heads/* alone entirely. + fastForwardLocalRefs: true # Per-branch build configuration. # Use "default" as the fallback for all branches not listed explicitly. @@ -220,6 +224,19 @@ Without the `main` override, direct pushes and merges to `main` would never buil A plain git origin (no Gitea/GitHub) serves no `refs/pull/*/head` at all, so gated branches would never build there. For such origins, disable all gates globally with `watcher.pullRequestGate: false` — typically in the machine-specific `.git/gittally/.gittally.yml`, so the committed configuration keeps the gates for forge-backed environments. +### Notes on `watcher.fastForwardLocalRefs` + +Builds run in worktrees that share the primary checkout's `.git`, so a build tool can read `refs/heads/*` there. +GitTally itself never needs those refs to be current — it builds the commit `refs/remotes/origin/` points at — but build tools do. +A common case is a check that refuses to run when the local main branch differs from its origin counterpart; without this key it would fail on every build once origin moved on, because nothing would ever advance the local ref. + +The fast-forward runs at the end of the poll cycle, after the due branches were enqueued. +That order is required, not cosmetic: a local ref lagging behind origin is exactly how the watcher recognizes new commits, so a ref kept in sync earlier — by this key, a cron job, or a mirroring fetch refspec (`+refs/heads/*:refs/heads/*`) — would silently stop the branch from ever being built. + +Only fast-forwards are applied, as a compare-and-swap against the commit just read. +A local branch that diverged from origin or is ahead of it stays untouched, so local work in the primary checkout is never lost. +The branch checked out in the primary checkout is advanced with `git merge --ff-only`, which refuses to overwrite conflicting uncommitted changes; a refusal is logged and the cycle continues. + ### Notes on `branches..docker` With `docker.enabled`, GitTally shells out to the `docker` CLI; the `docker` command must be on the `PATH`. diff --git a/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt b/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt index a06f72a..326ea13 100644 --- a/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt +++ b/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt @@ -177,6 +177,9 @@ class InitCommand( # honor branches..requirePullRequest; set false for a plain git origin # without pull-request refs (refs/pull/*/head) — gated branches then build on new commits pullRequestGate: true + # after enqueueing, fast-forward the primary checkout's local branch refs to origin, + # so build tools reading the shared .git see the same refs (diverged branches stay untouched) + fastForwardLocalRefs: true # Per-branch build configuration. # Use "default" as the fallback for all branches not listed explicitly. diff --git a/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt b/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt index 61c969e..80e7d11 100644 --- a/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt +++ b/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt @@ -107,6 +107,13 @@ data class WatcherConfig( * `.git/gittally/.gittally.yml` when the committed config enables the gates. */ val pullRequestGate: Boolean = true, + /** + * At the end of each poll cycle, fast-forward the primary checkout's local branch + * refs to their origin counterparts, so build tools reading the shared `.git` see + * the same refs as origin. Fast-forward only: diverged or ahead local branches are + * never touched. Set false to leave the local branch refs alone entirely. + */ + val fastForwardLocalRefs: Boolean = true, ) data class BranchConfig( diff --git a/src/main/kotlin/de/hoennig/gittally/git/GitService.kt b/src/main/kotlin/de/hoennig/gittally/git/GitService.kt index 5bb04ed..165e08a 100644 --- a/src/main/kotlin/de/hoennig/gittally/git/GitService.kt +++ b/src/main/kotlin/de/hoennig/gittally/git/GitService.kt @@ -1,6 +1,7 @@ package de.hoennig.gittally.git import de.hoennig.gittally.config.ConfigLoader +import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.nio.file.Path import java.nio.file.Paths @@ -13,6 +14,8 @@ class GitService( private val runner: GitCommandRunner, private val configLoader: ConfigLoader, ) { + private val log = LoggerFactory.getLogger(GitService::class.java) + fun getTopLevel(workingDir: Path = Paths.get(".")): Path { val result = runner.run(listOf("git", "rev-parse", "--show-toplevel"), workingDir) if (!result.isSuccess) { @@ -149,6 +152,53 @@ class GitService( } } + /** + * Fast-forwards local branch refs to their origin counterparts and returns the + * branches whose ref moved. Build worktrees share the primary checkout's `.git`, + * so tools running inside a build see these refs — checks that compare a local + * branch against its origin counterpart (hs.hsadmin.ng's `prQuickCheck` compares + * `master` with `origin/master`) otherwise fail on every build once origin moves on. + * + * Strictly non-destructive: a local ref that is not an ancestor of its origin + * counterpart (diverged, or ahead) is left alone, the ref update is a + * compare-and-swap against the commit just read, and the branch checked out in + * [workingDir] is advanced with `merge --ff-only`, which refuses to run over + * conflicting uncommitted changes. + * + * Call this only *after* the poll cycle picked its due branches: a local ref lagging + * behind origin is the watcher's change signal ([hasNewCommits], `newOriginBranches`), + * so syncing beforehand would silence the branch instead of building it. + */ + fun fastForwardLocalBranches(workingDir: Path = Paths.get(".")): List { + val originHeads = originBranchHeads(workingDir) + val checkedOut = currentBranch(workingDir) + return localBranches(workingDir).filter { branch -> + val origin = originHeads[branch] ?: return@filter false + val local = localHeadCommit(branch, workingDir) ?: return@filter false + if (local == origin || !isAncestor(local, origin, workingDir)) { + return@filter false + } + // the `refs/` prefix keeps the refname from being read as a git option + val result = + if (branch == checkedOut) { + runner.run(listOf("git", "merge", "--ff-only", "refs/remotes/origin/$branch"), workingDir) + } else { + runner.run(listOf("git", "update-ref", "refs/heads/$branch", origin, local), workingDir) + } + if (!result.isSuccess) { + log.warn("not fast-forwarding local branch {}: {}", branch, result.stderr.trim()) + } + result.isSuccess + } + } + + /** True when [ancestor] is reachable from [descendant]; false for diverged or unrelated commits. */ + private fun isAncestor( + ancestor: String, + descendant: String, + workingDir: Path, + ): Boolean = runner.run(listOf("git", "merge-base", "--is-ancestor", ancestor, descendant), workingDir).isSuccess + fun resetHardToOrigin( branch: String, workingDir: Path = Paths.get("."), diff --git a/src/main/kotlin/de/hoennig/gittally/watcher/Watcher.kt b/src/main/kotlin/de/hoennig/gittally/watcher/Watcher.kt index c92d61a..8584986 100644 --- a/src/main/kotlin/de/hoennig/gittally/watcher/Watcher.kt +++ b/src/main/kotlin/de/hoennig/gittally/watcher/Watcher.kt @@ -27,7 +27,8 @@ import java.util.concurrent.TimeUnit /** * Replaces the legacy blocking main loop: a non-blocking fixed-delay poll cycle that * fetches origin, enqueues due branches via the async [BuildExecutor], and prunes - * retention — it never waits for a build and never touches the primary checkout. + * retention — it never waits for a build and never builds in the primary checkout + * (whose branch refs it does fast-forward, see [fastForwardLocalRefs]). * The loop only runs after an explicit [start] (server/watch mode, step 07); * nothing is scheduled during CLI commands or tests. */ @@ -111,8 +112,9 @@ class Watcher( /** * One poll cycle, never blocking on a build: fetch origin (on failure: log, expose * in [state], retry next cycle), enqueue due branches — changed local branches - * first, then recent new origin branches, then due auto-build slots — and finally - * prune results, artifacts, and worktrees of branches gone from origin. + * first, then recent new origin branches, then due auto-build slots — then + * fast-forward the local branch refs, and finally prune results, artifacts, and + * worktrees of branches gone from origin. */ fun poll(workingDir: Path = Paths.get(".")) { val startedAt = clock.instant() @@ -126,6 +128,9 @@ class Watcher( val config = configLoader.load(workingDir) val originBranches = gitService.originBranches(workingDir) enqueueDueBranches(config, originBranches.toSet(), workingDir) + if (config.watcher.fastForwardLocalRefs) { + fastForwardLocalRefs(workingDir) + } prune(config, originBranches, workingDir) state = state.copy( @@ -149,6 +154,26 @@ class Watcher( } } + /** + * Brings the primary checkout's local branch refs up to origin, because a build worktree + * shares that `.git`: a build tool comparing a local branch with its origin counterpart + * would otherwise see a ref frozen at the state of the last checkout. + * + * Deliberately the last step before pruning — the enqueue decision above reads a + * lagging local ref as "this branch has new commits", so syncing earlier in the cycle + * would suppress the very build it prepares. Never fatal for the poll cycle. + */ + private fun fastForwardLocalRefs(workingDir: Path) { + try { + val moved = gitService.fastForwardLocalBranches(workingDir) + if (moved.isNotEmpty()) { + log.info("fast-forwarded local branch refs to origin: {}", moved.joinToString(", ")) + } + } catch (e: Exception) { + log.warn("fast-forwarding local branch refs failed: {}", e.message) + } + } + private fun enqueueDueBranches( config: GitTallyConfig, originBranches: Set, @@ -171,8 +196,9 @@ class Watcher( /** * Enqueues a build of the branch's origin head unless one is already pending or * running, or that commit was already built. Builds run detached in worktrees and - * never move local branch refs, so "already built" is tracked via the result - * repository, not by resetting the local ref like legacy. A new commit for a + * move no branch ref themselves, so "already built" is tracked via the result + * repository, not by resetting the local ref like legacy; the cycle's + * [fastForwardLocalRefs] runs only after this decision. A new commit for a * branch that is still pending/running waits for a later cycle (queue-behind). * With `requirePullRequest`, the branch head must match a pull-request head on * origin (`refs/pull//head`); manual `build` commands bypass this gate, and diff --git a/src/test/kotlin/de/hoennig/gittally/git/GitServiceTest.kt b/src/test/kotlin/de/hoennig/gittally/git/GitServiceTest.kt index c54401d..b710043 100644 --- a/src/test/kotlin/de/hoennig/gittally/git/GitServiceTest.kt +++ b/src/test/kotlin/de/hoennig/gittally/git/GitServiceTest.kt @@ -4,6 +4,7 @@ import de.hoennig.gittally.config.ConfigLoader import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.booleans.shouldBeFalse import io.kotest.matchers.booleans.shouldBeTrue +import io.kotest.matchers.collections.shouldBeEmpty import io.kotest.matchers.collections.shouldContainExactly import io.kotest.matchers.collections.shouldNotContain import io.kotest.matchers.nulls.shouldBeNull @@ -168,6 +169,56 @@ class GitServiceTest : FunSpec() { service.hasNewCommits("no-such-branch", fixture.work).shouldBeFalse() } + test("fastForwardLocalBranches advances the checked-out branch and its working tree") { + val fixture = Fixture() + fixture.commitFile(fixture.seed, "change.txt", "change") + fixture.git(fixture.seed, "push", "origin", "main") + service.fetchOrigin(fixture.work) + + service.fastForwardLocalBranches(fixture.work) shouldContainExactly listOf("main") + + service.localHeadCommit("main", fixture.work) shouldBe service.originHeadCommit("main", fixture.work) + Files.readString(fixture.work.resolve("change.txt")) shouldBe "change" + service.hasNewCommits("main", fixture.work).shouldBeFalse() + } + + test("fastForwardLocalBranches advances a branch that is not checked out") { + val fixture = Fixture() + fixture.pushNewSeedBranch("feature/x") + service.fetchOrigin(fixture.work) + fixture.git(fixture.work, "branch", "feature/x", "refs/remotes/origin/feature/x") + fixture.git(fixture.seed, "switch", "feature/x") + fixture.commitFile(fixture.seed, "more.txt", "more") + fixture.git(fixture.seed, "push", "origin", "feature/x") + fixture.git(fixture.seed, "switch", "main") + service.fetchOrigin(fixture.work) + + service.fastForwardLocalBranches(fixture.work) shouldContainExactly listOf("feature/x") + + service.localHeadCommit("feature/x", fixture.work) shouldBe + service.originHeadCommit("feature/x", fixture.work) + } + + test("fastForwardLocalBranches leaves a diverged local branch untouched") { + val fixture = Fixture() + fixture.commitFile(fixture.work, "local.txt", "local only") + val localHead = service.localHeadCommit("main", fixture.work) + fixture.commitFile(fixture.seed, "remote.txt", "remote only") + fixture.git(fixture.seed, "push", "origin", "main") + service.fetchOrigin(fixture.work) + + service.fastForwardLocalBranches(fixture.work).shouldBeEmpty() + + service.localHeadCommit("main", fixture.work) shouldBe localHead + } + + test("fastForwardLocalBranches ignores branches without an origin counterpart") { + val fixture = Fixture() + fixture.git(fixture.work, "branch", "local-only") + + service.fastForwardLocalBranches(fixture.work).shouldBeEmpty() + } + test("newOriginBranches lists recent origin-only branches") { val fixture = Fixture() fixture.pushNewSeedBranch("feature/x") diff --git a/src/test/kotlin/de/hoennig/gittally/watcher/WatcherTest.kt b/src/test/kotlin/de/hoennig/gittally/watcher/WatcherTest.kt index e447be2..b56b5e1 100644 --- a/src/test/kotlin/de/hoennig/gittally/watcher/WatcherTest.kt +++ b/src/test/kotlin/de/hoennig/gittally/watcher/WatcherTest.kt @@ -29,6 +29,7 @@ import io.kotest.matchers.string.shouldContain import io.mockk.every import io.mockk.mockk import io.mockk.verify +import io.mockk.verifyOrder import java.nio.file.Files import java.nio.file.Path import java.time.Clock @@ -75,6 +76,7 @@ class WatcherTest : FunSpec() { every { gitService.originHeadCommit(any(), any()) } returns null every { gitService.pullRequestHeads(any()) } returns emptySet() every { gitService.worktreePrune(any()) } returns Unit + every { gitService.fastForwardLocalBranches(any()) } returns emptyList() every { buildExecutor.currentBuilds() } returns emptyList() every { buildExecutor.startBuild(any(), any(), any()) } answers { val branch = firstArg() @@ -177,6 +179,47 @@ class WatcherTest : FunSpec() { listOf("main" to "commit-main", "feature/new" to "commit-feature") } + test("poll fast-forwards local branch refs only after the enqueue decision was made") { + val harness = Harness() + every { harness.gitService.originBranches(any()) } returns listOf("main") + every { harness.gitService.localBranches(any()) } returns listOf("main") + every { harness.gitService.hasNewCommits("main", any()) } returns true + every { harness.gitService.originHeadCommit("main", any()) } returns "commit-main" + every { harness.gitService.fastForwardLocalBranches(any()) } returns listOf("main") + + harness.watcher.poll(harness.workingDir) + + // syncing the ref before the decision would hide the very commit being enqueued here + harness.startedBuilds shouldContainExactly listOf("main" to "commit-main") + verifyOrder { + harness.gitService.hasNewCommits("main", any()) + harness.gitService.fastForwardLocalBranches(any()) + } + } + + test("a failing fast-forward does not abort the poll cycle") { + val harness = Harness() + every { harness.gitService.originBranches(any()) } returns listOf("main") + every { harness.gitService.fastForwardLocalBranches(any()) } throws RuntimeException("ref locked") + + harness.watcher.poll(harness.workingDir) + + harness.watcher + .state() + .lastPollError + .shouldBeNull() + verify { harness.artifactStore.prune(any()) } + } + + test("poll leaves local branch refs alone when fastForwardLocalRefs is disabled") { + val harness = Harness(GitTallyConfig(watcher = WatcherConfig(fastForwardLocalRefs = false))) + every { harness.gitService.originBranches(any()) } returns listOf("main") + + harness.watcher.poll(harness.workingDir) + + verify(exactly = 0) { harness.gitService.fastForwardLocalBranches(any()) } + } + test("poll skips a branch whose build is already pending or running") { val harness = Harness() harness.seed("main", BuildStatus.PENDING, commit = "commit-old")