Run follow-up builds after a green predecessor (PR#23, point 3)

FollowUpTrigger listens to build status transitions and, on a SUCCESS,
enqueues every definition of that branch whose afterSuccessOf names the
finished build — at the finished build's commit, not the branch's
current head, so a deployment ships the commit that was tested. Every
green run triggers, a repeated one again. The trigger is armed by
Watcher.start() and disarmed by stop(), so a CLI build never enqueues
into a JVM about to exit; the CLI names the follow-ups the server
would run instead. The event now carries the repository the result
belongs to, which the listener has to act on.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
mhoennig
2026-09-04 18:19:10 +02:00
co-authored by Claude Fable 5.1
parent ab522bc68d
commit bbb4969fb9
9 changed files with 416 additions and 5 deletions
@@ -119,7 +119,7 @@ class BuildExecutor(
artifactKey = runningBuild.artifactKey, artifactKey = runningBuild.artifactKey,
) )
repo.results.append(pending) repo.results.append(pending)
eventPublisher.publishEvent(BuildStatusChangedEvent(pending)) eventPublisher.publishEvent(BuildStatusChangedEvent(pending, repo))
val activeBuild = ActiveBuild(runningBuild, repo) val activeBuild = ActiveBuild(runningBuild, repo)
builds[runningBuild.artifactKey] = activeBuild builds[runningBuild.artifactKey] = activeBuild
publishGiteaStatus(activeBuild, BuildStatus.PENDING, duration = null) publishGiteaStatus(activeBuild, BuildStatus.PENDING, duration = null)
@@ -384,7 +384,7 @@ class BuildExecutor(
duration = duration, duration = duration,
artifactKey = runningBuild.artifactKey, artifactKey = runningBuild.artifactKey,
).also { build.repo.results.append(it) } ).also { build.repo.results.append(it) }
eventPublisher.publishEvent(BuildStatusChangedEvent(updated)) eventPublisher.publishEvent(BuildStatusChangedEvent(updated, build.repo))
publishGiteaStatus(build, status, duration) publishGiteaStatus(build, status, duration)
return updated return updated
} }
@@ -35,7 +35,12 @@ data class RunningBuild(
var runningSince: Instant? = null var runningSince: Instant? = null
} }
/** Published via Spring's `ApplicationEventPublisher` on every persisted status transition. */ /**
* Published via Spring's `ApplicationEventPublisher` on every persisted status transition.
* Carries the repository because a [BuildResult] does not: a listener that reacts to the
* transition — the follow-up trigger — has to act on that repository.
*/
data class BuildStatusChangedEvent( data class BuildStatusChangedEvent(
val result: BuildResult, val result: BuildResult,
val repo: RepoContext,
) )
@@ -1,9 +1,11 @@
package de.hoennig.werkator.commands package de.hoennig.werkator.commands
import de.hoennig.werkator.build.BuildStatus import de.hoennig.werkator.build.BuildStatus
import de.hoennig.werkator.config.BuildDefinition
import de.hoennig.werkator.git.GitService import de.hoennig.werkator.git.GitService
import de.hoennig.werkator.repo.RepoContext import de.hoennig.werkator.repo.RepoContext
import de.hoennig.werkator.repo.RepoRegistry import de.hoennig.werkator.repo.RepoRegistry
import de.hoennig.werkator.watcher.FollowUpTrigger
import org.springframework.stereotype.Component import org.springframework.stereotype.Component
import picocli.CommandLine.Command import picocli.CommandLine.Command
import picocli.CommandLine.ExitCode import picocli.CommandLine.ExitCode
@@ -27,6 +29,7 @@ class BuildCommand(
private val gitService: GitService, private val gitService: GitService,
private val consoleBuildRunner: ConsoleBuildRunner, private val consoleBuildRunner: ConsoleBuildRunner,
private val registry: RepoRegistry, private val registry: RepoRegistry,
private val followUpTrigger: FollowUpTrigger,
) : Callable<Int> { ) : Callable<Int> {
@Mixin @Mixin
var repoOption = RepoOption() var repoOption = RepoOption()
@@ -58,9 +61,33 @@ class BuildCommand(
} }
println("building branch $branch at commit ${commit.take(12)}") println("building branch $branch at commit ${commit.take(12)}")
val status = consoleBuildRunner.buildAndStream(repo, branch, commit) val status = consoleBuildRunner.buildAndStream(repo, branch, commit)
if (status == BuildStatus.SUCCESS) {
reportSkippedFollowUps(branch, commit)
}
return if (status == BuildStatus.SUCCESS) ExitCode.OK else ExitCode.SOFTWARE return if (status == BuildStatus.SUCCESS) ExitCode.OK else ExitCode.SOFTWARE
} }
/**
* A one-shot build ends with its process, so it never runs the follow-up builds the
* server would enqueue after a green run (PR#23) — it says which ones instead of
* leaving the operator to wonder why nothing was deployed.
*/
private fun reportSkippedFollowUps(
branch: String,
commit: String,
) {
val followUps =
try {
followUpTrigger.followUpsOf(repo, branch, commit, BuildDefinition.DEFAULT)
} catch (e: Exception) {
System.err.println("warning: could not determine the follow-up builds (${e.message})")
return
}
if (followUps.isNotEmpty()) {
println("note: the server would now run the follow-up build(s) ${followUps.joinToString(", ")}; a CLI build does not")
}
}
/** A one-shot build should still work offline, from the last fetched origin state. */ /** A one-shot build should still work offline, from the last fetched origin state. */
private fun fetchBestEffort() { private fun fetchBestEffort() {
try { try {
@@ -0,0 +1,134 @@
package de.hoennig.werkator.watcher
import de.hoennig.werkator.build.BuildExecutor
import de.hoennig.werkator.build.BuildResult
import de.hoennig.werkator.build.BuildStatus
import de.hoennig.werkator.build.BuildStatusChangedEvent
import de.hoennig.werkator.config.BuildDefinition
import de.hoennig.werkator.config.ConfigFiles
import de.hoennig.werkator.config.ConfigLoader
import de.hoennig.werkator.git.GitService
import de.hoennig.werkator.repo.RepoContext
import org.slf4j.LoggerFactory
import org.springframework.context.event.EventListener
import org.springframework.stereotype.Component
import java.time.Clock
import java.util.concurrent.atomic.AtomicBoolean
/**
* Runs the follow-up builds (PR#23): whenever a build ends with `SUCCESS`, every
* definition of that branch whose `trigger.afterSuccessOf` names the finished build —
* and whose selector selects the branch — is enqueued on the same branch at the *same
* commit*, never at the branch's current origin head, so a deployment always ships the
* commit that was tested. Every green run counts, whatever started it, and a repeated
* green run of the same commit triggers again: a run of a green build that is silently
* not deployed would be more confusing than a redundant deployment.
*
* The definitions are resolved with the branch's committed config at the finished
* build's commit — the same layering the watcher applies — so the follow-up's command
* comes with the repository while its trigger stays the host's (pinned by
* [ConfigLoader]). The pull-request gate is not consulted: the predecessor passed it for
* this very commit, and the host's selector is the follow-up's own gate.
*
* Armed by [Watcher.start] and disarmed by [Watcher.stop], so a CLI `build` — whose
* process ends with its build — never enqueues a follow-up into a JVM that is about to
* exit; the CLI names the follow-ups the server would have run instead.
*/
@Component
class FollowUpTrigger(
private val gitService: GitService,
private val configLoader: ConfigLoader,
private val buildExecutor: BuildExecutor,
private val clock: Clock,
) {
private val log = LoggerFactory.getLogger(FollowUpTrigger::class.java)
private val armed = AtomicBoolean(false)
fun arm() {
armed.set(true)
}
fun disarm() {
armed.set(false)
}
fun isArmed(): Boolean = armed.get()
@EventListener
fun onBuildStatusChanged(event: BuildStatusChangedEvent) {
if (!armed.get() || event.result.status != BuildStatus.SUCCESS) {
return
}
val result = event.result
try {
for (name in followUpsOf(event.repo, result)) {
log.info(
"[{}] enqueueing follow-up build {} of branch {} at commit {}, after {}",
event.repo.name,
name,
result.branch,
result.commit,
result.build,
)
buildExecutor.startBuild(event.repo, result.branch, result.commit, name)
}
} catch (e: Exception) {
log.error("[{}] could not enqueue the follow-ups of {} at {}", event.repo.name, result.name, result.commit, e)
}
}
/** The names of the builds that follow a green [result], in the order of their definitions; nothing is enqueued. */
fun followUpsOf(
repo: RepoContext,
result: BuildResult,
): List<String> = followUpsOf(repo, result.branch, result.commit, result.build)
/** The names of the builds that follow a green run of [build] on [branch] at [commit]; nothing is enqueued. */
fun followUpsOf(
repo: RepoContext,
branch: String,
commit: String,
build: String,
): List<String> {
val workingDir = repo.workingDir
val definitions = definitionsAt(repo, branch, commit)
val headCommittedAt = lazy { gitService.originBranchCommitTimes(workingDir)[branch] }
return definitions
.filter { (_, definition) -> definition.trigger.afterSuccessOf == build }
.filter { (_, definition) -> definition.trigger.selects(branch, { headCommittedAt.value }, clock.instant()) }
.keys
.toList()
}
/**
* The branch's definitions at [commit] — not at its head, which may have moved on
* since the predecessor started. An unreadable branch config falls back to the
* primary definitions, like the watcher does.
*/
private fun definitionsAt(
repo: RepoContext,
branch: String,
commit: String,
): Map<String, BuildDefinition> {
val workingDir = repo.workingDir
return try {
configLoader
.loadWithBranchLayer(
workingDir,
ConfigFiles.readCommitted { gitService.showFileAtCommit(commit, it, workingDir) },
branch,
).effectiveBuildDefinitions()
} catch (e: Exception) {
log.warn(
"[{}] ignoring the committed {} of branch {} at {} for its follow-ups: {}",
repo.name,
ConfigFiles.COMMITTED,
branch,
commit,
e.message ?: e.javaClass.simpleName,
)
configLoader.load(workingDir).effectiveBuildDefinitions()
}
}
}
@@ -41,6 +41,7 @@ class Watcher(
private val buildExecutor: BuildExecutor, private val buildExecutor: BuildExecutor,
private val configLoader: ConfigLoader, private val configLoader: ConfigLoader,
private val clock: Clock, private val clock: Clock,
private val followUpTrigger: FollowUpTrigger,
) { ) {
private val log = LoggerFactory.getLogger(Watcher::class.java) private val log = LoggerFactory.getLogger(Watcher::class.java)
@@ -83,11 +84,14 @@ class Watcher(
* Runs the startup recovery of every repository and schedules the poll loop with the * Runs the startup recovery of every repository and schedules the poll loop with the
* fixed delay `watcher.pollInterval` — one loop, one delay: the instance's setting, * fixed delay `watcher.pollInterval` — one loop, one delay: the instance's setting,
* which every repository's effective config carries; the first poll runs immediately. * which every repository's effective config carries; the first poll runs immediately.
* Arms the [FollowUpTrigger] first, so the recovery's re-enqueued builds get their
* follow-ups too.
*/ */
@Synchronized @Synchronized
fun start(repos: List<RepoContext>) { fun start(repos: List<RepoContext>) {
check(scheduler == null) { "watcher is already running" } check(scheduler == null) { "watcher is already running" }
require(repos.isNotEmpty()) { "no repository to watch" } require(repos.isNotEmpty()) { "no repository to watch" }
followUpTrigger.arm()
repos.forEach { recoverSafely(it) } repos.forEach { recoverSafely(it) }
val interval = DurationParser.parse(configLoader.load(repos.first().workingDir).watcher.pollInterval) val interval = DurationParser.parse(configLoader.load(repos.first().workingDir).watcher.pollInterval)
scheduler = scheduler =
@@ -111,6 +115,7 @@ class Watcher(
@Synchronized @Synchronized
fun stop() { fun stop() {
followUpTrigger.disarm()
scheduler?.shutdownNow() scheduler?.shutdownNow()
scheduler = null scheduler = null
state = state.copy(running = false) state = state.copy(running = false)
@@ -292,6 +292,42 @@ class BuildExecutorTest : FunSpec() {
.build shouldBe "default" .build shouldBe "default"
} }
test("a follow-up build runs in its branch's worktree after its predecessor") {
val h =
Harness(
"""
executor:
maxConcurrent: 2
builds:
default:
trigger:
onPush: true
cleanCommand: ""
buildCommand: "sleep 1; echo built > output.txt"
deploy:
trigger:
afterSuccessOf: default
buildCommand: "cat output.txt"
""".trimIndent(),
)
h.executor.startBuild(h.repo, "main", "c1", "default")
h.executor.startBuild(h.repo, "main", "c1", "deploy")
awaitStatus(h, "main@deploy", BuildStatus.SUCCESS)
awaitIdle(h)
// same branch, same commit: the same worktree, and the follow-up saw the predecessor's output
h.workspaceCalls shouldContainExactly listOf("main" to "c1", "main" to "c1")
val predecessor = h.repository.latestFor("main").shouldNotBeNull()
val followUp = h.repository.latestFor("main@deploy").shouldNotBeNull()
predecessor.status shouldBe BuildStatus.SUCCESS
followUp.runningSince
.shouldNotBeNull()
.isBefore(predecessor.runningSince.shouldNotBeNull())
.shouldBeFalse()
Files.readString(h.workingDir.resolve("output.txt")).trim() shouldBe "built"
}
test("a build whose definition was removed from the config falls back to the branch's settings") { test("a build whose definition was removed from the config falls back to the branch's settings") {
val h = harness(buildCommand = "echo regular-\$branch") val h = harness(buildCommand = "echo regular-\$branch")
@@ -4,6 +4,7 @@ import de.hoennig.werkator.build.BuildStatus
import de.hoennig.werkator.git.GitService import de.hoennig.werkator.git.GitService
import de.hoennig.werkator.repo.RepoContext import de.hoennig.werkator.repo.RepoContext
import de.hoennig.werkator.repo.RepoRegistry import de.hoennig.werkator.repo.RepoRegistry
import de.hoennig.werkator.watcher.FollowUpTrigger
import io.kotest.core.spec.style.FunSpec import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldBe
import io.kotest.matchers.string.shouldContain import io.kotest.matchers.string.shouldContain
@@ -22,16 +23,31 @@ class BuildCommandTest : FunSpec() {
private val dir: Path = Paths.get(".") private val dir: Path = Paths.get(".")
private val repo = RepoContext("test", dir, mockk(), mockk()) private val repo = RepoContext("test", dir, mockk(), mockk())
private val registry = mockk<RepoRegistry>().also { every { it.current() } returns repo } private val registry = mockk<RepoRegistry>().also { every { it.current() } returns repo }
private val followUpTrigger = mockk<FollowUpTrigger>()
private fun command(fragment: String? = null) = private fun command(fragment: String? = null) =
BuildCommand(gitService, consoleBuildRunner, registry).apply { BuildCommand(gitService, consoleBuildRunner, registry, followUpTrigger).apply {
branchFragment = fragment branchFragment = fragment
} }
init { init {
beforeEach { beforeEach {
clearMocks(gitService, consoleBuildRunner) clearMocks(gitService, consoleBuildRunner, followUpTrigger)
justRun { gitService.fetchOrigin(dir) } justRun { gitService.fetchOrigin(dir) }
every { followUpTrigger.followUpsOf(any(), any(), any(), any()) } returns emptyList()
}
test("a green CLI build names the follow-up builds the server would run, and runs none") {
every { gitService.currentBranch(dir) } returns "main"
every { gitService.localHeadCommit("main", dir) } returns "local-head"
every { gitService.hasNewCommits("main", dir) } returns false
every { consoleBuildRunner.buildAndStream(repo, "main", "local-head") } returns BuildStatus.SUCCESS
every { followUpTrigger.followUpsOf(repo, "main", "local-head", "default") } returns listOf("deploy")
val console = captureConsole { command().call() }
console.stdout.shouldContain("follow-up build(s) deploy")
verify(exactly = 1) { consoleBuildRunner.buildAndStream(repo, "main", "local-head") }
} }
test("builds the current branch at its local head when no branch is given") { test("builds the current branch at its local head when no branch is given") {
@@ -0,0 +1,173 @@
package de.hoennig.werkator.watcher
import de.hoennig.werkator.build.ArtifactKeys
import de.hoennig.werkator.build.BuildExecutor
import de.hoennig.werkator.build.BuildResult
import de.hoennig.werkator.build.BuildStatus
import de.hoennig.werkator.build.BuildStatusChangedEvent
import de.hoennig.werkator.build.RunningBuild
import de.hoennig.werkator.config.BuildDefinition
import de.hoennig.werkator.config.ConfigLoader
import de.hoennig.werkator.config.TriggerConfig
import de.hoennig.werkator.config.WerkatorConfig
import de.hoennig.werkator.git.GitService
import de.hoennig.werkator.repo.RepoContext
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.collections.shouldBeEmpty
import io.kotest.matchers.collections.shouldContainExactly
import io.mockk.every
import io.mockk.mockk
import java.nio.file.Files
import java.time.Clock
import java.time.Instant
import java.time.ZoneOffset
import java.util.concurrent.CopyOnWriteArrayList
class FollowUpTriggerTest : FunSpec() {
private val noon = Instant.parse("2026-09-04T12:00:00Z")
/** Which build was started on which branch at which commit. */
private data class Started(
val branch: String,
val commit: String,
val build: String,
)
private inner class Harness(
config: WerkatorConfig,
) {
val workingDir = Files.createTempDirectory("werkator-followup-test")
val gitService = mockk<GitService>()
val configLoader = mockk<ConfigLoader>()
val buildExecutor = mockk<BuildExecutor>()
val repo = RepoContext("test", workingDir, mockk(), mockk())
val started = CopyOnWriteArrayList<Started>()
val trigger = FollowUpTrigger(gitService, configLoader, buildExecutor, Clock.fixed(noon, ZoneOffset.UTC))
init {
every { configLoader.load(any()) } returns config
every { configLoader.loadWithBranchLayer(any(), anyNullable(), anyNullable()) } returns config
every { gitService.showFileAtCommit(any(), any(), any()) } returns null
// the branch moved on since the predecessor started
every { gitService.originHeadCommit(any(), any()) } returns "c2"
every { gitService.originBranchCommitTimes(any()) } returns mapOf("main" to noon.minusSeconds(60))
every { buildExecutor.startBuild(any(), any(), any(), any()) } answers {
val branch = secondArg<String>()
val commit = thirdArg<String>()
val build = arg<String>(3)
started += Started(branch, commit, build)
val staging = Files.createTempDirectory("werkator-followup-staging")
RunningBuild(
repo = repo,
branch = branch,
build = build,
commit = commit,
artifactKey = ArtifactKeys.buildKey(BuildDefinition.poolName(branch, build), noon),
startedAt = noon,
stagingDir = staging,
liveLogFile = staging.resolve("build.log"),
)
}
}
fun finished(
build: String,
status: BuildStatus,
branch: String = "main",
commit: String = "c1",
) {
val result =
BuildResult(
branch = branch,
build = build,
commit = commit,
status = status,
startedAt = noon,
artifactKey = ArtifactKeys.buildKey(BuildDefinition.poolName(branch, build), noon),
)
trigger.onBuildStatusChanged(BuildStatusChangedEvent(result, repo))
}
}
private fun deployAfter(
predecessor: String,
branches: List<String> = emptyList(),
): WerkatorConfig =
WerkatorConfig(
buildDefinitions =
mapOf(
"frontend" to BuildDefinition(trigger = TriggerConfig(onPush = true)),
"backend" to BuildDefinition(trigger = TriggerConfig(onPush = true)),
"deploy" to
BuildDefinition(
trigger = TriggerConfig(afterSuccessOf = predecessor, branches = branches),
buildCommand = "scripts/deploy.sh",
),
),
)
init {
test("a green predecessor enqueues the follow-up at the predecessor's commit") {
val h = Harness(deployAfter("frontend", branches = listOf("main")))
h.trigger.arm()
h.finished("frontend", BuildStatus.SUCCESS, commit = "c1")
// c1, not the origin head c2 the branch has moved on to
h.started shouldContainExactly listOf(Started("main", "c1", "deploy"))
}
test("every green run of the predecessor triggers the follow-up again") {
val h = Harness(deployAfter("frontend"))
h.trigger.arm()
h.finished("frontend", BuildStatus.SUCCESS, commit = "c1")
h.finished("frontend", BuildStatus.SUCCESS, commit = "c1")
h.started shouldContainExactly
listOf(
Started("main", "c1", "deploy"),
Started("main", "c1", "deploy"),
)
}
test("only a SUCCESS of the named predecessor triggers") {
val h = Harness(deployAfter("frontend", branches = listOf("main")))
h.trigger.arm()
h.finished("frontend", BuildStatus.FAILED)
h.finished("frontend", BuildStatus.CANCELLED)
h.finished("frontend", BuildStatus.INTERRUPTED)
h.finished("frontend", BuildStatus.PENDING)
h.finished("frontend", BuildStatus.RUNNING)
h.finished("backend", BuildStatus.SUCCESS)
// a branch the host's selector does not name never runs the follow-up
h.finished("frontend", BuildStatus.SUCCESS, branch = "feature/x")
h.started.shouldBeEmpty()
}
test("the trigger listens only while the watcher runs") {
val h = Harness(deployAfter("frontend"))
h.finished("frontend", BuildStatus.SUCCESS)
h.started.shouldBeEmpty()
h.trigger.arm()
h.finished("frontend", BuildStatus.SUCCESS)
h.started shouldContainExactly listOf(Started("main", "c1", "deploy"))
h.trigger.disarm()
h.finished("frontend", BuildStatus.SUCCESS)
h.started shouldContainExactly listOf(Started("main", "c1", "deploy"))
}
test("followUpsOf names the follow-ups without enqueueing anything") {
val h = Harness(deployAfter("default"))
h.trigger.followUpsOf(h.repo, "main", "c1", "default") shouldContainExactly listOf("deploy")
h.trigger.followUpsOf(h.repo, "main", "c1", "frontend").shouldBeEmpty()
h.started.shouldBeEmpty()
}
}
}
@@ -63,6 +63,7 @@ class WatcherTest : FunSpec() {
val artifactStore = mockk<ArtifactStore>() val artifactStore = mockk<ArtifactStore>()
val startedBuilds = CopyOnWriteArrayList<Pair<String, String>>() val startedBuilds = CopyOnWriteArrayList<Pair<String, String>>()
val configLoader = mockk<ConfigLoader>() val configLoader = mockk<ConfigLoader>()
val followUpTrigger = mockk<FollowUpTrigger>(relaxed = true)
val repo = RepoContext("test", workingDir, repository, artifactStore) val repo = RepoContext("test", workingDir, repository, artifactStore)
val watcher = val watcher =
Watcher( Watcher(
@@ -70,6 +71,7 @@ class WatcherTest : FunSpec() {
buildExecutor = buildExecutor, buildExecutor = buildExecutor,
configLoader = configLoader, configLoader = configLoader,
clock = Clock.fixed(noon, ZoneOffset.UTC), clock = Clock.fixed(noon, ZoneOffset.UTC),
followUpTrigger = followUpTrigger,
) )
private var seedCounter = 0L private var seedCounter = 0L
@@ -157,6 +159,19 @@ class WatcherTest : FunSpec() {
) )
init { init {
test("start arms the follow-up trigger before the recovery, stop disarms it") {
val harness = Harness()
harness.watcher.start(listOf(harness.repo))
try {
verify(exactly = 1) { harness.followUpTrigger.arm() }
verify(exactly = 0) { harness.followUpTrigger.disarm() }
} finally {
harness.watcher.stop()
}
verify(exactly = 1) { harness.followUpTrigger.disarm() }
}
test("a fetch failure is exposed in the state and only retried next cycle") { test("a fetch failure is exposed in the state and only retried next cycle") {
val harness = Harness() val harness = Harness()
every { harness.gitService.fetchOrigin(any()) } throws RuntimeException("origin unreachable") every { harness.gitService.fetchOrigin(any()) } throws RuntimeException("origin unreachable")