Record builds interrupted by a server shutdown as INTERRUPTED, not FAILED

A systemd stop killed the running build process and BuildExecutor
classified the death as FAILED, posting a red Gitea status; FAILED is
not restartable, so the startup recovery never re-enqueued the build.

A ContextClosedEvent listener now sets a shuttingDown flag, terminates
the process trees of executing builds, and drains until their
INTERRUPTED results are persisted. Queued builds stay PENDING without
starting a process; recovery re-enqueues both after the restart.
INTERRUPTED publishes as Gitea state "pending" instead of "failure",
since the build is going to be re-run.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
mhoennig
2026-08-10 17:26:10 +02:00
co-authored by Claude Fable 5
parent fbdc6f54b7
commit 40f54a1298
7 changed files with 236 additions and 20 deletions
@@ -292,6 +292,74 @@ class BuildExecutorTest : FunSpec() {
}
}
test("shutdown kills an executing build and records INTERRUPTED, not FAILED") {
val h = harness("echo \$\$ > pid-file; sleep 30")
val build = h.executor.startBuild("main", "abc123", h.workingDir)
eventually(10.seconds) {
Files.exists(h.workingDir.resolve("pid-file")).shouldBeTrue()
}
val pid =
Files
.readString(h.workingDir.resolve("pid-file"))
.trim()
.toLong()
h.executor.shutdown()
// shutdown() drains synchronously, so the result is already persisted
h.repository.latestFor("main")?.status shouldBe BuildStatus.INTERRUPTED
h.executor.currentBuilds().shouldBeEmpty()
ProcessHandle.of(pid).map { it.isAlive }.orElse(false) shouldBe false
h.events.map { it.result.status } shouldContainExactly
listOf(BuildStatus.PENDING, BuildStatus.RUNNING, BuildStatus.INTERRUPTED)
Files.readString(build.liveLogFile) shouldContain "build interrupted by shutdown"
verify { h.giteaClient.publishStatus("abc123", BuildStatus.INTERRUPTED, any(), null, h.workingDir) }
verify { h.artifactStore.persist(match { it.status == BuildStatus.INTERRUPTED }, build.stagingDir, any()) }
}
test("a build still queued at shutdown stays PENDING for the startup recovery") {
val h = harness("sleep 30")
val first = h.executor.startBuild("main", "sha-1", h.workingDir)
val second = h.executor.startBuild("main", "sha-2", h.workingDir)
eventually(10.seconds) {
h.repository
.history()
.first { it.artifactKey == first.artifactKey }
.status shouldBe BuildStatus.RUNNING
}
h.executor.shutdown()
h.repository
.history()
.first { it.artifactKey == first.artifactKey }
.status shouldBe BuildStatus.INTERRUPTED
// the branch worker drops the queued build without a transition or a process
awaitIdle(h)
h.repository
.history()
.first { it.artifactKey == second.artifactKey }
.status shouldBe BuildStatus.PENDING
h.events
.filter { it.result.artifactKey == second.artifactKey }
.map { it.result.status } shouldContainExactly listOf(BuildStatus.PENDING)
verify(exactly = 0) { h.giteaClient.publishStatus("sha-2", BuildStatus.RUNNING, any(), any(), any()) }
}
test("shutdown without any build in flight is a no-op") {
val h = harness("echo ok")
h.executor.startBuild("main", "abc123", h.workingDir)
awaitStatus(h, "main", BuildStatus.SUCCESS)
awaitIdle(h)
h.executor.shutdown()
h.repository.latestFor("main")?.status shouldBe BuildStatus.SUCCESS
}
test("cancel with an unknown artifact key returns false") {
val h = harness("echo ok")
@@ -121,10 +121,11 @@ class GiteaClientTest :
mapOf(
BuildStatus.SUCCESS to "success",
BuildStatus.FAILED to "failure",
BuildStatus.INTERRUPTED to "failure",
BuildStatus.CANCELLED to "failure",
BuildStatus.PENDING to "pending",
BuildStatus.RUNNING to "pending",
// interrupted builds are re-enqueued on startup, so the commit must not turn red
BuildStatus.INTERRUPTED to "pending",
)
expectedStates.forEach { (status, state) ->
@@ -11,10 +11,11 @@ class GiteaStateMappingTest :
test("maps each build status to its Gitea state") {
BuildStatus.SUCCESS.toGiteaState() shouldBe "success"
BuildStatus.FAILED.toGiteaState() shouldBe "failure"
BuildStatus.INTERRUPTED.toGiteaState() shouldBe "failure"
BuildStatus.CANCELLED.toGiteaState() shouldBe "failure"
BuildStatus.PENDING.toGiteaState() shouldBe "pending"
BuildStatus.RUNNING.toGiteaState() shouldBe "pending"
// interrupted builds are re-enqueued on startup, so the commit must not turn red
BuildStatus.INTERRUPTED.toGiteaState() shouldBe "pending"
}
test("maps Gitea states back to build statuses") {