Show an unreachable origin in the web UI (step 19)

A watcher that cannot fetch leaves the server perfectly reachable and
every branch row stale — a calm list that implies nothing changed. The
state was already recorded and served at /api/watcher; only nothing
rendered it.

The banner lives in the shared nav fragment, so every view inherits it,
and is fed from the view's own polling tick without being chained to it.
It stays separate from live-indicator: that one reports whether the
browser reaches the server, this one whether the server reaches origin.

Also caps the log volume the outage exposed: a lasting fetch failure is
logged when its message changes, not on every cycle, and the recovery is
logged once. Same for an invalid atTimes slot.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/main/kotlin/de/hoennig/gittally/watcher/AutoBuildState.kt

@@ -12,6 +12,7 @@ import java.nio.file.StandardCopyOption
 import java.time.LocalDate
 import java.time.LocalTime
 import java.time.format.DateTimeParseException
+import java.util.concurrent.ConcurrentHashMap
 
 /**
  * One recorded scheduled-build trigger: the result pool [branch] (a branch, or
@@ -29,6 +30,9 @@ data class AutoBuildTrigger(
 object AutoBuildSlots {
     private val log = LoggerFactory.getLogger(AutoBuildSlots::class.java)
 
+    /** Slots already reported as invalid; the poll loop would otherwise warn about each one forever. */
+    private val warnedInvalidSlots = ConcurrentHashMap.newKeySet<String>()
+
     /**
      * The latest valid slot at or before [now], or null when no slot is due yet today.
      * The returned slot is always a concrete `HH:MM` — an hourly pattern is expanded
@@ -44,7 +48,9 @@ object AutoBuildSlots {
                 try {
                     LocalTime.parse(slot) to slot
                 } catch (_: DateTimeParseException) {
-                    log.warn("skipping invalid scheduled-build time slot '{}': expected HH:MM or ??:MM", slot)
+                    if (warnedInvalidSlots.add(slot)) {
+                        log.warn("skipping invalid scheduled-build time slot '{}': expected HH:MM or ??:MM", slot)
+                    }
                     null
                 }
             }.filter { (parsed, _) -> !parsed.isAfter(now) }

src/main/kotlin/de/hoennig/gittally/watcher/Watcher.kt

@@ -54,6 +54,15 @@ class Watcher(
     @Volatile
     private var warnedDeprecatedAutoBuild = false
 
+    /**
+     * The fetch failure last written to the log, so a lasting outage does not repeat the
+     * same warning on every poll — one wrong token produced 297 identical lines before
+     * this. Null while the last fetch succeeded, which is also what makes the recovery
+     * loggable.
+     */
+    @Volatile
+    private var loggedFetchError: String? = null
+
     /** Build definitions per branch, cached by the branch's head commit — see [definitionsFor]. */
     private val branchDefinitions = ConcurrentHashMap<String, CachedDefinitions>()
 
@@ -125,8 +134,8 @@ class Watcher(
     }
 
     /**
-     * One poll cycle, never blocking on a build: fetch origin (on failure: log, expose
-     * in [state], retry next cycle), enqueue due branches — changed local branches
+     * One poll cycle, never blocking on a build: fetch origin (on failure: log once per
+     * message, expose in [state], retry next cycle), enqueue due branches — changed local branches
      * first, then recent new origin branches, then due auto-build slots — then
      * fast-forward the local branch refs, and finally prune results, artifacts, and
      * worktrees of branches gone from origin.
@@ -135,9 +144,17 @@ class Watcher(
         val startedAt = clock.instant()
         try {
             gitService.fetchOrigin(workingDir)
+            if (loggedFetchError != null) {
+                log.info("fetching origin succeeded again")
+                loggedFetchError = null
+            }
         } catch (e: Exception) {
-            log.warn("fetching origin failed; retrying next cycle: {}", e.message)
-            state = state.copy(lastPollAt = startedAt, lastFetchError = e.message ?: e.javaClass.simpleName)
+            val failure = e.message ?: e.javaClass.simpleName
+            if (loggedFetchError != failure) {
+                log.warn("fetching origin failed; retrying every cycle until it succeeds: {}", failure)
+                loggedFetchError = failure
+            }
+            state = state.copy(lastPollAt = startedAt, lastFetchError = failure)
             return
         }
         val config = configLoader.load(workingDir)

src/main/resources/static/gittally.css

@@ -86,6 +86,12 @@ tbody tr:nth-child(even) { background: var(--row); }
 tbody tr:last-child td { border-bottom: 0; }
 tbody tr:hover { background: color-mix(in srgb, var(--link) 8%, transparent); }
 tbody.is-stale { opacity: 0.55; }
+
+/* Server-side staleness: the watcher cannot reach origin. Distinct from the live
+   indicator, which reports whether this browser reaches the server. */
+.watcher-banner { display: flex; flex-wrap: wrap; align-items: baseline; gap: 10px; margin: -8px 0 18px; padding: 9px 12px; border-radius: 8px; background: var(--failed-bg); color: var(--failed-text); font-size: 13px; }
+.watcher-banner strong { text-transform: uppercase; letter-spacing: 0.03em; font-size: 12px; }
+.watcher-banner[hidden] { display: none; }
 .branch { font-weight: 650; }
 .duration-cell { white-space: nowrap; }
 /* queue wait time of a pending build — italic to distinguish it from real build time */

src/main/resources/static/gittally.js

@@ -208,6 +208,9 @@ function startPolling(refresh, intervalMs) {
         refresh()
             .then(() => setLiveIndicator(true, "last update " + formatTimestamp(new Date().toISOString())))
             .catch((error) => setLiveIndicator(false, String(error)));
+        // separate request, deliberately not chained: whether the watcher is healthy must
+        // not depend on this view's refresh, nor delay it
+        refreshWatcherBanner();
     };
     const start = () => {
         stop();
@@ -248,6 +251,49 @@ function elem(tag, className, text) {
     return element;
 }
 
+/**
+ * What the watcher's health means for the page in front of the reader, or null while
+ * nothing is wrong. Three different failures, one message: what you see is not current.
+ *
+ * This is not the `live-indicator`, which says whether *this browser* reaches the server.
+ * A watcher that cannot fetch leaves the server perfectly reachable and every row stale,
+ * which is exactly the outage that went unnoticed for 57 minutes on 2026-08-30.
+ */
+function watcherBannerText(state) {
+    const since = state.lastPollAt ? " Last attempt " + formatTimestamp(state.lastPollAt) + "." : "";
+    if (state.running === false) {
+        return ["watcher stopped", "No branch is being polled; nothing below will change." + since];
+    }
+    if (state.lastFetchError) {
+        return ["origin unreachable", "The list below is not updating." + since + " " + state.lastFetchError];
+    }
+    if (state.lastPollError) {
+        return ["poll cycle failed", "The list below may be incomplete." + since + " " + state.lastPollError];
+    }
+    return null;
+}
+
+/** Never rejects: an unreachable server is the live indicator's business, not the banner's. */
+async function refreshWatcherBanner() {
+    const banner = document.getElementById("watcher-banner");
+    if (!banner) {
+        return;
+    }
+    let state;
+    try {
+        state = await fetchJson("/api/watcher");
+    } catch (error) {
+        // leave the banner as it stands rather than claiming health we could not confirm
+        return;
+    }
+    const text = watcherBannerText(state);
+    banner.replaceChildren();
+    banner.hidden = text === null;
+    if (text) {
+        banner.append(elem("strong", null, text[0]), elem("span", null, text[1]));
+    }
+}
+
 function externalLink(href, text) {
     const anchor = elem("a", null, text);
     anchor.href = href;
@@ -665,5 +711,9 @@ initBuildsTable();
 initCurrentBuilds();
 initSystemTable();
 initReloadButton();
+// pages with a poller update the banner from their own tick; the static ones ask once
+if (!refreshNow) {
+    refreshWatcherBanner();
+}
 setInterval(tickRunningDurations, 1000);
 tickRunningDurations();

src/main/resources/templates/fragments.html

@@ -16,7 +16,8 @@
     <span class="repo-name" th:unless="${#strings.isEmpty(repoName)}" th:text="${repoName}">owner/repo</span>
 </h1>
 
-<div th:fragment="nav(view)" class="view-row">
+<th:block th:fragment="nav(view)">
+<div class="view-row">
     <nav class="view-toggle">
         <span th:if="${view == 'latest'}">Latest</span>
         <a th:unless="${view == 'latest'}" href="/">Latest</a>
@@ -33,6 +34,10 @@
         <button id="reload-button" class="reload-button" type="button" title="Reload view" aria-label="Reload view">⟳</button>
     </span>
 </div>
+<!-- Filled by gittally.js from /api/watcher: while the watcher cannot reach origin, every
+     row below is as old as its last successful poll, and a calm list would imply otherwise. -->
+<div id="watcher-banner" class="watcher-banner" role="status" hidden></div>
+</th:block>
 
 <footer th:fragment="footer" class="site-footer">
     <strong><a href="/releases" title="Release notes"><em th:text="'GitTally v' + ${version}">GitTally</em></a></strong>

src/test/kotlin/de/hoennig/gittally/watcher/WatcherTest.kt

@@ -1,5 +1,8 @@
 package de.hoennig.gittally.watcher
 
+import ch.qos.logback.classic.Logger
+import ch.qos.logback.classic.spi.ILoggingEvent
+import ch.qos.logback.core.read.ListAppender
 import de.hoennig.gittally.build.ArtifactKeys
 import de.hoennig.gittally.build.ArtifactStore
 import de.hoennig.gittally.build.BuildExecutor
@@ -24,6 +27,7 @@ import io.kotest.matchers.booleans.shouldBeTrue
 import io.kotest.matchers.collections.shouldBeEmpty
 import io.kotest.matchers.collections.shouldContainExactly
 import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
+import io.kotest.matchers.collections.shouldHaveSize
 import io.kotest.matchers.nulls.shouldBeNull
 import io.kotest.matchers.nulls.shouldNotBeNull
 import io.kotest.matchers.shouldBe
@@ -32,6 +36,7 @@ import io.mockk.every
 import io.mockk.mockk
 import io.mockk.verify
 import io.mockk.verifyOrder
+import org.slf4j.LoggerFactory
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Clock
@@ -172,6 +177,28 @@ class WatcherTest : FunSpec() {
                 .shouldBeNull()
         }
 
+        test("a lasting fetch failure is logged once, and so is the recovery") {
+            val harness = Harness()
+            val logged = captureWatcherLog()
+            every { harness.gitService.fetchOrigin(any()) } throws RuntimeException("origin unreachable")
+
+            repeat(5) { harness.watcher.poll(harness.workingDir) }
+
+            // one wrong token used to write a warning every ten seconds, 297 of them in an hour
+            logged().filter { it.contains("fetching origin failed") } shouldHaveSize 1
+
+            every { harness.gitService.fetchOrigin(any()) } returns Unit
+            repeat(3) { harness.watcher.poll(harness.workingDir) }
+
+            logged().filter { it.contains("fetching origin succeeded again") } shouldHaveSize 1
+
+            // a different failure is a different message and is worth saying again
+            every { harness.gitService.fetchOrigin(any()) } throws RuntimeException("host is down")
+            harness.watcher.poll(harness.workingDir)
+
+            logged().filter { it.contains("fetching origin failed") } shouldHaveSize 2
+        }
+
         test("poll enqueues changed local branches before recent new origin branches") {
             val harness = Harness()
             every { harness.gitService.originBranches(any()) } returns listOf("main", "feature/new")
@@ -745,3 +772,12 @@ class WatcherTest : FunSpec() {
         }
     }
 }
+
+/** Collects this spec's [Watcher] log messages; the returned lambda reads them at any point. */
+private fun captureWatcherLog(): () -> List<String> {
+    val logger = LoggerFactory.getLogger(Watcher::class.java) as Logger
+    val appender = ListAppender<ILoggingEvent>()
+    appender.start()
+    logger.addAppender(appender)
+    return { appender.list.map { it.formattedMessage } }
+}