implemented 09-system-metrics.md: added system metrics: introduced system monitoring with a metrics collector, REST API endpoint, Thymeleaf UI rendering, and lifecycle management

2026-07-07 12:55:11 +02:00
parent 67c9f0ade9
commit a6a2c9de0b
21 changed files with 1069 additions and 6 deletions
@@ -1,6 +1,7 @@
 package de.hoennig.gittally

 import com.ninjasquad.springmockk.MockkBean
+import de.hoennig.gittally.metrics.SystemMetricsCollector
 import de.hoennig.gittally.watcher.Watcher
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.booleans.shouldBeTrue
@@ -12,8 +13,9 @@ import org.springframework.test.context.ActiveProfiles
 import org.springframework.web.client.RestClient

 /**
- * Proves the `server` profile boots a real web server and starts the watcher.
- * The watcher is mocked so the test never fetches origin or enqueues builds.
+ * Proves the `server` profile boots a real web server and starts the watcher and
+ * the metrics collector. Both are mocked so the test never fetches origin,
+ * enqueues builds, or walks the repository for its size.
 */
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles("server")
@@ -21,6 +23,9 @@ class ServerModeApplicationTest : FunSpec() {
    @MockkBean(relaxUnitFun = true)
    lateinit var watcher: Watcher

+    @MockkBean(relaxUnitFun = true)
+    lateinit var metricsCollector: SystemMetricsCollector
+
    @LocalServerPort
    var port: Int = 0

@@ -40,6 +45,7 @@ class ServerModeApplicationTest : FunSpec() {
                .shouldBeTrue()

            verify { watcher.start(any()) }
+            verify { metricsCollector.start() }
        }
    }
 }
@@ -0,0 +1,217 @@
+package de.hoennig.gittally.metrics
+
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.nulls.shouldNotBeNull
+import io.kotest.matchers.shouldBe
+import java.nio.file.Files
+import java.nio.file.Path
+import java.time.Clock
+import java.time.Instant
+import java.time.ZoneOffset
+
+class SystemMetricsCollectorTest : FunSpec() {
+    private lateinit var tempDir: Path
+
+    private val now = Instant.parse("2026-07-07T10:00:00Z")
+
+    private fun collector(
+        cpuCount: Int = 4,
+        diskSpace: (Path) -> SystemMetricsCollector.DiskSpace = { GIB_100_DISK },
+        repoSizeBytes: (Path) -> Long = { HALF_GIB_BYTES },
+    ) = SystemMetricsCollector(
+        stateFile = { tempDir.resolve("system-metrics-state.json") },
+        workingDir = tempDir,
+        clock = Clock.fixed(now, ZoneOffset.UTC),
+        procStat = tempDir.resolve("stat"),
+        procMeminfo = tempDir.resolve("meminfo"),
+        cpuCount = cpuCount,
+        diskSpace = diskSpace,
+        repoSizeBytes = repoSizeBytes,
+    )
+
+    private fun writeStat(
+        total: Long,
+        idle: Long,
+    ) {
+        // "cpu" totals across: user nice system idle iowait irq softirq steal guest guest_nice
+        val user = total - idle
+        Files.writeString(tempDir.resolve("stat"), "cpu  $user 0 0 $idle 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\n")
+    }
+
+    private fun writeMeminfo(
+        totalKib: Long,
+        availableKib: Long,
+    ) {
+        Files.writeString(
+            tempDir.resolve("meminfo"),
+            "MemTotal:       $totalKib kB\nMemFree:         1000000 kB\nMemAvailable:   $availableKib kB\n",
+        )
+    }
+
+    init {
+        beforeEach {
+            tempDir = Files.createTempDirectory("gittally-metrics-test")
+        }
+
+        afterEach {
+            tempDir.toFile().deleteRecursively()
+        }
+
+        test("CPU load is computed from /proc/stat deltas, so the first sample has no CPU metric yet") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            val collector = collector()
+
+            collector.sample()
+            collector.snapshot().cpuUsed shouldBe null
+
+            writeStat(total = 1100, idle = 775)
+            collector.sample()
+            val snapshot = collector.snapshot()
+            // 4 cores * (100 total - 75 idle) / 100 total = 1 core used
+            snapshot.cpuUsed.shouldNotBeNull().current shouldBe 1.0
+            snapshot.cpuIdle.shouldNotBeNull().current shouldBe 3.0
+        }
+
+        test("RAM comes from MemTotal and MemAvailable in GiB") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            val collector = collector()
+
+            collector.sample()
+
+            val snapshot = collector.snapshot()
+            snapshot.ramTotalGib shouldBe 32.0
+            snapshot.ramUsedGib.shouldNotBeNull().current shouldBe 8.0
+            snapshot.ramFreeGib.shouldNotBeNull().current shouldBe 24.0
+        }
+
+        test("disk and repository size are reported in GiB") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            val collector = collector()
+
+            collector.sample()
+
+            val snapshot = collector.snapshot()
+            snapshot.diskTotalGib shouldBe 100.0
+            snapshot.diskUsedGib.shouldNotBeNull().current shouldBe 40.0
+            snapshot.diskFreeGib.shouldNotBeNull().current shouldBe 55.0
+            snapshot.repoSizeGib.shouldNotBeNull().current shouldBe 0.5
+        }
+
+        test("min, max, and avg aggregate over all samples") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            val usedGibValues = mutableListOf(40L, 20L, 60L)
+            val collector =
+                collector(diskSpace = {
+                    SystemMetricsCollector.DiskSpace(
+                        totalBytes = 100L * GIB,
+                        usedBytes = usedGibValues.removeFirst() * GIB,
+                        freeBytes = 30L * GIB,
+                    )
+                })
+
+            repeat(3) { collector.sample() }
+
+            val diskUsed = collector.snapshot().diskUsedGib.shouldNotBeNull()
+            diskUsed.current shouldBe 60.0
+            diskUsed.min shouldBe 20.0
+            diskUsed.max shouldBe 60.0
+            diskUsed.avg shouldBe 40.0
+        }
+
+        test("a restart loads the persisted aggregation state and continues the series") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            collector(repoSizeBytes = { 4L * GIB }).sample()
+
+            val restarted = collector(repoSizeBytes = { 2L * GIB })
+            restarted.sample()
+
+            val snapshot = restarted.snapshot()
+            snapshot.sampleCount shouldBe 2
+            val repoSize = snapshot.repoSizeGib.shouldNotBeNull()
+            repoSize.current shouldBe 2.0
+            repoSize.min shouldBe 2.0
+            repoSize.max shouldBe 4.0
+            repoSize.avg shouldBe 3.0
+        }
+
+        test("a corrupt state file starts a fresh series instead of failing") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            Files.writeString(tempDir.resolve("system-metrics-state.json"), "not json {")
+
+            val collector = collector()
+            collector.sample()
+
+            collector.snapshot().sampleCount shouldBe 1
+        }
+
+        test("unreadable sources degrade to null metrics, never fail the sample") {
+            // no stat/meminfo files written, disk and repo-size probes fail
+            val collector =
+                collector(
+                    diskSpace = { error("no file store") },
+                    repoSizeBytes = { error("walk failed") },
+                )
+
+            collector.sample()
+
+            val snapshot = collector.snapshot()
+            snapshot.timestamp shouldBe now
+            snapshot.sampleCount shouldBe 1
+            snapshot.cpuCount shouldBe 4
+            snapshot.cpuUsed shouldBe null
+            snapshot.ramTotalGib shouldBe null
+            snapshot.ramUsedGib shouldBe null
+            snapshot.diskTotalGib shouldBe null
+            snapshot.diskUsedGib shouldBe null
+            snapshot.repoSizeGib shouldBe null
+        }
+
+        test("the repository size probe is throttled to every 10th sample") {
+            writeStat(total = 1000, idle = 700)
+            writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824)
+            var probes = 0
+            val collector =
+                collector(repoSizeBytes = {
+                    probes++
+                    HALF_GIB_BYTES
+                })
+
+            repeat(11) { collector.sample() }
+
+            probes shouldBe 2
+            collector
+                .snapshot()
+                .repoSizeGib
+                .shouldNotBeNull()
+                .current shouldBe 0.5
+        }
+
+        test("the snapshot before the first sample is empty but well-formed") {
+            val snapshot = collector().snapshot()
+
+            snapshot.timestamp shouldBe null
+            snapshot.sampleCount shouldBe 0
+            snapshot.cpuCount shouldBe 4
+            snapshot.cpuUsed shouldBe null
+        }
+    }
+
+    companion object {
+        private const val GIB = 1_073_741_824L
+
+        private const val HALF_GIB_BYTES = GIB / 2
+
+        private val GIB_100_DISK =
+            SystemMetricsCollector.DiskSpace(
+                totalBytes = 100L * GIB,
+                usedBytes = 40L * GIB,
+                freeBytes = 55L * GIB,
+            )
+    }
+}
@@ -0,0 +1,81 @@
+package de.hoennig.gittally.server
+
+import com.ninjasquad.springmockk.MockkBean
+import de.hoennig.gittally.metrics.MetricAggregate
+import de.hoennig.gittally.metrics.SystemMetrics
+import de.hoennig.gittally.metrics.SystemMetricsCollector
+import io.kotest.core.spec.style.FunSpec
+import io.mockk.clearMocks
+import io.mockk.every
+import org.hamcrest.Matchers.nullValue
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest
+import org.springframework.test.web.servlet.MockMvc
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
+import java.time.Instant
+
+@WebMvcTest(SystemApiController::class, properties = ["spring.main.web-application-type=servlet"])
+class SystemApiControllerTest : FunSpec() {
+    @Autowired
+    lateinit var mockMvc: MockMvc
+
+    @MockkBean
+    lateinit var collector: SystemMetricsCollector
+
+    private val emptySnapshot =
+        SystemMetrics(
+            timestamp = null,
+            sampleCount = 0,
+            cpuCount = 8,
+            ramTotalGib = null,
+            diskTotalGib = null,
+            cpuUsed = null,
+            cpuIdle = null,
+            ramUsedGib = null,
+            ramFreeGib = null,
+            diskUsedGib = null,
+            diskFreeGib = null,
+            repoSizeGib = null,
+        )
+
+    init {
+        beforeEach { clearMocks(collector) }
+
+        test("the system endpoint answers snapshot and aggregates") {
+            every { collector.snapshot() } returns
+                emptySnapshot.copy(
+                    timestamp = Instant.parse("2026-07-07T10:00:00Z"),
+                    sampleCount = 5,
+                    ramTotalGib = 32.0,
+                    cpuUsed = MetricAggregate(current = 1.0, min = 0.5, max = 2.0, avg = 1.25),
+                )
+
+            mockMvc
+                .perform(get("/api/system"))
+                .andExpect(status().isOk)
+                .andExpect(jsonPath("$.timestamp").value("2026-07-07T10:00:00Z"))
+                .andExpect(jsonPath("$.sampleCount").value(5))
+                .andExpect(jsonPath("$.cpuCount").value(8))
+                .andExpect(jsonPath("$.ramTotalGib").value(32.0))
+                .andExpect(jsonPath("$.cpuUsed.current").value(1.0))
+                .andExpect(jsonPath("$.cpuUsed.min").value(0.5))
+                .andExpect(jsonPath("$.cpuUsed.max").value(2.0))
+                .andExpect(jsonPath("$.cpuUsed.avg").value(1.25))
+        }
+
+        test("unavailable metrics are explicit nulls, and the endpoint still answers 200") {
+            every { collector.snapshot() } returns emptySnapshot
+
+            mockMvc
+                .perform(get("/api/system"))
+                .andExpect(status().isOk)
+                .andExpect(jsonPath("$.sampleCount").value(0))
+                .andExpect(jsonPath("$.timestamp").value(nullValue()))
+                .andExpect(jsonPath("$.cpuUsed").value(nullValue()))
+                .andExpect(jsonPath("$.ramUsedGib").value(nullValue()))
+                .andExpect(jsonPath("$.repoSizeGib").value(nullValue()))
+        }
+    }
+}
@@ -11,6 +11,9 @@ import de.hoennig.gittally.config.ConfigLoader
 import de.hoennig.gittally.config.GitTallyConfig
 import de.hoennig.gittally.config.GiteaConfig
 import de.hoennig.gittally.config.ServerConfig
+import de.hoennig.gittally.metrics.MetricAggregate
+import de.hoennig.gittally.metrics.SystemMetrics
+import de.hoennig.gittally.metrics.SystemMetricsCollector
 import io.kotest.core.spec.style.FunSpec
 import io.mockk.clearMocks
 import io.mockk.every
@@ -49,8 +52,27 @@ class UiControllerTest : FunSpec() {
    @MockkBean
    lateinit var configLoader: ConfigLoader

+    @MockkBean
+    lateinit var metricsCollector: SystemMetricsCollector
+
    private val startedAt = Instant.parse("2026-07-07T10:00:00Z")

+    private val emptySystemMetrics =
+        SystemMetrics(
+            timestamp = null,
+            sampleCount = 0,
+            cpuCount = 8,
+            ramTotalGib = null,
+            diskTotalGib = null,
+            cpuUsed = null,
+            cpuIdle = null,
+            ramUsedGib = null,
+            ramFreeGib = null,
+            diskUsedGib = null,
+            diskFreeGib = null,
+            repoSizeGib = null,
+        )
+
    private val successResult =
        BuildResult(
            branch = "main",
@@ -63,7 +85,7 @@ class UiControllerTest : FunSpec() {

    init {
        beforeEach {
-            clearMocks(repository, buildExecutor, artifactStore, controlTokens, configLoader)
+            clearMocks(repository, buildExecutor, artifactStore, controlTokens, configLoader, metricsCollector)
            every { configLoader.load(any()) } returns
                GitTallyConfig(
                    server = ServerConfig(impressumUrl = "https://example.org/imprint"),
@@ -191,6 +213,36 @@ class UiControllerTest : FunSpec() {
                .andExpect(status().isNotFound)
        }

+        test("system view renders metric rows, totals, and the polling hook") {
+            every { metricsCollector.snapshot() } returns
+                emptySystemMetrics.copy(
+                    timestamp = Instant.parse("2026-07-07T10:00:00Z"),
+                    sampleCount = 5,
+                    ramTotalGib = 32.0,
+                    cpuUsed = MetricAggregate(current = 1.0, min = 0.5, max = 2.0, avg = 1.25),
+                )
+
+            mockMvc
+                .perform(get("/system"))
+                .andExpect(status().isOk)
+                .andExpect(content().string(containsString("""data-api="/api/system"""")))
+                .andExpect(content().string(containsString("""data-metric="cpuUsed"""")))
+                .andExpect(content().string(containsString("CPU used (cores)")))
+                .andExpect(content().string(containsString("1.25")))
+                .andExpect(content().string(containsString("8 cores")))
+                .andExpect(content().string(containsString("32.00 GiB")))
+        }
+
+        test("system view renders n/a for unavailable metrics") {
+            every { metricsCollector.snapshot() } returns emptySystemMetrics
+
+            mockMvc
+                .perform(get("/system"))
+                .andExpect(status().isOk)
+                .andExpect(content().string(containsString("Repo size (GiB)")))
+                .andExpect(content().string(containsString("n/a")))
+        }
+
        test("branch names with HTML metacharacters render escaped") {
            val nasty = "feat/<script>alert('x')</script>"
            every { repository.latestPerBranch() } returns listOf(successResult.copy(branch = nasty))
@@ -14,6 +14,14 @@ class UiViewsTest : FunSpec() {
            UiFormats.duration(Duration.ofSeconds(3600 + 62)) shouldBe "1:01:02"
        }

+        test("metric values format with two decimals and a dot, n/a when unavailable") {
+            UiFormats.metric(null) shouldBe "n/a"
+            UiFormats.metric(Double.NaN) shouldBe "n/a"
+            UiFormats.metric(0.0) shouldBe "0.00"
+            UiFormats.metric(1.234) shouldBe "1.23"
+            UiFormats.metric(31.288) shouldBe "31.29"
+        }
+
        test("Gitea web links escape branch segments but keep slashes") {
            val links =
                GiteaWebLinks(GiteaConfig(baseUrl = "https://git.example.org/", owner = "acme", repo = "widget"))