implemented 09-system-metrics.md: added system metrics: introduced system monitoring with a metrics collector, REST API endpoint, Thymeleaf UI rendering, and lifecycle management

2026-07-07 12:55:11 +02:00
parent 67c9f0ade9
commit a6a2c9de0b
21 changed files with 1069 additions and 6 deletions
@@ -102,7 +102,7 @@ class FileArtifactStore(

    private fun branchesDir(): Path = rootDir().resolve("branches")

-    private fun rootDir(): Path {
+    override fun rootDir(): Path {
        val configured =
            configLoader
                .load(workingDir)
@@ -28,4 +28,11 @@ interface ArtifactStore {

    /** The stored artifact directory for [artifactKey], or null if none exists. */
    fun artifactDir(artifactKey: String): Path?
+
+    /**
+     * The artifact root directory (which need not exist yet). Besides the stored
+     * builds it hosts sibling state like the system-metrics aggregation — legacy
+     * kept its `system_state.dat` in the artifact root, too.
+     */
+    fun rootDir(): Path
 }
@@ -0,0 +1,25 @@
+package de.hoennig.gittally.metrics
+
+import de.hoennig.gittally.build.ArtifactStore
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import java.time.Clock
+
+@Configuration
+class MetricsConfiguration {
+    /**
+     * The aggregation state lives in the artifact root (like the legacy
+     * `system_state.dat`), resolved lazily because the root comes from the config.
+     * Nothing is sampled or written until [SystemMetricsCollector.start], which only
+     * `ServerMetricsLifecycle` calls — the bean is inert in CLI runs and tests.
+     */
+    @Bean
+    fun systemMetricsCollector(
+        artifactStore: ArtifactStore,
+        clock: Clock,
+    ): SystemMetricsCollector =
+        SystemMetricsCollector(
+            stateFile = { artifactStore.rootDir().resolve(SystemMetricsCollector.STATE_FILE_NAME) },
+            clock = clock,
+        )
+}
@@ -0,0 +1,54 @@
+package de.hoennig.gittally.metrics
+
+import java.time.Instant
+
+/** Current value of one metric plus its min/max/avg over all recorded samples. */
+data class MetricAggregate(
+    val current: Double,
+    val min: Double,
+    val max: Double,
+    val avg: Double,
+)
+
+/**
+ * One system snapshot with aggregates, the payload of `GET /api/system`
+ * (the legacy `system.json`, with camelCase names like the rest of the API).
+ * A metric is null when its source is unavailable — e.g. no `/proc` outside
+ * Linux — and the UI shows `n/a`; the endpoint itself works everywhere.
+ */
+data class SystemMetrics(
+    val timestamp: Instant?,
+    val sampleCount: Long,
+    val cpuCount: Int,
+    val ramTotalGib: Double?,
+    val diskTotalGib: Double?,
+    val cpuUsed: MetricAggregate?,
+    val cpuIdle: MetricAggregate?,
+    val ramUsedGib: MetricAggregate?,
+    val ramFreeGib: MetricAggregate?,
+    val diskUsedGib: MetricAggregate?,
+    val diskFreeGib: MetricAggregate?,
+    val repoSizeGib: MetricAggregate?,
+)
+
+/** Running aggregation of one metric; persisted so restarts continue the series. */
+data class MetricSeries(
+    val min: Double,
+    val max: Double,
+    val sum: Double,
+    val count: Long,
+) {
+    operator fun plus(value: Double) =
+        MetricSeries(
+            min = minOf(min, value),
+            max = maxOf(max, value),
+            sum = sum + value,
+            count = count + 1,
+        )
+
+    fun aggregate(current: Double) = MetricAggregate(current = current, min = min, max = max, avg = sum / count)
+
+    companion object {
+        fun of(value: Double) = MetricSeries(min = value, max = value, sum = value, count = 1)
+    }
+}
@@ -0,0 +1,352 @@
+package de.hoennig.gittally.metrics
+
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.fasterxml.jackson.module.kotlin.readValue
+import com.fasterxml.jackson.module.kotlin.registerKotlinModule
+import org.slf4j.LoggerFactory
+import java.io.IOException
+import java.nio.file.FileVisitResult
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.SimpleFileVisitor
+import java.nio.file.StandardCopyOption
+import java.nio.file.attribute.BasicFileAttributes
+import java.time.Clock
+import java.util.concurrent.Executors
+import java.util.concurrent.ScheduledExecutorService
+import java.util.concurrent.TimeUnit
+
+/** The aggregation state persisted in the artifact root (the legacy `system_state.dat`, but as JSON). */
+data class PersistedMetricsState(
+    val sampleCount: Long = 0,
+    val series: Map<String, MetricSeries> = emptyMap(),
+)
+
+/**
+ * Samples CPU (from `/proc/stat` deltas), RAM (from `/proc/meminfo`), disk, and
+ * repository size every 60 seconds and keeps running min/max/avg per metric.
+ * The aggregation state is persisted, so restarts continue the series.
+ * Every source degrades gracefully: an unreadable source makes its metric null
+ * (shown as `n/a`), never fails a sample. Like the [de.hoennig.gittally.watcher.Watcher],
+ * nothing is scheduled until [start] is called (server mode only).
+ */
+class SystemMetricsCollector(
+    private val stateFile: () -> Path,
+    private val workingDir: Path = Paths.get("."),
+    private val clock: Clock = Clock.systemUTC(),
+    private val procStat: Path = Paths.get("/proc/stat"),
+    private val procMeminfo: Path = Paths.get("/proc/meminfo"),
+    private val cpuCount: Int = Runtime.getRuntime().availableProcessors(),
+    private val diskSpace: (Path) -> DiskSpace = { dir -> fileStoreDiskSpace(dir) },
+    private val repoSizeBytes: (Path) -> Long = { dir -> directorySizeBytes(dir) },
+) {
+    /** Disk usage in bytes; used/free follow `df` semantics (free is what a user can still allocate). */
+    data class DiskSpace(
+        val totalBytes: Long,
+        val usedBytes: Long,
+        val freeBytes: Long,
+    )
+
+    private val log = LoggerFactory.getLogger(SystemMetricsCollector::class.java)
+
+    private val json =
+        ObjectMapper()
+            .registerKotlinModule()
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+            .configure(SerializationFeature.INDENT_OUTPUT, true)
+
+    private var scheduler: ScheduledExecutorService? = null
+
+    private var state: PersistedMetricsState? = null
+
+    private var previousCpu: CpuCounters? = null
+
+    private var lastRepoSizeGib: Double? = null
+
+    private var samplesSinceRepoSizeProbe = 0
+
+    private val warnedSources = mutableSetOf<String>()
+
+    @Volatile
+    private var snapshot: SystemMetrics? = null
+
+    /** The latest sample, or an empty snapshot (all metrics `null`) before the first one. */
+    fun snapshot(): SystemMetrics = snapshot ?: emptySnapshot()
+
+    /** Schedules the fixed-delay sampling loop; the first sample runs immediately. */
+    @Synchronized
+    fun start() {
+        check(scheduler == null) { "metrics collector is already running" }
+        scheduler =
+            Executors
+                .newSingleThreadScheduledExecutor { runnable ->
+                    Thread(runnable, "gittally-metrics").apply { isDaemon = true }
+                }.also {
+                    it.scheduleWithFixedDelay(::sampleSafely, 0, SAMPLE_INTERVAL_SECONDS, TimeUnit.SECONDS)
+                }
+    }
+
+    @Synchronized
+    fun stop() {
+        scheduler?.shutdownNow()
+        scheduler = null
+    }
+
+    /**
+     * Takes one sample, updates the aggregates, and persists the aggregation state.
+     * CPU needs a counter delta, so the very first sample after process start reports
+     * no CPU metric yet (legacy aggregated a meaningless near-zero first delta instead).
+     */
+    @Synchronized
+    fun sample() {
+        val previousState = state ?: loadState()
+        val series = previousState.series.toMutableMap()
+        val sampleCount = previousState.sampleCount + 1
+
+        fun record(
+            key: String,
+            value: Double?,
+        ): MetricAggregate? {
+            if (value == null) {
+                return null
+            }
+            val updated = series[key]?.plus(value) ?: MetricSeries.of(value)
+            series[key] = updated
+            return updated.aggregate(value)
+        }
+
+        val cpu = readCpuLoad()
+        val ram = readRam()
+        val disk = readDisk()
+        val repoSizeGib = readRepoSizeThrottled()
+        snapshot =
+            SystemMetrics(
+                timestamp = clock.instant(),
+                sampleCount = sampleCount,
+                cpuCount = cpuCount,
+                ramTotalGib = ram?.totalGib,
+                diskTotalGib = disk?.totalBytes?.let { it / BYTES_PER_GIB },
+                cpuUsed = record("cpuUsed", cpu?.usedCores),
+                cpuIdle = record("cpuIdle", cpu?.idleCores),
+                ramUsedGib = record("ramUsedGib", ram?.usedGib),
+                ramFreeGib = record("ramFreeGib", ram?.freeGib),
+                diskUsedGib = record("diskUsedGib", disk?.usedBytes?.let { it / BYTES_PER_GIB }),
+                diskFreeGib = record("diskFreeGib", disk?.freeBytes?.let { it / BYTES_PER_GIB }),
+                repoSizeGib = record("repoSizeGib", repoSizeGib),
+            )
+        state = PersistedMetricsState(sampleCount = sampleCount, series = series)
+        saveState(state!!)
+    }
+
+    private fun sampleSafely() {
+        try {
+            sample()
+        } catch (e: Exception) {
+            log.error("metrics sample failed", e)
+        }
+    }
+
+    private fun emptySnapshot() =
+        SystemMetrics(
+            timestamp = null,
+            sampleCount = 0,
+            cpuCount = cpuCount,
+            ramTotalGib = null,
+            diskTotalGib = null,
+            cpuUsed = null,
+            cpuIdle = null,
+            ramUsedGib = null,
+            ramFreeGib = null,
+            diskUsedGib = null,
+            diskFreeGib = null,
+            repoSizeGib = null,
+        )
+
+    // ---- sources, each null when unavailable --------------------------------
+
+    private data class CpuCounters(
+        val total: Long,
+        val idle: Long,
+    )
+
+    private data class CpuLoad(
+        val usedCores: Double,
+        val idleCores: Double,
+    )
+
+    /** Like the legacy monitor: used cores = `cpuCount * (totalDiff - idleDiff) / totalDiff`. */
+    private fun readCpuLoad(): CpuLoad? {
+        val counters = readCpuCounters() ?: return null
+        val previous = previousCpu
+        previousCpu = counters
+        if (previous == null) {
+            return null
+        }
+        val totalDiff = counters.total - previous.total
+        val idleDiff = counters.idle - previous.idle
+        val usedCores =
+            if (totalDiff > 0) {
+                cpuCount * (totalDiff - idleDiff).toDouble() / totalDiff
+            } else {
+                0.0
+            }
+        return CpuLoad(usedCores = usedCores, idleCores = cpuCount - usedCores)
+    }
+
+    /** The aggregate `cpu` line of `/proc/stat`: total is the sum of all fields, idle is the 4th. */
+    private fun readCpuCounters(): CpuCounters? =
+        readSource("cpu") {
+            val fields =
+                Files
+                    .readAllLines(procStat)
+                    .first { it.startsWith("cpu ") }
+                    .split(Regex("\\s+"))
+                    .drop(1)
+                    .map { it.toLong() }
+            CpuCounters(total = fields.sum(), idle = fields[3])
+        }
+
+    private data class RamUsage(
+        val totalGib: Double,
+        val usedGib: Double,
+        val freeGib: Double,
+    )
+
+    /** `MemTotal` and `MemAvailable` (KiB) from `/proc/meminfo`; used = total - available. */
+    private fun readRam(): RamUsage? =
+        readSource("ram") {
+            val lines = Files.readAllLines(procMeminfo)
+            val totalKib = memValueKib(lines, "MemTotal")
+            val availableKib = memValueKib(lines, "MemAvailable")
+            RamUsage(
+                totalGib = totalKib / KIB_PER_GIB,
+                usedGib = (totalKib - availableKib) / KIB_PER_GIB,
+                freeGib = availableKib / KIB_PER_GIB,
+            )
+        }
+
+    private fun memValueKib(
+        lines: List<String>,
+        key: String,
+    ): Long =
+        lines
+            .first { it.startsWith("$key:") }
+            .removePrefix("$key:")
+            .trim()
+            .removeSuffix(" kB")
+            .toLong()
+
+    private fun readDisk(): DiskSpace? = readSource("disk") { diskSpace(workingDir) }
+
+    /**
+     * The repository size is expensive to determine (a full file walk), so unlike
+     * legacy — which ran `du -sk` every cycle — it is re-probed only every
+     * [REPO_SIZE_PROBE_EVERY_SAMPLES] samples and reused in between.
+     */
+    private fun readRepoSizeThrottled(): Double? {
+        if (lastRepoSizeGib != null && samplesSinceRepoSizeProbe < REPO_SIZE_PROBE_EVERY_SAMPLES) {
+            samplesSinceRepoSizeProbe++
+            return lastRepoSizeGib
+        }
+        lastRepoSizeGib = readSource("repo size") { repoSizeBytes(workingDir) / BYTES_PER_GIB }
+        samplesSinceRepoSizeProbe = 1
+        return lastRepoSizeGib
+    }
+
+    /** Wraps one source read; a failure yields null and is logged once, not every 60s. */
+    private fun <T> readSource(
+        source: String,
+        read: () -> T,
+    ): T? =
+        try {
+            read().also { warnedSources.remove(source) }
+        } catch (e: Exception) {
+            if (warnedSources.add(source)) {
+                log.warn("cannot read {} metrics; showing n/a: {}", source, e.toString())
+            }
+            null
+        }
+
+    // ---- aggregation state persistence ---------------------------------------
+
+    private fun loadState(): PersistedMetricsState {
+        val file = stateFile()
+        if (!Files.exists(file)) {
+            return PersistedMetricsState()
+        }
+        return try {
+            json.readValue<PersistedMetricsState>(file.toFile())
+        } catch (e: Exception) {
+            log.warn("ignoring unreadable metrics state file {}; starting a fresh series: {}", file, e.message)
+            PersistedMetricsState()
+        }
+    }
+
+    private fun saveState(state: PersistedMetricsState) {
+        try {
+            val file = stateFile()
+            Files.createDirectories(file.parent)
+            val tempFile = Files.createTempFile(file.parent, file.fileName.toString(), ".tmp")
+            try {
+                json.writeValue(tempFile.toFile(), state)
+                Files.move(tempFile, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING)
+            } finally {
+                Files.deleteIfExists(tempFile)
+            }
+        } catch (e: Exception) {
+            log.warn("cannot persist metrics state; aggregates restart on the next server start: {}", e.toString())
+        }
+    }
+
+    companion object {
+        const val SAMPLE_INTERVAL_SECONDS = 60L
+
+        /** With 60s samples, the repository size is re-measured every 10 minutes. */
+        const val REPO_SIZE_PROBE_EVERY_SAMPLES = 10
+
+        /** State file name in the artifact root, next to the `branches/` directory. */
+        const val STATE_FILE_NAME = "system-metrics-state.json"
+
+        private const val KIB_PER_GIB = 1_048_576.0
+
+        private const val BYTES_PER_GIB = 1_073_741_824.0
+
+        /** `df` semantics via [java.nio.file.FileStore]: free is the space a user can still allocate. */
+        fun fileStoreDiskSpace(dir: Path): DiskSpace {
+            val store = Files.getFileStore(dir)
+            return DiskSpace(
+                totalBytes = store.totalSpace,
+                usedBytes = store.totalSpace - store.unallocatedSpace,
+                freeBytes = store.usableSpace,
+            )
+        }
+
+        /** File-walk replacement for the legacy `du -sk`; unreadable subtrees are skipped, links are not followed. */
+        fun directorySizeBytes(dir: Path): Long {
+            var size = 0L
+            Files.walkFileTree(
+                dir,
+                object : SimpleFileVisitor<Path>() {
+                    override fun visitFile(
+                        file: Path,
+                        attrs: BasicFileAttributes,
+                    ): FileVisitResult {
+                        if (attrs.isRegularFile) {
+                            size += attrs.size()
+                        }
+                        return FileVisitResult.CONTINUE
+                    }
+
+                    override fun visitFileFailed(
+                        file: Path,
+                        exc: IOException,
+                    ): FileVisitResult = FileVisitResult.CONTINUE
+                },
+            )
+            return size
+        }
+    }
+}
@@ -0,0 +1,28 @@
+package de.hoennig.gittally.server
+
+import de.hoennig.gittally.metrics.SystemMetricsCollector
+import jakarta.annotation.PreDestroy
+import org.springframework.boot.context.event.ApplicationReadyEvent
+import org.springframework.context.annotation.Profile
+import org.springframework.context.event.EventListener
+import org.springframework.stereotype.Component
+
+/**
+ * Starts the system-metrics sampling loop once the server context is ready and
+ * stops it on shutdown. Only in the `server` profile, like [ServerWatcherLifecycle].
+ */
+@Component
+@Profile("server")
+class ServerMetricsLifecycle(
+    private val collector: SystemMetricsCollector,
+) {
+    @EventListener(ApplicationReadyEvent::class)
+    fun onApplicationReady() {
+        collector.start()
+    }
+
+    @PreDestroy
+    fun onShutdown() {
+        collector.stop()
+    }
+}
@@ -0,0 +1,18 @@
+package de.hoennig.gittally.server
+
+import de.hoennig.gittally.metrics.SystemMetrics
+import de.hoennig.gittally.metrics.SystemMetricsCollector
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+class SystemApiController(
+    private val collector: SystemMetricsCollector,
+) {
+    /**
+     * The current system snapshot plus min/max/avg aggregates (the legacy
+     * `system.json`). Unavailable metrics are null, never an error.
+     */
+    @GetMapping("/api/system")
+    fun system(): SystemMetrics = collector.snapshot()
+}
@@ -5,6 +5,7 @@ import de.hoennig.gittally.build.BuildExecutor
 import de.hoennig.gittally.build.BuildResultRepository
 import de.hoennig.gittally.build.BuildStatus
 import de.hoennig.gittally.config.ConfigLoader
+import de.hoennig.gittally.metrics.SystemMetricsCollector
 import org.springframework.beans.factory.ObjectProvider
 import org.springframework.boot.info.BuildProperties
 import org.springframework.http.HttpStatus
@@ -32,6 +33,7 @@ class UiController(
    private val artifactStore: ArtifactStore,
    private val controlTokens: ControlTokenService,
    private val configLoader: ConfigLoader,
+    private val metricsCollector: SystemMetricsCollector,
    private val buildProperties: ObjectProvider<BuildProperties>,
 ) {
    var workingDir: Path = Paths.get(".")
@@ -80,6 +82,13 @@ class UiController(
        return "current"
    }

+    @GetMapping("/system")
+    fun system(model: Model): String {
+        baseModel(model, view = "system", pageTitle = "System Metrics")
+        model.addAttribute("metrics", SystemMetricsView.from(metricsCollector.snapshot()))
+        return "system"
+    }
+
    /** Artifact index rendered from the artifact store — legacy pre-generated this page as static HTML. */
    @GetMapping("/builds/{artifactKey}")
    fun artifactIndex(
@@ -2,12 +2,15 @@ package de.hoennig.gittally.server

 import de.hoennig.gittally.build.BuildResult
 import de.hoennig.gittally.config.GiteaConfig
+import de.hoennig.gittally.metrics.MetricAggregate
+import de.hoennig.gittally.metrics.SystemMetrics
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets
 import java.time.Duration
 import java.time.Instant
 import java.time.ZoneId
 import java.time.format.DateTimeFormatter
+import java.util.Locale

 /** Links into the Gitea web UI, like legacy `gitea_branch_web_url`; null when Gitea is not configured. */
 class GiteaWebLinks(
@@ -33,6 +36,8 @@ class GiteaWebLinks(
 object UiFormats {
    private val timestampFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm").withZone(ZoneId.systemDefault())

+    private val timeOfDayFormat = DateTimeFormatter.ofPattern("HH:mm:ss").withZone(ZoneId.systemDefault())
+
    fun timestamp(instant: Instant): String = timestampFormat.format(instant)

    /** `m:ss`, or `h:mm:ss` from one hour — like the legacy `MM:SS` duration column. */
@@ -50,6 +55,16 @@ object UiFormats {
            "%d:%02d".format(minutes, rest)
        }
    }
+
+    /** Two-decimal metric value with a dot like JS `toFixed(2)`; `n/a` when the source is unavailable. */
+    fun metric(value: Double?): String =
+        if (value == null || !value.isFinite()) {
+            "n/a"
+        } else {
+            String.format(Locale.ROOT, "%.2f", value)
+        }
+
+    fun timeOfDay(instant: Instant): String = timeOfDayFormat.format(instant)
 }

 /** One row of the latest/history build tables. */
@@ -96,3 +111,60 @@ data class CurrentBuildView(
    val branchUrl: String?,
    val commitUrl: String?,
 )
+
+/**
+ * One row of the system-metrics table. The [key] matches the JSON field of
+ * `GET /api/system`, so `gittally.js` can update the cells in place.
+ */
+data class MetricRowView(
+    val key: String,
+    val label: String,
+    val current: String,
+    val min: String,
+    val max: String,
+    val avg: String,
+) {
+    companion object {
+        fun from(
+            key: String,
+            label: String,
+            aggregate: MetricAggregate?,
+        ) = MetricRowView(
+            key = key,
+            label = label,
+            current = UiFormats.metric(aggregate?.current),
+            min = UiFormats.metric(aggregate?.min),
+            max = UiFormats.metric(aggregate?.max),
+            avg = UiFormats.metric(aggregate?.avg),
+        )
+    }
+}
+
+/** The system view: the metric rows plus the totals/updated info line, like the legacy system page. */
+data class SystemMetricsView(
+    val rows: List<MetricRowView>,
+    val cpuCount: String,
+    val ramTotal: String,
+    val diskTotal: String,
+    val updated: String,
+) {
+    companion object {
+        fun from(metrics: SystemMetrics) =
+            SystemMetricsView(
+                rows =
+                    listOf(
+                        MetricRowView.from("cpuUsed", "CPU used (cores)", metrics.cpuUsed),
+                        MetricRowView.from("cpuIdle", "CPU idle (cores)", metrics.cpuIdle),
+                        MetricRowView.from("ramUsedGib", "RAM used (GiB)", metrics.ramUsedGib),
+                        MetricRowView.from("ramFreeGib", "RAM free (GiB)", metrics.ramFreeGib),
+                        MetricRowView.from("diskUsedGib", "Disk used (GiB)", metrics.diskUsedGib),
+                        MetricRowView.from("diskFreeGib", "Disk free (GiB)", metrics.diskFreeGib),
+                        MetricRowView.from("repoSizeGib", "Repo size (GiB)", metrics.repoSizeGib),
+                    ),
+                cpuCount = "${metrics.cpuCount} cores",
+                ramTotal = metrics.ramTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a",
+                diskTotal = metrics.diskTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a",
+                updated = metrics.timestamp?.let { UiFormats.timeOfDay(it) } ?: "n/a",
+            )
+    }
+}