diff --git a/.claude/skills/architecture/SKILL.md b/.claude/skills/architecture/SKILL.md index 6a8c131..5c28ac0 100644 --- a/.claude/skills/architecture/SKILL.md +++ b/.claude/skills/architecture/SKILL.md @@ -83,3 +83,5 @@ After the enqueue decision — and only after it, because a local ref lagging be ## System Metrics `SystemMetricsCollector` samples CPU (`/proc/stat` deltas), RAM (`/proc/meminfo`), disk, and repository size every 60s, but only after `ServerMetricsLifecycle` (server profile) calls `start()` — like the watcher, nothing is scheduled in CLI runs or tests. Min/max/avg aggregation state persists as JSON in the artifact root (`ArtifactStore.rootDir()`), so restarts continue the series. Unavailable sources (e.g. no `/proc` outside Linux) yield null metrics served as HTTP 200 by `GET /api/system` — the `/system` page shows `n/a`, never an error. + +The disk metric is the tightest of the user quota, the group quota, and the volume itself (`DiskQuota`, PR#16): `readDisk()` shells out `quota -u -g --no-wrap --raw-grace` next to the `Files.getFileStore` read, parses it into candidate budgets, and `bindingDiskSpace` picks whichever has the smallest headroom — the volume by default, a quota on a Hostsharing Managed Webspace where it is the actual limit (`werkdock doctor` already reads the same number in Go, as a one-off pre-build check). `diskTotalGib`/`diskUsedGib`/`diskFreeGib` always come from that one binding source; `SystemMetrics.diskSource` names it (`UiFormats.diskTotal`/`werkator.js`'s `formatDiskTotal` render the same info-line suffix), and a source change resets the disk min/max/avg (`PersistedMetricsState.diskSourceKey`) so a stale volume-sized ceiling never survives a newly detected quota. diff --git a/docs/deployment.md b/docs/deployment.md index d1c4191..f525deb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -345,3 +345,5 @@ tools/remote --env-file .env.mih34 werkator instance-update ``` The previous runtime stays as `.werkator/werkator.prev` for one deployment as the rollback asset. + +The `/system` page's disk metric is quota-aware (PR#16): on a Managed Webspace the binding limit is usually the package's group quota, not the free space of the shared host volume, so `diskTotalGib` there is the quota's soft limit — the info line names it (`group quota , hard limit … GiB`) instead of showing the host's full disk size. diff --git a/docs/plan/09-system-metrics.md b/docs/plan/09-system-metrics.md index 6e5bc8b..aa2c661 100644 --- a/docs/plan/09-system-metrics.md +++ b/docs/plan/09-system-metrics.md @@ -55,6 +55,8 @@ Deviations and decisions: - The legacy `generation` field was not ported; it only guarded the legacy JS against monitor restarts. - The CPU count comes from `Runtime.availableProcessors()` instead of `nproc`. +**Implementation note (PR#16, 2026-09-03):** the disk metric is now the tightest of the user quota, the group quota, and the volume, not the volume alone — `DiskQuota` parses `quota -u -g --no-wrap --raw-grace` and `SystemMetricsCollector.readDisk()` picks whichever candidate has the smallest headroom. On hosts without a binding quota (Docker hosts, developer machines) nothing changes; on a Hostsharing Managed Webspace the group quota is usually the real limit, so `diskTotalGib` there is the quota's soft limit instead of the host volume's size, and the info line names the source. A source change (volume → quota, or one quota subject to another) resets `diskUsedGib`/`diskFreeGib`'s min/max/avg so a stale ceiling from the previous source never survives. + Manual smoke test (2026-07-07): scratch repository with a bare origin, server on port 18986, observed through a real browser tab (via a TCP proxy, so the tab outlived backend restarts). The first sample rendered RAM/disk/repo values immediately with CPU `n/a` and the totals line (`8 cores`, RAM/disk GiB, updated time). After the next 60s poll the open tab updated in place without reload: the updated time ticked, CPU used appeared (1.58 cores, idle 6.42 = 8 total), and min/max diverged. diff --git a/docs/prs/2026-09-03-PR#16-quota-aware-disk-metrics.md b/docs/prs/2026-09-03-PR#16-quota-aware-disk-metrics.md index 0352f3e..07c422f 100644 --- a/docs/prs/2026-09-03-PR#16-quota-aware-disk-metrics.md +++ b/docs/prs/2026-09-03-PR#16-quota-aware-disk-metrics.md @@ -23,7 +23,7 @@ The system page should show the same truth, continuously. ## Non-Goals -- Implementing the change — this PR is the plan only; the implementation follows in the next PR. +- Deploying to `mih09` and verifying it live — the code lands in this PR, the rollout is a separate, later step (see "Where it is verified live" below). - Inode (file-count) quotas: `quota(1)` reports them, but the Gradle caches on `mih09` use 14 226 of 16.7 M files; a follow-up if it ever matters. - Alerting or refusing to start a build on a full quota — the page only shows; `werkdock doctor` keeps the one-off pre-build check. - A configuration switch: the quota is detected, never declared (see Open Questions). @@ -58,8 +58,8 @@ So that the operator of a Managed Webspace sees the budget the package can fill, ##### Verified by -- [SystemMetricsCollectorTest — "a group quota on the repository's filesystem replaces the file-store disk numbers"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) (planned) -- [DiskQuotaTest — "the mih09 output parses into one group line per filesystem and no user line"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) (planned, fixture: the attachment below) +- [SystemMetricsCollectorTest — "a group quota on the repository's filesystem replaces the file-store disk numbers"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) +- [DiskQuotaTest — "the mih09 output parses into one group line per filesystem and no user line"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) (fixture: the attachment below) #### Scenario#16.02: The tightest of user quota, group quota and volume binds @@ -74,7 +74,7 @@ So that neither a user quota below the group's, nor a nearly full host volume be ##### Verified by -- [DiskQuotaTest — "among user quota, group quota and volume the smallest headroom binds"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) (planned) +- [DiskQuotaTest — "among user quota, group quota and volume the smallest headroom binds"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) #### Scenario#16.03: Only the quota of the repository's filesystem counts @@ -88,7 +88,7 @@ So that a full quota on another volume (on `mih09`: `/dev/sdb1`) does not shrink ##### Verified by -- [DiskQuotaTest — "only the lines of the directory's file store are considered, matched exactly or by device name"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) (planned) +- [DiskQuotaTest — "only the lines of the directory's file store are considered, matched exactly or by device name"](../../src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt) #### Scenario#16.04: Without a quota the volume stays the source @@ -102,7 +102,7 @@ So that hosts without quota tooling, without a quota, or with an unreadable `quo ##### Verified by -- [SystemMetricsCollectorTest — "without a quota the volume stays the disk source"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) (planned) +- [SystemMetricsCollectorTest — "without a quota the volume stays the disk source"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) - [SystemMetricsCollectorTest — "unreadable sources degrade to null metrics, never fail the sample"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) (existing, extended by the quota source) #### Scenario#16.05: A changed disk source restarts the disk series @@ -117,7 +117,7 @@ So that the min/max/avg of a 1 GiB quota metric are not poisoned by the 37 GiB v ##### Verified by -- [SystemMetricsCollectorTest — "a changed disk source restarts the disk series and keeps the others"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) (planned) +- [SystemMetricsCollectorTest — "a changed disk source restarts the disk series and keeps the others"](../../src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt) #### Scenario#16.06: The page says which budget it shows @@ -133,7 +133,7 @@ So that `Disk total: 8.00 GiB` on a 71 GiB host is not mistaken for a broken met ##### Verified by -- [UiViewsTest — "the disk total names the binding source: user quota, group quota, or the volume"](../../src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt) (planned) +- [UiViewsTest — "the disk total names the binding source: user quota, group quota, or the volume"](../../src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt) - `werkator.js` mirrors `UiFormats.diskTotal` (manual: the polled line must equal the rendered one after the first refresh) #### Scenario#16.07: The highlighting follows the quota @@ -150,7 +150,7 @@ So that the warn/critical colours fire before a build hits "Disk quota exceeded" ## The Solution -This PR records the plan; the code lands in the next PR. +The implementation follows the plan below with one shape difference: `DiskSpace` itself stays the plain `{totalBytes, usedBytes, freeBytes}` value it already was, and a new `DiskCandidate(space, source)` pairs it with a `DiskSource` only where a source needs naming (quota parsing, the binding choice, the collector's disk reading) — kept `DiskSpace` reusable by the unchanged volume-only tests instead of every caller now supplying a source. **Read the quota through the CLI, like git and Docker.** Linux exposes quotas only through the `quotactl` syscall, which Java cannot reach without JNI/JNA — a new runtime dependency and a native layer for one number. @@ -161,13 +161,13 @@ This is the same decision `werkdock doctor` took in Go; the two parsers stay sep **Choose the binding candidate in a pure function.** `DiskQuota` (new, package `de.hoennig.werkator.metrics`) parses the output into lines `{kind user|group, subject, filesystem, blocksKib, softKib, hardKib}` and keeps the lines of the directory's file store (`Files.getFileStore(dir).name()` is the mount's device string, the same string `quota` prints; a resolved device path is matched by its last segment as `werkdock` does). -Each remaining line becomes a candidate `DiskSpace` with `total = soft`, `used = blocks`, `free = max(0, total − blocks)`; a line whose soft limit is 0 (unset) uses the hard limit as total, a line with both 0 is no candidate. -The volume's `fileStoreDiskSpace(dir)` is the last candidate, and `bindingDiskSpace(candidates)` returns the one with the smallest `free` — the tightest budget wins, and total, used and free always come from that one source. +Each remaining line becomes a `DiskCandidate` with `total = soft`, `used = blocks`, `free = max(0, total − blocks)`; a line whose soft limit is 0 (unset) uses the hard limit as total, a line with both 0 is no candidate. +The volume's `fileStoreDiskSpace(dir)`, wrapped as `DiskCandidate` with `DiskSource.volume()`, is the last candidate, and `bindingDiskSpace(candidates)` returns the one with the smallest `free` — the tightest budget wins, and total, used and free always come from that one source. All of it is pure over strings and numbers, so the whole matrix — user only, group only, both, none, volume tighter than the quotas, two filesystems, `*` marker, `none` line — is a Kotest table. **The collector gets one more injectable source.** -`SystemMetricsCollector` gains `quotaOutput: () -> String?` next to `diskSpace` (the process call with a 5 s timeout in production, a string in tests); `readDisk()` collects the quota candidates plus the file store and takes the binding one — a failing `quota` simply leaves the volume as the only candidate. -`DiskSpace` gains a `source: DiskSource` (`kind` `volume|user|group`, `subject`, `filesystem`, and for a quota `softLimitGib`/`hardLimitGib`), carried into `SystemMetrics.diskSource` — an additive JSON field, the three existing disk fields keep their names; `quotasPresent: Boolean` says whether a quota lost against the volume, for the info line. +`SystemMetricsCollector` gains `quotaOutput: () -> String?` and `fileStoreName: (Path) -> String` next to `diskSpace` (the process call with a 5 s timeout in production, a fixed string in tests); `readDisk()` collects the quota candidates plus the file store and takes the binding one — a failing `quota` is read under its own `readSource("quota")`, logged once and separately from a failing file-store read, and simply leaves the volume as the only candidate. +The binding candidate's `DiskSource` (`kind` `volume|user|group`, `subject`, `filesystem`, and for a quota `softLimitGib`/`hardLimitGib`) is carried into `SystemMetrics.diskSource` — an additive JSON field, the three existing disk fields keep their names; `quotasPresent: Boolean` says whether a quota lost against the volume, for the info line. The persisted state gains `diskSource` (`"volume"` or `"quota:::"`); a mismatch drops the two disk series before the sample is recorded (Scenario#16.05). That reset also fires when the binding candidate switches at runtime, e.g. from the group quota to a newly introduced user quota — the series then describe one budget at a time. The quota is read every sample: it is one syscall behind a small process, cheaper than the repo-size walk, and a raised quota should show within a minute. @@ -176,18 +176,19 @@ The quota is read every sample: it is one syscall behind a small process, cheape `UiFormats.diskTotal(metrics)` formats `8.00 GiB (group quota mih09, hard limit 12.00 GiB)`, `… (user quota …)`, `70.99 GiB (volume, tighter than the quotas)` or the plain total when no quota exists; `werkator.js` gets the identical function for the poll — the UI invariant that server-rendered and polled output match. Rows, labels and the highlighting stay as they are: `utilizationClass(used, total)` simply receives the quota as the total. -**Where it is verified live.** +**Where it is verified live — pending.** After the deployment on `mih09` the page must read `Disk total: 8.00 GiB (group quota mih09, hard limit 12.00 GiB)`, `Disk used` about 1.04 GiB and `Disk free` about 6.96 GiB, with the `Repo size` row unchanged at about 0.73 GiB — the used value is the whole package's usage (every user of group `mih09`), which is what counts against the budget, while the repo size stays Werkator's own share. On `vm4006` (Docker host, no quota) the page must render exactly as before. The first sample after the update restarts the disk min/max/avg, visible as `Max` dropping from 37.13 GiB to the current value. +Not yet done as of this PR — step 5 below is still open. **Order of work for the implementing PR:** -1. `DiskQuota` parser and selection with the table test and the `mih09` fixture. -2. `SystemMetricsCollector`: the quota source, the fallback, `diskSource` in the state, the series reset. -3. `SystemMetrics.diskSource`, `UiFormats.diskTotal`, `SystemMetricsView`, `werkator.js`, `UiViewsTest`. -4. Docs: the metrics paragraph of the architecture skill, one sentence in `docs/deployment.md` (Hostsharing section) and in `docs/plan/09-system-metrics.md` (implementation note), and this PR-doc's "Verified by" links turned from planned into real. -5. Deploy to `mih09` via `tools/remote --env-file .env.mih09 werkator instance-update`, check the page and the journal for the one-time source log line. +1. ✅ `DiskQuota` parser and selection with the table test and the `mih09` fixture. +2. ✅ `SystemMetricsCollector`: the quota source, the fallback, `diskSource` in the state, the series reset. +3. ✅ `SystemMetrics.diskSource`, `UiFormats.diskTotal`, `SystemMetricsView`, `werkator.js`, `UiViewsTest`. +4. ✅ Docs: the metrics paragraph of the architecture skill, one sentence in `docs/deployment.md` (Hostsharing section) and in `docs/plan/09-system-metrics.md` (implementation note), and this PR-doc's "Verified by" links turned from planned into real. +5. ⬜ Deploy to `mih09` via `tools/remote --env-file .env.mih09 werkator instance-update`, check the page and the journal for the one-time source log line. ## Open Questions diff --git a/src/main/kotlin/de/hoennig/werkator/metrics/DiskQuota.kt b/src/main/kotlin/de/hoennig/werkator/metrics/DiskQuota.kt new file mode 100644 index 0000000..a9969f3 --- /dev/null +++ b/src/main/kotlin/de/hoennig/werkator/metrics/DiskQuota.kt @@ -0,0 +1,133 @@ +package de.hoennig.werkator.metrics + +/** One `blocks`/`quota`/`limit` line of `quota -u -g --no-wrap --raw-grace`, KiB throughout. */ +data class QuotaLine( + val kind: String, + val subject: String, + val filesystem: String, + val blocksKib: Long, + val softKib: Long, + val hardKib: Long, +) + +/** + * Where the binding [SystemMetricsCollector.DiskSpace] came from: the volume itself, or a + * user/group quota. `kind` is `"volume"`, `"user"` or `"group"`; `subject`/`filesystem`/the + * limits are set only for a quota. Serialized as-is into `GET /api/system`. + */ +data class DiskSource( + val kind: String, + val subject: String? = null, + val filesystem: String? = null, + val softLimitGib: Double? = null, + val hardLimitGib: Double? = null, +) { + companion object { + fun volume() = DiskSource(kind = "volume") + } +} + +/** One candidate budget for a directory: a quota line or the volume, each with its source. */ +data class DiskCandidate( + val space: SystemMetricsCollector.DiskSpace, + val source: DiskSource, +) + +/** + * Parses `quota -u -g --no-wrap --raw-grace` and picks the tightest of user quota, group + * quota, and the volume — pure functions over strings and numbers, see PR#16. + */ +object DiskQuota { + private val subjectHeader = Regex("""^Disk quotas for (user|group) (\S+) \([ug]id \d+\):\s*(none)?\s*$""") + + /** `null`/blank output, and a subject reported `none`, both yield no line for that subject. */ + fun parse(output: String): List { + val lines = mutableListOf() + var kind: String? = null + var subject: String? = null + for (rawLine in output.lines()) { + val header = subjectHeader.find(rawLine.trimEnd()) + if (header != null) { + kind = header.groupValues[1] + subject = header.groupValues[2] + continue + } + val currentKind = kind ?: continue + val currentSubject = subject ?: continue + val trimmed = rawLine.trim() + if (trimmed.isEmpty() || trimmed.startsWith("Filesystem")) { + continue + } + val fields = trimmed.split(Regex("\\s+")) + if (fields.size < 4) { + continue + } + val blocksKib = fields[1].trimEnd('*').toLongOrNull() ?: continue + val softKib = fields[2].toLongOrNull() ?: continue + val hardKib = fields[3].toLongOrNull() ?: continue + lines += + QuotaLine( + kind = currentKind, + subject = currentSubject, + filesystem = fields[0], + blocksKib = blocksKib, + softKib = softKib, + hardKib = hardKib, + ) + } + return lines + } + + /** + * The lines whose filesystem matches [directoryFilesystem] — exactly, or by the last path + * segment when one side is a resolved device path (`/dev/sdb1` vs `/dev/disk/by-id/…`) — each + * turned into a candidate. A line with no soft and no hard limit (both zero) is no candidate; + * a line with only a hard limit uses it as the total. + */ + fun candidatesFor( + lines: List, + directoryFilesystem: String, + ): List = + lines + .filter { matchesFilesystem(it.filesystem, directoryFilesystem) } + .mapNotNull { line -> + val softBytes = line.softKib * BYTES_PER_KIB + val hardBytes = line.hardKib * BYTES_PER_KIB + val totalBytes = if (softBytes > 0) softBytes else hardBytes + if (totalBytes <= 0) { + return@mapNotNull null + } + val usedBytes = line.blocksKib * BYTES_PER_KIB + val freeBytes = maxOf(0L, totalBytes - usedBytes) + DiskCandidate( + space = + SystemMetricsCollector.DiskSpace( + totalBytes = totalBytes, + usedBytes = usedBytes, + freeBytes = freeBytes, + ), + source = + DiskSource( + kind = line.kind, + subject = line.subject, + filesystem = line.filesystem, + softLimitGib = softBytes / BYTES_PER_GIB, + hardLimitGib = hardBytes / BYTES_PER_GIB, + ), + ) + } + + private fun matchesFilesystem( + quotaFilesystem: String, + directoryFilesystem: String, + ): Boolean = + quotaFilesystem == directoryFilesystem || + quotaFilesystem.substringAfterLast('/') == directoryFilesystem.substringAfterLast('/') + + /** The candidate with the smallest headroom; the tightest budget always wins. */ + fun bindingDiskSpace(candidates: List): DiskCandidate = candidates.minBy { it.space.freeBytes } + + private const val BYTES_PER_KIB = 1024L + + private const val BYTES_PER_GIB = 1_073_741_824.0 +} diff --git a/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetrics.kt b/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetrics.kt index 633b074..48e083b 100644 --- a/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetrics.kt +++ b/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetrics.kt @@ -22,6 +22,10 @@ data class SystemMetrics( val cpuCount: Int, val ramTotalGib: Double?, val diskTotalGib: Double?, + /** What `diskTotalGib`/`diskUsedGib`/`diskFreeGib` describe: the volume, or a binding quota. */ + val diskSource: DiskSource? = null, + /** True when a user or group quota existed but the volume was still the tighter budget. */ + val quotasPresent: Boolean = false, val cpuUsed: MetricAggregate?, val cpuIdle: MetricAggregate?, val ramUsedGib: MetricAggregate?, diff --git a/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollector.kt b/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollector.kt index 4c03841..fe394fb 100644 --- a/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollector.kt +++ b/src/main/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollector.kt @@ -19,10 +19,15 @@ import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit -/** The aggregation state persisted in the artifact root (the legacy `system_state.dat`, but as JSON). */ +/** + * The aggregation state persisted in the artifact root (the legacy `system_state.dat`, but as JSON). + * [diskSourceKey] defaults to `"volume"` — the only source before quota-awareness existed — so an + * older state file upgrades cleanly: it only triggers the disk-series reset when a quota now binds. + */ data class PersistedMetricsState( val sampleCount: Long = 0, val series: Map = emptyMap(), + val diskSourceKey: String = "volume", ) /** @@ -42,6 +47,8 @@ class SystemMetricsCollector( private val procMeminfo: Path = Paths.get("/proc/meminfo"), private val cpuCount: Int = Runtime.getRuntime().availableProcessors(), private val diskSpace: (Path) -> DiskSpace = { dir -> fileStoreDiskSpace(dir) }, + private val fileStoreName: (Path) -> String = { dir -> Files.getFileStore(dir).name() }, + private val quotaOutput: () -> String? = { readQuotaOutput() }, private val repoSizeBytes: (Path) -> Long = { dir -> directorySizeBytes(dir) }, ) { /** Disk usage in bytes; used/free follow `df` semantics (free is what a user can still allocate). */ @@ -122,6 +129,13 @@ class SystemMetricsCollector( val cpu = readCpuLoad() val ram = readRam() val disk = readDisk() + val diskSourceKey = disk?.let { diskSourceKey(it.source) } ?: previousState.diskSourceKey + if (disk != null && diskSourceKey != previousState.diskSourceKey) { + // the previous binary/binding source measured a different budget (Scenario#16.05): + // continuing its min/max/avg would poison the new series with a stale ceiling + series.remove("diskUsedGib") + series.remove("diskFreeGib") + } val repoSizeGib = readRepoSizeThrottled() snapshot = SystemMetrics( @@ -129,19 +143,25 @@ class SystemMetricsCollector( sampleCount = sampleCount, cpuCount = cpuCount, ramTotalGib = ram?.totalGib, - diskTotalGib = disk?.totalBytes?.let { it / BYTES_PER_GIB }, + diskTotalGib = disk?.space?.totalBytes?.let { it / BYTES_PER_GIB }, + diskSource = disk?.source, + quotasPresent = disk?.quotasPresent ?: false, cpuUsed = record("cpuUsed", cpu?.usedCores), cpuIdle = record("cpuIdle", cpu?.idleCores), ramUsedGib = record("ramUsedGib", ram?.usedGib), ramFreeGib = record("ramFreeGib", ram?.freeGib), - diskUsedGib = record("diskUsedGib", disk?.usedBytes?.let { it / BYTES_PER_GIB }), - diskFreeGib = record("diskFreeGib", disk?.freeBytes?.let { it / BYTES_PER_GIB }), + diskUsedGib = record("diskUsedGib", disk?.space?.usedBytes?.let { it / BYTES_PER_GIB }), + diskFreeGib = record("diskFreeGib", disk?.space?.freeBytes?.let { it / BYTES_PER_GIB }), repoSizeGib = record("repoSizeGib", repoSizeGib), ) - state = PersistedMetricsState(sampleCount = sampleCount, series = series) + state = PersistedMetricsState(sampleCount = sampleCount, series = series, diskSourceKey = diskSourceKey) saveState(state!!) } + /** `"volume"`, or `"quota:::"` — the reset trigger of Scenario#16.05. */ + private fun diskSourceKey(source: DiskSource): String = + if (source.kind == "volume") "volume" else "quota:${source.kind}:${source.subject}:${source.filesystem}" + private fun sampleSafely() { try { sample() @@ -157,6 +177,8 @@ class SystemMetricsCollector( cpuCount = cpuCount, ramTotalGib = null, diskTotalGib = null, + diskSource = null, + quotasPresent = false, cpuUsed = null, cpuIdle = null, ramUsedGib = null, @@ -240,7 +262,27 @@ class SystemMetricsCollector( .removeSuffix(" kB") .toLong() - private fun readDisk(): DiskSpace? = readSource("disk") { diskSpace(repoDirs().first()) } + /** Disk reading plus whether a quota lost against the volume, for the info line (Scenario#16.06). */ + private data class DiskReading( + val space: DiskSpace, + val source: DiskSource, + val quotasPresent: Boolean, + ) + + /** + * The tightest of the user quota, the group quota, and the volume itself (Scenario#16.02): + * a failing or absent `quota` binary just leaves the volume as the only candidate, logged + * once under its own "quota" source, separately from a failing file-store read. + */ + private fun readDisk(): DiskReading? = + readSource("disk") { + val dir = repoDirs().first() + val volume = DiskCandidate(diskSpace(dir), DiskSource.volume()) + val quotaLines = readSource("quota") { quotaOutput()?.let(DiskQuota::parse) ?: emptyList() } ?: emptyList() + val quotaCandidates = DiskQuota.candidatesFor(quotaLines, fileStoreName(dir)) + val binding = DiskQuota.bindingDiskSpace(quotaCandidates + volume) + DiskReading(space = binding.space, source = binding.source, quotasPresent = quotaCandidates.isNotEmpty()) + } /** * The repository size is expensive to determine (a full file walk), so unlike @@ -325,6 +367,26 @@ class SystemMetricsCollector( ) } + /** + * `quota -u -g --no-wrap --raw-grace`: `--no-wrap` keeps long device names on one line, + * `--raw-grace` prints the grace columns as numbers, so every filesystem line has the + * same nine fields ([DiskQuota] needs no column heuristics). The exit status is not read — + * `quota` also uses it to say "over quota" — only the parsed output counts; an absent + * binary throws (caught by the caller's `readSource`, logged once), a hung process is + * killed after 5s and treated the same way. + */ + fun readQuotaOutput(): String { + val process = + ProcessBuilder("quota", "-u", "-g", "--no-wrap", "--raw-grace") + .redirectErrorStream(true) + .start() + if (!process.waitFor(5, TimeUnit.SECONDS)) { + process.destroyForcibly() + error("quota command timed out") + } + return process.inputStream.bufferedReader().readText() + } + /** File-walk replacement for the legacy `du -sk`; unreadable subtrees are skipped, links are not followed. */ fun directorySizeBytes(dir: Path): Long { var size = 0L diff --git a/src/main/kotlin/de/hoennig/werkator/server/UiViews.kt b/src/main/kotlin/de/hoennig/werkator/server/UiViews.kt index 479a747..75b14b4 100644 --- a/src/main/kotlin/de/hoennig/werkator/server/UiViews.kt +++ b/src/main/kotlin/de/hoennig/werkator/server/UiViews.kt @@ -88,6 +88,22 @@ object UiFormats { private const val UTILIZATION_WARN = 0.80 private const val UTILIZATION_CRIT = 0.90 + + /** + * `8.00 GiB (group quota mih09, hard limit 12.00 GiB)`, `70.99 GiB (volume, tighter than the + * quotas)`, or the plain total when no quota exists — `werkator.js` (`formatDiskTotal`) mirrors + * this exactly, since it renders the same field from the polled `GET /api/system`. + */ + fun diskTotal(metrics: SystemMetrics): String { + val total = metrics.diskTotalGib ?: return "n/a" + val totalText = "${metric(total)} GiB" + val source = metrics.diskSource + if (source == null || source.kind == "volume") { + return if (metrics.quotasPresent) "$totalText (volume, tighter than the quotas)" else totalText + } + val hardLimit = metric(source.hardLimitGib) + return "$totalText (${source.kind} quota ${source.subject}, hard limit $hardLimit GiB)" + } } /** One row of the build tables; [latestGreenUrl] only on the build that permanent link resolves to. */ @@ -247,7 +263,7 @@ data class SystemMetricsView( ), cpuCount = "${metrics.cpuCount} cores", ramTotal = metrics.ramTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a", - diskTotal = metrics.diskTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a", + diskTotal = UiFormats.diskTotal(metrics), updated = metrics.timestamp?.let { UiFormats.timeOfDay(it) } ?: "n/a", ) } diff --git a/src/main/resources/static/werkator.js b/src/main/resources/static/werkator.js index f1c7ddb..d71a862 100644 --- a/src/main/resources/static/werkator.js +++ b/src/main/resources/static/werkator.js @@ -599,6 +599,20 @@ function initCurrentBuilds() { /** The total each utilization metric is compared against for the critical highlighting. */ const UTILIZATION_TOTALS = { cpuUsed: "cpuCount", ramUsedGib: "ramTotalGib", diskUsedGib: "diskTotalGib" }; +/** Mirrors UiFormats.diskTotal exactly, since both render the same field from GET /api/system. */ +function formatDiskTotal(metrics) { + if (metrics.diskTotalGib == null) { + return "n/a"; + } + const totalText = formatMetric(metrics.diskTotalGib) + " GiB"; + const source = metrics.diskSource; + if (!source || source.kind === "volume") { + return metrics.quotasPresent ? totalText + " (volume, tighter than the quotas)" : totalText; + } + const hardLimit = formatMetric(source.hardLimitGib); + return totalText + " (" + source.kind + " quota " + source.subject + ", hard limit " + hardLimit + " GiB)"; +} + /** Same thresholds as UiFormats.utilizationClass: warn from 80% of the total, critical from 90%. */ function utilizationClass(used, total) { if (used == null || total == null || !(total > 0)) { @@ -647,7 +661,7 @@ function initSystemTable() { }); setText("info-cpu-count", metrics.cpuCount != null ? metrics.cpuCount + " cores" : "n/a"); setText("info-ram-total", metrics.ramTotalGib != null ? formatMetric(metrics.ramTotalGib) + " GiB" : "n/a"); - setText("info-disk-total", metrics.diskTotalGib != null ? formatMetric(metrics.diskTotalGib) + " GiB" : "n/a"); + setText("info-disk-total", formatDiskTotal(metrics)); setText("info-updated", formatTimeOfDay(metrics.timestamp)); } diff --git a/src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt b/src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt new file mode 100644 index 0000000..97430f5 --- /dev/null +++ b/src/test/kotlin/de/hoennig/werkator/metrics/DiskQuotaTest.kt @@ -0,0 +1,153 @@ +package de.hoennig.werkator.metrics + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe + +/** `quota -u -g --no-wrap --raw-grace` on mih09, 2026-09-03 (the PR#16 attachment); the parser fixture. */ +private val MIH09_QUOTA_OUTPUT = + """ + Disk quotas for user mih09-werkator (uid 120974): none + Disk quotas for group mih09 (gid 102180): + Filesystem blocks quota limit grace files quota limit grace + /dev/disk/by-id/wwn-0x0000000000000001-part2 1088116 8388608 12582912 0 14226 16777216 25165824 0 + /dev/sdb1 1456376 10485760 15728640 0 52983 20971520 31457280 0 + """.trimIndent() + +class DiskQuotaTest : FunSpec() { + init { + test("the mih09 output parses into one group line per filesystem and no user line") { + val lines = DiskQuota.parse(MIH09_QUOTA_OUTPUT) + + lines shouldHaveSize 2 + lines.none { it.kind == "user" } shouldBe true + lines[0] shouldBe + QuotaLine( + kind = "group", + subject = "mih09", + filesystem = "/dev/disk/by-id/wwn-0x0000000000000001-part2", + blocksKib = 1_088_116, + softKib = 8_388_608, + hardKib = 12_582_912, + ) + lines[1].filesystem shouldBe "/dev/sdb1" + } + + test("no quota tooling, an empty output, and a 'none'-only output all parse to no lines") { + DiskQuota.parse("").shouldBeEmpty() + DiskQuota.parse("Disk quotas for user mih09-werkator (uid 120974): none").shouldBeEmpty() + DiskQuota + .parse( + """ + Disk quotas for user mih09-werkator (uid 120974): none + Disk quotas for group mih09 (gid 102180): none + """.trimIndent(), + ).shouldBeEmpty() + } + + test("only the lines of the directory's file store are considered, matched exactly or by device name") { + val lines = DiskQuota.parse(MIH09_QUOTA_OUTPUT) + + val exact = DiskQuota.candidatesFor(lines, "/dev/disk/by-id/wwn-0x0000000000000001-part2") + exact shouldHaveSize 1 + exact[0].source.filesystem shouldBe "/dev/disk/by-id/wwn-0x0000000000000001-part2" + + val byDeviceName = DiskQuota.candidatesFor(lines, "/dev/sdb1") + byDeviceName shouldHaveSize 1 + byDeviceName[0].source.filesystem shouldBe "/dev/sdb1" + + DiskQuota.candidatesFor(lines, "/dev/mapper/unrelated").shouldBeEmpty() + } + + test("a group quota candidate reports soft limit as total, blocks as used, and the hard limit alongside") { + val lines = DiskQuota.parse(MIH09_QUOTA_OUTPUT) + + val candidates = DiskQuota.candidatesFor(lines, "/dev/disk/by-id/wwn-0x0000000000000001-part2") + + val candidate = candidates.single() + candidate.space.totalBytes shouldBe 8_388_608L * 1024 + candidate.space.usedBytes shouldBe 1_088_116L * 1024 + candidate.space.freeBytes shouldBe (8_388_608L - 1_088_116L) * 1024 + candidate.source.kind shouldBe "group" + candidate.source.subject shouldBe "mih09" + candidate.source.softLimitGib!! shouldBe (8_388_608.0 * 1024 / GIB) + candidate.source.hardLimitGib!! shouldBe (12_582_912.0 * 1024 / GIB) + } + + test("an over-quota '*' marker on the blocks field does not break parsing") { + val output = + """ + Disk quotas for group mih09 (gid 102180): + Filesystem blocks quota limit grace files quota limit grace + /dev/sda1 9000000* 8388608 12582912 604800 14226 16777216 25165824 0 + """.trimIndent() + + val lines = DiskQuota.parse(output) + + lines.single().blocksKib shouldBe 9_000_000 + } + + test("a line with only a hard limit uses it as the total; a line with neither is no candidate") { + val output = + """ + Disk quotas for group mih09 (gid 102180): + Filesystem blocks quota limit grace files quota limit grace + /dev/sda1 1000000 0 2000000 0 1 0 0 0 + /dev/sdc1 1000000 0 0 0 1 0 0 0 + """.trimIndent() + + val lines = DiskQuota.parse(output) + + val hardOnly = DiskQuota.candidatesFor(lines, "/dev/sda1") + hardOnly.single().space.totalBytes shouldBe 2_000_000L * 1024 + + DiskQuota.candidatesFor(lines, "/dev/sdc1").shouldBeEmpty() + } + + test("among user quota, group quota and volume the smallest headroom binds") { + val userQuota = + DiskCandidate( + space = + SystemMetricsCollector.DiskSpace( + totalBytes = 10L * GIB.toLong(), + usedBytes = 8L * GIB.toLong(), + freeBytes = + 2L * GIB.toLong(), + ), + source = DiskSource(kind = "user", subject = "mih09-werkator", filesystem = "/dev/x"), + ) + val groupQuota = + DiskCandidate( + space = + SystemMetricsCollector.DiskSpace( + totalBytes = 15L * GIB.toLong(), + usedBytes = 8L * GIB.toLong(), + freeBytes = + 7L * GIB.toLong(), + ), + source = DiskSource(kind = "group", subject = "mih09", filesystem = "/dev/x"), + ) + val roomyVolume = + DiskCandidate( + space = + SystemMetricsCollector.DiskSpace( + totalBytes = 71L * GIB.toLong(), + usedBytes = 37L * GIB.toLong(), + freeBytes = + 34L * GIB.toLong(), + ), + source = DiskSource.volume(), + ) + val tightVolume = roomyVolume.copy(space = roomyVolume.space.copy(freeBytes = 1L * GIB.toLong())) + + DiskQuota.bindingDiskSpace(listOf(userQuota, groupQuota, roomyVolume)).source.kind shouldBe "user" + DiskQuota.bindingDiskSpace(listOf(groupQuota, roomyVolume)).source.kind shouldBe "group" + DiskQuota.bindingDiskSpace(listOf(groupQuota, tightVolume)).source.kind shouldBe "volume" + } + } + + companion object { + private const val GIB = 1_073_741_824.0 + } +} diff --git a/src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt b/src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt index 3cd9da9..f67eb4a 100644 --- a/src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt +++ b/src/test/kotlin/de/hoennig/werkator/metrics/SystemMetricsCollectorTest.kt @@ -17,6 +17,8 @@ class SystemMetricsCollectorTest : FunSpec() { private fun collector( cpuCount: Int = 4, diskSpace: (Path) -> SystemMetricsCollector.DiskSpace = { GIB_100_DISK }, + fileStoreName: (Path) -> String = { "/dev/volume" }, + quotaOutput: () -> String? = { null }, repoSizeBytes: (Path) -> Long = { HALF_GIB_BYTES }, ) = SystemMetricsCollector( stateFile = { tempDir.resolve("system-metrics-state.json") }, @@ -26,6 +28,8 @@ class SystemMetricsCollectorTest : FunSpec() { procMeminfo = tempDir.resolve("meminfo"), cpuCount = cpuCount, diskSpace = diskSpace, + fileStoreName = fileStoreName, + quotaOutput = quotaOutput, repoSizeBytes = repoSizeBytes, ) @@ -200,6 +204,94 @@ class SystemMetricsCollectorTest : FunSpec() { snapshot.cpuCount shouldBe 4 snapshot.cpuUsed shouldBe null } + + test("a group quota on the repository's filesystem replaces the file-store disk numbers") { + writeStat(total = 1000, idle = 700) + writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824) + val collector = + collector( + diskSpace = { SystemMetricsCollector.DiskSpace(totalBytes = 71L * GIB, usedBytes = 37L * GIB, freeBytes = 34L * GIB) }, + fileStoreName = { MIH09_FILESYSTEM }, + quotaOutput = { MIH09_QUOTA_OUTPUT }, + ) + + collector.sample() + + val snapshot = collector.snapshot() + snapshot.diskTotalGib shouldBe 8_388_608.0 * 1024 / GIB + snapshot.diskUsedGib.shouldNotBeNull().current shouldBe 1_088_116.0 * 1024 / GIB + snapshot.diskFreeGib.shouldNotBeNull().current shouldBe (8_388_608.0 - 1_088_116.0) * 1024 / GIB + snapshot.diskSource.shouldNotBeNull().let { + it.kind shouldBe "group" + it.subject shouldBe "mih09" + it.hardLimitGib shouldBe 12_582_912.0 * 1024 / GIB + } + snapshot.quotasPresent shouldBe true + } + + test("without a quota the volume stays the disk source") { + writeStat(total = 1000, idle = 700) + writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824) + val collector = collector(quotaOutput = { null }) + + collector.sample() + + val snapshot = collector.snapshot() + snapshot.diskTotalGib shouldBe 100.0 + snapshot.diskSource.shouldNotBeNull().kind shouldBe "volume" + snapshot.quotasPresent shouldBe false + } + + test("a failing quota command degrades to the volume, logged once, the sample still succeeds") { + writeStat(total = 1000, idle = 700) + writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824) + val collector = collector(quotaOutput = { error("quota: command not found") }) + + collector.sample() + + val snapshot = collector.snapshot() + snapshot.diskTotalGib shouldBe 100.0 + snapshot.diskSource.shouldNotBeNull().kind shouldBe "volume" + } + + test("a changed disk source restarts the disk series and keeps the others") { + writeStat(total = 1000, idle = 700) + writeMeminfo(totalKib = 33_554_432, availableKib = 25_165_824) + // sample 1, on the legacy volume-only binary: disk used 37 GiB, poisoning any later max + collector( + diskSpace = { SystemMetricsCollector.DiskSpace(totalBytes = 71L * GIB, usedBytes = 37L * GIB, freeBytes = 34L * GIB) }, + ).sample() + + // sample 2, after the update: the first sample to find a binding group quota (~1 GiB used) + val quotaCollector = { output: String -> + collector( + diskSpace = { SystemMetricsCollector.DiskSpace(totalBytes = 71L * GIB, usedBytes = 37L * GIB, freeBytes = 34L * GIB) }, + fileStoreName = { MIH09_FILESYSTEM }, + quotaOutput = { output }, + ) + } + val sample2 = quotaCollector(MIH09_QUOTA_OUTPUT) + sample2.sample() + + val afterUpdate = sample2.snapshot() + afterUpdate.sampleCount shouldBe 2 + val diskUsedAfterUpdate = afterUpdate.diskUsedGib.shouldNotBeNull() + // a fresh series of one sample: min/max are the quota's ~1 GiB, not the 37 GiB volume history + diskUsedAfterUpdate.max shouldBe diskUsedAfterUpdate.current + diskUsedAfterUpdate.min shouldBe diskUsedAfterUpdate.current + // ram, unaffected by the disk-source change, keeps aggregating across both samples + afterUpdate.ramUsedGib.shouldNotBeNull().min shouldBe 8.0 + + // sample 3, same (unchanged) quota source but higher usage: the series must now continue + val sample3 = quotaCollector(MIH09_QUOTA_OUTPUT_HIGHER_USAGE) + sample3.sample() + + val afterRestart = sample3.snapshot() + afterRestart.sampleCount shouldBe 3 + val diskUsedAfterRestart = afterRestart.diskUsedGib.shouldNotBeNull() + diskUsedAfterRestart.min shouldBe diskUsedAfterUpdate.current + diskUsedAfterRestart.max shouldBe (2_000_000.0 * 1024 / GIB) + } } companion object { @@ -213,5 +305,19 @@ class SystemMetricsCollectorTest : FunSpec() { usedBytes = 40L * GIB, freeBytes = 55L * GIB, ) + + private const val MIH09_FILESYSTEM = "/dev/disk/by-id/wwn-0x0000000000000001-part2" + + /** `quota -u -g --no-wrap --raw-grace` on mih09, 2026-09-03 (the PR#16 attachment). */ + private val MIH09_QUOTA_OUTPUT = + """ + Disk quotas for user mih09-werkator (uid 120974): none + Disk quotas for group mih09 (gid 102180): + Filesystem blocks quota limit grace files quota limit grace + $MIH09_FILESYSTEM 1088116 8388608 12582912 0 14226 16777216 25165824 0 + """.trimIndent() + + /** Same group quota, but usage risen from 1088116 to 2000000 KiB — for the series-continuation test. */ + private val MIH09_QUOTA_OUTPUT_HIGHER_USAGE = MIH09_QUOTA_OUTPUT.replace("1088116", "2000000") } } diff --git a/src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt b/src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt index 6887d24..8bc6e1d 100644 --- a/src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt +++ b/src/test/kotlin/de/hoennig/werkator/server/UiViewsTest.kt @@ -1,6 +1,7 @@ package de.hoennig.werkator.server import de.hoennig.werkator.config.GiteaConfig +import de.hoennig.werkator.metrics.DiskSource import de.hoennig.werkator.metrics.MetricAggregate import de.hoennig.werkator.metrics.SystemMetrics import io.kotest.core.spec.style.FunSpec @@ -8,6 +9,27 @@ import io.kotest.matchers.shouldBe import java.time.Duration import java.time.Instant +private fun metricsWithDisk( + diskTotalGib: Double, + diskSource: DiskSource? = null, + quotasPresent: Boolean = false, +) = SystemMetrics( + timestamp = Instant.parse("2026-09-03T12:00:00Z"), + sampleCount = 1, + cpuCount = 4, + ramTotalGib = 8.0, + diskTotalGib = diskTotalGib, + diskSource = diskSource, + quotasPresent = quotasPresent, + cpuUsed = null, + cpuIdle = null, + ramUsedGib = null, + ramFreeGib = null, + diskUsedGib = null, + diskFreeGib = null, + repoSizeGib = null, +) + class UiViewsTest : FunSpec() { init { test("durations format as m:ss and h:mm:ss like the legacy duration column") { @@ -71,6 +93,36 @@ class UiViewsTest : FunSpec() { classesByKey["repoSizeGib"] shouldBe "" } + test("the disk total names the binding source: user quota, group quota, or the volume") { + UiFormats.diskTotal(metricsWithDisk(70.99, diskSource = null)) shouldBe "70.99 GiB" + + UiFormats.diskTotal( + metricsWithDisk( + 8.0, + diskSource = DiskSource(kind = "group", subject = "mih09", hardLimitGib = 12.0), + ), + ) shouldBe "8.00 GiB (group quota mih09, hard limit 12.00 GiB)" + + UiFormats.diskTotal( + metricsWithDisk( + 4.0, + diskSource = DiskSource(kind = "user", subject = "mih09-werkator", hardLimitGib = 6.0), + ), + ) shouldBe "4.00 GiB (user quota mih09-werkator, hard limit 6.00 GiB)" + + UiFormats.diskTotal( + metricsWithDisk(70.99, diskSource = DiskSource.volume(), quotasPresent = true), + ) shouldBe "70.99 GiB (volume, tighter than the quotas)" + + UiFormats.diskTotal( + metricsWithDisk(70.99, diskSource = DiskSource.volume(), quotasPresent = false), + ) shouldBe "70.99 GiB" + } + + test("the disk total is n/a when the disk metric itself is unavailable") { + UiFormats.diskTotal(metricsWithDisk(0.0, diskSource = null).copy(diskTotalGib = null)) shouldBe "n/a" + } + test("Gitea web links escape branch segments but keep slashes") { val links = GiteaWebLinks(GiteaConfig(baseUrl = "https://git.example.org/", owner = "acme", repo = "widget"))