Implement quota-aware disk metrics (PR#16)
The system page showed the disk of the host volume, not the budget the instance can actually fill — on a Hostsharing Managed Webspace that is a group quota, tighter than the volume by an order of magnitude, so the warn/critical highlighting could never fire before a build failed with "Disk quota exceeded". DiskQuota parses `quota -u -g --no-wrap --raw-grace` and picks the tightest of the user quota, the group quota, and the volume itself; SystemMetricsCollector reports whichever binds, resets the disk min/max/avg when the binding source changes, and the system page names the source in its info line. A host without a binding quota (Docker hosts, developer machines) renders exactly as before. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
mhoennig
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
be965617cd
commit
61c235313f
@@ -0,0 +1,133 @@
|
||||
package de.hoennig.werkator.metrics
|
||||
|
||||
/** One `blocks`/`quota`/`limit` line of `quota -u -g --no-wrap --raw-grace`, KiB throughout. */
|
||||
data class QuotaLine(
|
||||
val kind: String,
|
||||
val subject: String,
|
||||
val filesystem: String,
|
||||
val blocksKib: Long,
|
||||
val softKib: Long,
|
||||
val hardKib: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* Where the binding [SystemMetricsCollector.DiskSpace] came from: the volume itself, or a
|
||||
* user/group quota. `kind` is `"volume"`, `"user"` or `"group"`; `subject`/`filesystem`/the
|
||||
* limits are set only for a quota. Serialized as-is into `GET /api/system`.
|
||||
*/
|
||||
data class DiskSource(
|
||||
val kind: String,
|
||||
val subject: String? = null,
|
||||
val filesystem: String? = null,
|
||||
val softLimitGib: Double? = null,
|
||||
val hardLimitGib: Double? = null,
|
||||
) {
|
||||
companion object {
|
||||
fun volume() = DiskSource(kind = "volume")
|
||||
}
|
||||
}
|
||||
|
||||
/** One candidate budget for a directory: a quota line or the volume, each with its source. */
|
||||
data class DiskCandidate(
|
||||
val space: SystemMetricsCollector.DiskSpace,
|
||||
val source: DiskSource,
|
||||
)
|
||||
|
||||
/**
|
||||
* Parses `quota -u -g --no-wrap --raw-grace` and picks the tightest of user quota, group
|
||||
* quota, and the volume — pure functions over strings and numbers, see PR#16.
|
||||
*/
|
||||
object DiskQuota {
|
||||
private val subjectHeader = Regex("""^Disk quotas for (user|group) (\S+) \([ug]id \d+\):\s*(none)?\s*$""")
|
||||
|
||||
/** `null`/blank output, and a subject reported `none`, both yield no line for that subject. */
|
||||
fun parse(output: String): List<QuotaLine> {
|
||||
val lines = mutableListOf<QuotaLine>()
|
||||
var kind: String? = null
|
||||
var subject: String? = null
|
||||
for (rawLine in output.lines()) {
|
||||
val header = subjectHeader.find(rawLine.trimEnd())
|
||||
if (header != null) {
|
||||
kind = header.groupValues[1]
|
||||
subject = header.groupValues[2]
|
||||
continue
|
||||
}
|
||||
val currentKind = kind ?: continue
|
||||
val currentSubject = subject ?: continue
|
||||
val trimmed = rawLine.trim()
|
||||
if (trimmed.isEmpty() || trimmed.startsWith("Filesystem")) {
|
||||
continue
|
||||
}
|
||||
val fields = trimmed.split(Regex("\\s+"))
|
||||
if (fields.size < 4) {
|
||||
continue
|
||||
}
|
||||
val blocksKib = fields[1].trimEnd('*').toLongOrNull() ?: continue
|
||||
val softKib = fields[2].toLongOrNull() ?: continue
|
||||
val hardKib = fields[3].toLongOrNull() ?: continue
|
||||
lines +=
|
||||
QuotaLine(
|
||||
kind = currentKind,
|
||||
subject = currentSubject,
|
||||
filesystem = fields[0],
|
||||
blocksKib = blocksKib,
|
||||
softKib = softKib,
|
||||
hardKib = hardKib,
|
||||
)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
/**
|
||||
* The lines whose filesystem matches [directoryFilesystem] — exactly, or by the last path
|
||||
* segment when one side is a resolved device path (`/dev/sdb1` vs `/dev/disk/by-id/…`) — each
|
||||
* turned into a candidate. A line with no soft and no hard limit (both zero) is no candidate;
|
||||
* a line with only a hard limit uses it as the total.
|
||||
*/
|
||||
fun candidatesFor(
|
||||
lines: List<QuotaLine>,
|
||||
directoryFilesystem: String,
|
||||
): List<DiskCandidate> =
|
||||
lines
|
||||
.filter { matchesFilesystem(it.filesystem, directoryFilesystem) }
|
||||
.mapNotNull { line ->
|
||||
val softBytes = line.softKib * BYTES_PER_KIB
|
||||
val hardBytes = line.hardKib * BYTES_PER_KIB
|
||||
val totalBytes = if (softBytes > 0) softBytes else hardBytes
|
||||
if (totalBytes <= 0) {
|
||||
return@mapNotNull null
|
||||
}
|
||||
val usedBytes = line.blocksKib * BYTES_PER_KIB
|
||||
val freeBytes = maxOf(0L, totalBytes - usedBytes)
|
||||
DiskCandidate(
|
||||
space =
|
||||
SystemMetricsCollector.DiskSpace(
|
||||
totalBytes = totalBytes,
|
||||
usedBytes = usedBytes,
|
||||
freeBytes = freeBytes,
|
||||
),
|
||||
source =
|
||||
DiskSource(
|
||||
kind = line.kind,
|
||||
subject = line.subject,
|
||||
filesystem = line.filesystem,
|
||||
softLimitGib = softBytes / BYTES_PER_GIB,
|
||||
hardLimitGib = hardBytes / BYTES_PER_GIB,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
private fun matchesFilesystem(
|
||||
quotaFilesystem: String,
|
||||
directoryFilesystem: String,
|
||||
): Boolean =
|
||||
quotaFilesystem == directoryFilesystem ||
|
||||
quotaFilesystem.substringAfterLast('/') == directoryFilesystem.substringAfterLast('/')
|
||||
|
||||
/** The candidate with the smallest headroom; the tightest budget always wins. */
|
||||
fun bindingDiskSpace(candidates: List<DiskCandidate>): DiskCandidate = candidates.minBy { it.space.freeBytes }
|
||||
|
||||
private const val BYTES_PER_KIB = 1024L
|
||||
|
||||
private const val BYTES_PER_GIB = 1_073_741_824.0
|
||||
}
|
||||
@@ -22,6 +22,10 @@ data class SystemMetrics(
|
||||
val cpuCount: Int,
|
||||
val ramTotalGib: Double?,
|
||||
val diskTotalGib: Double?,
|
||||
/** What `diskTotalGib`/`diskUsedGib`/`diskFreeGib` describe: the volume, or a binding quota. */
|
||||
val diskSource: DiskSource? = null,
|
||||
/** True when a user or group quota existed but the volume was still the tighter budget. */
|
||||
val quotasPresent: Boolean = false,
|
||||
val cpuUsed: MetricAggregate?,
|
||||
val cpuIdle: MetricAggregate?,
|
||||
val ramUsedGib: MetricAggregate?,
|
||||
|
||||
@@ -19,10 +19,15 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.ScheduledExecutorService
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/** The aggregation state persisted in the artifact root (the legacy `system_state.dat`, but as JSON). */
|
||||
/**
|
||||
* The aggregation state persisted in the artifact root (the legacy `system_state.dat`, but as JSON).
|
||||
* [diskSourceKey] defaults to `"volume"` — the only source before quota-awareness existed — so an
|
||||
* older state file upgrades cleanly: it only triggers the disk-series reset when a quota now binds.
|
||||
*/
|
||||
data class PersistedMetricsState(
|
||||
val sampleCount: Long = 0,
|
||||
val series: Map<String, MetricSeries> = emptyMap(),
|
||||
val diskSourceKey: String = "volume",
|
||||
)
|
||||
|
||||
/**
|
||||
@@ -42,6 +47,8 @@ class SystemMetricsCollector(
|
||||
private val procMeminfo: Path = Paths.get("/proc/meminfo"),
|
||||
private val cpuCount: Int = Runtime.getRuntime().availableProcessors(),
|
||||
private val diskSpace: (Path) -> DiskSpace = { dir -> fileStoreDiskSpace(dir) },
|
||||
private val fileStoreName: (Path) -> String = { dir -> Files.getFileStore(dir).name() },
|
||||
private val quotaOutput: () -> String? = { readQuotaOutput() },
|
||||
private val repoSizeBytes: (Path) -> Long = { dir -> directorySizeBytes(dir) },
|
||||
) {
|
||||
/** Disk usage in bytes; used/free follow `df` semantics (free is what a user can still allocate). */
|
||||
@@ -122,6 +129,13 @@ class SystemMetricsCollector(
|
||||
val cpu = readCpuLoad()
|
||||
val ram = readRam()
|
||||
val disk = readDisk()
|
||||
val diskSourceKey = disk?.let { diskSourceKey(it.source) } ?: previousState.diskSourceKey
|
||||
if (disk != null && diskSourceKey != previousState.diskSourceKey) {
|
||||
// the previous binary/binding source measured a different budget (Scenario#16.05):
|
||||
// continuing its min/max/avg would poison the new series with a stale ceiling
|
||||
series.remove("diskUsedGib")
|
||||
series.remove("diskFreeGib")
|
||||
}
|
||||
val repoSizeGib = readRepoSizeThrottled()
|
||||
snapshot =
|
||||
SystemMetrics(
|
||||
@@ -129,19 +143,25 @@ class SystemMetricsCollector(
|
||||
sampleCount = sampleCount,
|
||||
cpuCount = cpuCount,
|
||||
ramTotalGib = ram?.totalGib,
|
||||
diskTotalGib = disk?.totalBytes?.let { it / BYTES_PER_GIB },
|
||||
diskTotalGib = disk?.space?.totalBytes?.let { it / BYTES_PER_GIB },
|
||||
diskSource = disk?.source,
|
||||
quotasPresent = disk?.quotasPresent ?: false,
|
||||
cpuUsed = record("cpuUsed", cpu?.usedCores),
|
||||
cpuIdle = record("cpuIdle", cpu?.idleCores),
|
||||
ramUsedGib = record("ramUsedGib", ram?.usedGib),
|
||||
ramFreeGib = record("ramFreeGib", ram?.freeGib),
|
||||
diskUsedGib = record("diskUsedGib", disk?.usedBytes?.let { it / BYTES_PER_GIB }),
|
||||
diskFreeGib = record("diskFreeGib", disk?.freeBytes?.let { it / BYTES_PER_GIB }),
|
||||
diskUsedGib = record("diskUsedGib", disk?.space?.usedBytes?.let { it / BYTES_PER_GIB }),
|
||||
diskFreeGib = record("diskFreeGib", disk?.space?.freeBytes?.let { it / BYTES_PER_GIB }),
|
||||
repoSizeGib = record("repoSizeGib", repoSizeGib),
|
||||
)
|
||||
state = PersistedMetricsState(sampleCount = sampleCount, series = series)
|
||||
state = PersistedMetricsState(sampleCount = sampleCount, series = series, diskSourceKey = diskSourceKey)
|
||||
saveState(state!!)
|
||||
}
|
||||
|
||||
/** `"volume"`, or `"quota:<kind>:<subject>:<filesystem>"` — the reset trigger of Scenario#16.05. */
|
||||
private fun diskSourceKey(source: DiskSource): String =
|
||||
if (source.kind == "volume") "volume" else "quota:${source.kind}:${source.subject}:${source.filesystem}"
|
||||
|
||||
private fun sampleSafely() {
|
||||
try {
|
||||
sample()
|
||||
@@ -157,6 +177,8 @@ class SystemMetricsCollector(
|
||||
cpuCount = cpuCount,
|
||||
ramTotalGib = null,
|
||||
diskTotalGib = null,
|
||||
diskSource = null,
|
||||
quotasPresent = false,
|
||||
cpuUsed = null,
|
||||
cpuIdle = null,
|
||||
ramUsedGib = null,
|
||||
@@ -240,7 +262,27 @@ class SystemMetricsCollector(
|
||||
.removeSuffix(" kB")
|
||||
.toLong()
|
||||
|
||||
private fun readDisk(): DiskSpace? = readSource("disk") { diskSpace(repoDirs().first()) }
|
||||
/** Disk reading plus whether a quota lost against the volume, for the info line (Scenario#16.06). */
|
||||
private data class DiskReading(
|
||||
val space: DiskSpace,
|
||||
val source: DiskSource,
|
||||
val quotasPresent: Boolean,
|
||||
)
|
||||
|
||||
/**
|
||||
* The tightest of the user quota, the group quota, and the volume itself (Scenario#16.02):
|
||||
* a failing or absent `quota` binary just leaves the volume as the only candidate, logged
|
||||
* once under its own "quota" source, separately from a failing file-store read.
|
||||
*/
|
||||
private fun readDisk(): DiskReading? =
|
||||
readSource("disk") {
|
||||
val dir = repoDirs().first()
|
||||
val volume = DiskCandidate(diskSpace(dir), DiskSource.volume())
|
||||
val quotaLines = readSource("quota") { quotaOutput()?.let(DiskQuota::parse) ?: emptyList() } ?: emptyList()
|
||||
val quotaCandidates = DiskQuota.candidatesFor(quotaLines, fileStoreName(dir))
|
||||
val binding = DiskQuota.bindingDiskSpace(quotaCandidates + volume)
|
||||
DiskReading(space = binding.space, source = binding.source, quotasPresent = quotaCandidates.isNotEmpty())
|
||||
}
|
||||
|
||||
/**
|
||||
* The repository size is expensive to determine (a full file walk), so unlike
|
||||
@@ -325,6 +367,26 @@ class SystemMetricsCollector(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* `quota -u -g --no-wrap --raw-grace`: `--no-wrap` keeps long device names on one line,
|
||||
* `--raw-grace` prints the grace columns as numbers, so every filesystem line has the
|
||||
* same nine fields ([DiskQuota] needs no column heuristics). The exit status is not read —
|
||||
* `quota` also uses it to say "over quota" — only the parsed output counts; an absent
|
||||
* binary throws (caught by the caller's `readSource`, logged once), a hung process is
|
||||
* killed after 5s and treated the same way.
|
||||
*/
|
||||
fun readQuotaOutput(): String {
|
||||
val process =
|
||||
ProcessBuilder("quota", "-u", "-g", "--no-wrap", "--raw-grace")
|
||||
.redirectErrorStream(true)
|
||||
.start()
|
||||
if (!process.waitFor(5, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly()
|
||||
error("quota command timed out")
|
||||
}
|
||||
return process.inputStream.bufferedReader().readText()
|
||||
}
|
||||
|
||||
/** File-walk replacement for the legacy `du -sk`; unreadable subtrees are skipped, links are not followed. */
|
||||
fun directorySizeBytes(dir: Path): Long {
|
||||
var size = 0L
|
||||
|
||||
@@ -88,6 +88,22 @@ object UiFormats {
|
||||
|
||||
private const val UTILIZATION_WARN = 0.80
|
||||
private const val UTILIZATION_CRIT = 0.90
|
||||
|
||||
/**
|
||||
* `8.00 GiB (group quota mih09, hard limit 12.00 GiB)`, `70.99 GiB (volume, tighter than the
|
||||
* quotas)`, or the plain total when no quota exists — `werkator.js` (`formatDiskTotal`) mirrors
|
||||
* this exactly, since it renders the same field from the polled `GET /api/system`.
|
||||
*/
|
||||
fun diskTotal(metrics: SystemMetrics): String {
|
||||
val total = metrics.diskTotalGib ?: return "n/a"
|
||||
val totalText = "${metric(total)} GiB"
|
||||
val source = metrics.diskSource
|
||||
if (source == null || source.kind == "volume") {
|
||||
return if (metrics.quotasPresent) "$totalText (volume, tighter than the quotas)" else totalText
|
||||
}
|
||||
val hardLimit = metric(source.hardLimitGib)
|
||||
return "$totalText (${source.kind} quota ${source.subject}, hard limit $hardLimit GiB)"
|
||||
}
|
||||
}
|
||||
|
||||
/** One row of the build tables; [latestGreenUrl] only on the build that permanent link resolves to. */
|
||||
@@ -247,7 +263,7 @@ data class SystemMetricsView(
|
||||
),
|
||||
cpuCount = "${metrics.cpuCount} cores",
|
||||
ramTotal = metrics.ramTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a",
|
||||
diskTotal = metrics.diskTotalGib?.let { "${UiFormats.metric(it)} GiB" } ?: "n/a",
|
||||
diskTotal = UiFormats.diskTotal(metrics),
|
||||
updated = metrics.timestamp?.let { UiFormats.timeOfDay(it) } ?: "n/a",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -599,6 +599,20 @@ function initCurrentBuilds() {
|
||||
/** The total each utilization metric is compared against for the critical highlighting. */
|
||||
const UTILIZATION_TOTALS = { cpuUsed: "cpuCount", ramUsedGib: "ramTotalGib", diskUsedGib: "diskTotalGib" };
|
||||
|
||||
/** Mirrors UiFormats.diskTotal exactly, since both render the same field from GET /api/system. */
|
||||
function formatDiskTotal(metrics) {
|
||||
if (metrics.diskTotalGib == null) {
|
||||
return "n/a";
|
||||
}
|
||||
const totalText = formatMetric(metrics.diskTotalGib) + " GiB";
|
||||
const source = metrics.diskSource;
|
||||
if (!source || source.kind === "volume") {
|
||||
return metrics.quotasPresent ? totalText + " (volume, tighter than the quotas)" : totalText;
|
||||
}
|
||||
const hardLimit = formatMetric(source.hardLimitGib);
|
||||
return totalText + " (" + source.kind + " quota " + source.subject + ", hard limit " + hardLimit + " GiB)";
|
||||
}
|
||||
|
||||
/** Same thresholds as UiFormats.utilizationClass: warn from 80% of the total, critical from 90%. */
|
||||
function utilizationClass(used, total) {
|
||||
if (used == null || total == null || !(total > 0)) {
|
||||
@@ -647,7 +661,7 @@ function initSystemTable() {
|
||||
});
|
||||
setText("info-cpu-count", metrics.cpuCount != null ? metrics.cpuCount + " cores" : "n/a");
|
||||
setText("info-ram-total", metrics.ramTotalGib != null ? formatMetric(metrics.ramTotalGib) + " GiB" : "n/a");
|
||||
setText("info-disk-total", metrics.diskTotalGib != null ? formatMetric(metrics.diskTotalGib) + " GiB" : "n/a");
|
||||
setText("info-disk-total", formatDiskTotal(metrics));
|
||||
setText("info-updated", formatTimeOfDay(metrics.timestamp));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user