From 823c8adc36255d4aa5ec5ceeaf90ad606cf34bdc Mon Sep 17 00:00:00 2001 From: mhoennig Date: Tue, 1 Sep 2026 15:46:19 +0200 Subject: [PATCH] Step 21 session C: BwrapBuildRunner delegates to the werkdock CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner no longer assembles raw bwrap argv: it checks image existence via 'werkdock images', loads the rootfs archive once per source as image werkator-buildenv- into werkdock's store (shared across every repository of the OS user — resolving step 22's buildenv-sharing question), and runs builds through 'werkdock run --rm' with the git-metadata mask expressed as ordered -v/--tmpfs flags. New pinned config key bwrap.werkdock (the executing binary, default via PATH) — a branch must not substitute the executing binary; synced in WerkatorConfig, BwrapOverrides, the pinned-key strip, the init template, docs/configuration.md, and AGENTS.md. werkdock's --clearenv means the server environment no longer leaks into builds; the TMPDIR workaround is gone with the raw invocation. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 +- docs/configuration.md | 21 +- ...erkdock-extraction-and-webspace-install.md | 7 +- .../werkator/build/BwrapBuildRunner.kt | 207 ++++++------------ .../hoennig/werkator/commands/InitCommand.kt | 1 + .../werkator/config/BuildDefinition.kt | 3 + .../hoennig/werkator/config/ConfigLoader.kt | 4 +- .../hoennig/werkator/config/WerkatorConfig.kt | 5 + .../werkator/build/BwrapBuildRunnerTest.kt | 197 +++++++++-------- 9 files changed, 202 insertions(+), 247 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ffbbbe1..b2edb5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,9 +39,9 @@ All production code lives under `de.hoennig.werkator`, with sub-packages `comman - Builds run detached in worktrees under `.git/werkator/worktrees/`; the primary checkout is never used for builds; never assume a single running build. - When config keys change, three places must stay in sync: the `WerkatorConfig` data classes, the `InitCommand` templates, and `docs/configuration.md`. - Every config file may declare `werkator.version.since`/`below` (the Werkator it is written for, never a format version — no API is involved). `since` is enforced in both directions, using `ConfigVersions.FORMAT_BROKE_IN` for "file predates a breaking change"; `below` only warns. A violation aborts the start for the machine and project config, but fails only that branch's builds for a branch config. -- A branch describes its own CI: its committed `.werkator.yml` is the branch layer (`ConfigLoader.loadWithBranchLayer`, used by the watcher per origin branch and by `loadForWorktree` at build time) and takes precedence over `.git`/project — including the whole `builds` section, so a new configuration can be tried out on a branch without affecting other branches. Only the pinned set is stripped from that layer: secrets (`git`), host/repository sections (`server`, `gitea`, `executor`, `watcher`), the docker (`docker.enabled`, `docker.network`) and bubblewrap (`bwrap.enabled`, `bwrap.rootfs`) sandbox policies, and the trust gate (`requirePullRequest`). A branch must never reach credentials, disable its container or sandbox, change its network, substitute a foreign rootfs, raise global concurrency, or bypass its own pull-request gate; a branch's definitions apply to that branch alone. +- A branch describes its own CI: its committed `.werkator.yml` is the branch layer (`ConfigLoader.loadWithBranchLayer`, used by the watcher per origin branch and by `loadForWorktree` at build time) and takes precedence over `.git`/project — including the whole `builds` section, so a new configuration can be tried out on a branch without affecting other branches. Only the pinned set is stripped from that layer: secrets (`git`), host/repository sections (`server`, `gitea`, `executor`, `watcher`), the docker (`docker.enabled`, `docker.network`) and bubblewrap (`bwrap.enabled`, `bwrap.rootfs`, `bwrap.werkdock`) sandbox policies, and the trust gate (`requirePullRequest`). A branch must never reach credentials, disable its container or sandbox, change its network, substitute a foreign rootfs, raise global concurrency, or bypass its own pull-request gate; a branch's definitions apply to that branch alone. - A build definition carries the complete description of its build, split in two: the `trigger` block (`onPush`, `atTimes`, `branches`, `activeWithin`) says when and for which branches it runs, everything else what it does. `builds.default` is the base every other definition inherits its settings — never its `trigger` — from. The split is structural so that a selector added to `TriggerConfig` later is non-inheritable by construction; writing a trigger key flat is refused, never ignored, because ignoring it leaves a build that silently stops running. A `!` prefix in `trigger.branches` excludes and always wins. -- The inheritance is applied after all layers are merged: that order is what makes a build a branch invents inherit the host's sandbox policy instead of the data-class default, so the pinning also holds for a build the host has never heard of. Pinned are `requirePullRequest`, `statusContext`, `docker.enabled`, `docker.network`, `bwrap.enabled`, and `bwrap.rootfs`. Docker and bwrap are mutually exclusive per branch — enabling both is rejected at start. +- The inheritance is applied after all layers are merged: that order is what makes a build a branch invents inherit the host's sandbox policy instead of the data-class default, so the pinning also holds for a build the host has never heard of. Pinned are `requirePullRequest`, `statusContext`, `docker.enabled`, `docker.network`, `bwrap.enabled`, `bwrap.rootfs`, and `bwrap.werkdock`. Docker and bwrap are mutually exclusive per branch — enabling both is rejected at start. - `builds` or the legacy `branches`, never both: `branches` is read only while the merged config defines no build at all (`builds.maxConcurrent` is not one), and ignored with a warning as soon as one exists. The section is deprecated and goes away once the repositories have migrated; then `ConfigVersions.FORMAT_BROKE_IN` gets set and a leftover `branches:` key must be rejected by name — the version check alone cannot catch a file that declares no version. - Web UI: server-rendered Thymeleaf plus one hand-written `static/werkator.js` — no SPA framework, no frontend build pipeline; every fetch has a timeout and an explicit error badge; `UiFormats` and `werkator.js` must produce identical display formats. - Git and Docker access shells out to the CLIs (`GitCommandRunner`, `docker`) — no JGit, no Docker SDK. diff --git a/docs/configuration.md b/docs/configuration.md index ee34b71..97a0bfc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,7 +91,7 @@ single branch may decide it: - the repository-side settings: the whole `gitea`, `executor`, and `watcher` sections; - the trust gate: `requirePullRequest`, and the Gitea status context: `statusContext`; - the container sandbox policy: `docker.enabled`/`docker.network` and - `bwrap.enabled`/`bwrap.rootfs` — host-pinned as + `bwrap.enabled`/`bwrap.rootfs`/`bwrap.werkdock` — host-pinned as long as only the host's configuration sets them, master-pinned once the committed configuration does. @@ -353,7 +353,7 @@ Both parts combine as an intersection. Settings: `buildCommand`, `cleanCommand`, `artifactDirs`, `stdoutLog`/`stderrLog`, `requirePullRequest`, `statusContext`, and `docker` and `bwrap` with all their keys. A definition carries the complete description of its build; unset keys fall back to `builds.default` and then to Werkator's own defaults. -`requirePullRequest`, `statusContext`, `docker.enabled`, `docker.network`, `bwrap.enabled`, and `bwrap.rootfs` are pinned (master-pinned, see [the branch layer](#the-branch-layer-a-branch-describes-its-own-ci)): they are read from the repo install/project config even when a branch sets them in its own committed config. +`requirePullRequest`, `statusContext`, `docker.enabled`, `docker.network`, `bwrap.enabled`, `bwrap.rootfs`, and `bwrap.werkdock` are pinned (master-pinned, see [the branch layer](#the-branch-layer-a-branch-describes-its-own-ci)): they are read from the repo install/project config even when a branch sets them in its own committed config. Inheritance from `builds.default` covers the settings only — the `trigger` block says when and where *this* build runs and is never inherited. Definitions are part of the branch layer: a branch may add its own and override those from the project config, for its own builds only. Because the inheritance is applied after all layers are merged, a build a branch invents still inherits the host's `builds.default` — its sandbox policy included, which is what keeps the pinning effective for a build the host has never heard of. @@ -411,16 +411,17 @@ All Werkator containers carry `org.hoennig.werkator` labels; stale build contain ### Notes on `builds..bwrap` -With `bwrap.enabled`, Werkator shells out to the `bwrap` CLI (bubblewrap) instead of native execution. -This is the third runtime, for hosts without root and without a Docker daemon (e.g. Hostsharing managed webspaces); see `docs/plan/17-bwrap-build-runtime.md` and ADR 0007. -`bwrap` must be on the `PATH`. +With `bwrap.enabled`, Werkator runs the build in a bubblewrap sandbox instead of native execution. +This is the third runtime, for hosts without root and without a Docker daemon (e.g. Hostsharing managed webspaces); see `docs/plan/17-bwrap-build-runtime.md` and ADR 0008. +Since step 21 session C the sandbox is executed by the `werkdock` CLI (`bwrap.werkdock`, default: resolved via `PATH`) — Werkator no longer invokes `bwrap` itself; `bwrap` must be installed for werkdock. +`werkdock doctor` checks the host's capability, superseding `tools/werkator-build-prerequisites.sh`. `bwrap.rootfs` names the prepared root filesystem archive — a Debian-base rootfs with the build tools (JDK, git, locales, project-specific tooling) built elsewhere, since `debootstrap` is unavailable on the target. -It is a local path or an `http(s)` URL; a URL is downloaded once. -Build the archive with `tools/build-bwrap-rootfs.sh` on any machine with Docker; verify the host's user-namespace capability first with `tools/werkator-build-prerequisites.sh`. -The archive is unpacked on demand (`tar --no-same-owner`) into `.git/werkator/buildenv//rootfs`, shared across all branch worktrees like the Docker Gradle cache volume; `` derives from a hash of the source, so a changed `rootfs` unpacks a fresh environment and stale ones can be pruned. -Per-branch Gradle caches persist in `.git/werkator/buildenv/home`, bound as `/root`. -`bwrap.env` adds environment variables inside the sandbox. +It is a local path or an `http(s)` URL; a URL is downloaded once into `.git/werkator/buildenv/`. +Build the archive with `tools/build-bwrap-rootfs.sh` on any machine with Docker. +The archive is loaded once per source as the werkdock image `werkator-buildenv-` into werkdock's store (`$WERKDOCK_HOME`, default `~/.werkdock`) — shared by every repository of this OS user; the hash derives from the source string, so a changed `rootfs` loads a fresh image and stale ones can be removed from the store. +Per-repo Gradle caches persist in `.git/werkator/buildenv/home`, bound as `/root`. +`bwrap.env` adds environment variables inside the sandbox; the environment is otherwise cleared (docker semantics) — the server's environment does not leak in. Files created inside the sandbox are owned by the host user, because uid 0 maps back to the unprivileged webspace user. `docker` and `bwrap` are mutually exclusive per branch: enabling both is rejected at start, not silently picked. diff --git a/docs/plan/21-werkdock-extraction-and-webspace-install.md b/docs/plan/21-werkdock-extraction-and-webspace-install.md index f1a2ab1..591d709 100644 --- a/docs/plan/21-werkdock-extraction-and-webspace-install.md +++ b/docs/plan/21-werkdock-extraction-and-webspace-install.md @@ -64,11 +64,14 @@ A docker-like CLI over `bwrap`, filesystem isolation only. - Own docs, plan, and ADRs under `werkdock/` from the start, so the later repository split is a directory move; the Werkator side only keeps what is Werkator-specific (the git-metadata mounts of step 16 and the config pinning). - Keep `werkdock/` self-contained: no imports from Werkator code, no Gradle coupling to the Werkator build — it must build and test on its own. -### C — Werkator consumes Werkdock (this repo, after B) +### C — Werkator consumes Werkdock (this repo, after B; implemented 2026-09-01 on branch `werkator-consumes-werkdock`) - `BwrapBuildRunner` shells out to `werkdock run` instead of assembling the raw `bwrap` argv — same pattern as git and docker: CLI, no library. - Config keys (`bwrap.enabled`, `bwrap.rootfs`) and their pinning stay as they are; only the executor behind them changes. -- Decide in the step: whether the git-metadata mounts stay Werkator-side (passed as extra `--bind`/`--tmpfs` options to `werkdock run`) or become a Werkdock feature; the secrets-masking of `.git/werkator/` must hold either way. + One key was added: `bwrap.werkdock` (the executing binary, default via PATH) — pinned like the rest of the sandbox policy, since a branch must not substitute the executing binary. +- Decided: the git-metadata mounts stay Werkator-side, passed as `-v …:ro` / `--tmpfs` / `-v` options whose flag order werkdock preserves (it grew `--tmpfs` and an ordered mount list for exactly this); the secrets-masking of `.git/werkator/` holds unchanged. +- Decided: the rootfs archive becomes a werkdock *image* (`werkator-buildenv-`, checked via `werkdock images`, loaded via `werkdock load`) in werkdock's own store — shared across every repository of the OS user, which resolves step 22's buildenv-sharing question; only the URL download cache and the persistent `/root` toolchain home stay under `.git/werkator/buildenv/`. +- Consequence of werkdock's `--clearenv`: the server environment no longer leaks into builds, and the runner's TMPDIR workaround is gone. ### D — The Managed-Webspace install path (this repo, independent of B/C) diff --git a/src/main/kotlin/de/hoennig/werkator/build/BwrapBuildRunner.kt b/src/main/kotlin/de/hoennig/werkator/build/BwrapBuildRunner.kt index 63301f8..3735bc5 100644 --- a/src/main/kotlin/de/hoennig/werkator/build/BwrapBuildRunner.kt +++ b/src/main/kotlin/de/hoennig/werkator/build/BwrapBuildRunner.kt @@ -10,20 +10,30 @@ import java.nio.file.Path import java.security.MessageDigest /** - * Runs build commands inside a bubblewrap user-namespace sandbox (Step 17 / ADR 0007), + * Runs build commands inside a bubblewrap user-namespace sandbox (Step 17 / ADR 0008), * for hosts without root and without a Docker daemon (e.g. Hostsharing managed - * webspaces). Shells out to the `bwrap` CLI via the generic [GitCommandRunner] process - * wrapper — no library, consistent with git and docker. + * webspaces). Since step 21 session C it no longer assembles the raw `bwrap` argv: + * it shells out to the `werkdock` CLI (`bwrap.werkdock`, default via PATH) — the same + * pattern as git and docker, CLI, no library. * - * The prepared rootfs (a Debian-base archive built elsewhere, since `debootstrap` is not - * available on the target) is unpacked on demand into `.git/werkator/buildenv//rootfs`, - * shared across all branch worktrees like the Docker gradle cache volume; `` derives - * from a hash of the archive source, so a changed source unpacks a fresh rootfs and stale - * ones can be pruned. The returned [Process] is the attached `bwrap` process, so log - * streaming and cancellation work exactly like native builds (`--die-with-parent` plus - * `--unshare-pid` tear down the whole tree on cancel). Git works inside the sandbox with - * the same layered mounts as the Docker runner: the primary `.git` read-only with - * `.git/werkator/` masked, see [gitMetadataMounts]. + * The rootfs archive becomes a werkdock *image*, loaded once per source + * (`werkator-buildenv-`, the hash over the source string, so a changed source + * loads a fresh image) into werkdock's own store (`$WERKDOCK_HOME`, default + * `~/.werkdock`) — shared by every repository of this OS user, unlike the old + * per-repo unpack. Only the download cache for URL sources and the persistent + * toolchain home (bound to `/root` for Gradle/Go caches) stay under + * `.git/werkator/buildenv/`. + * + * Werkdock clears the environment inside the sandbox (docker semantics), so the + * server's environment no longer leaks in — only the explicit `-e` variables below + * plus werkdock's own `HOME`/`PATH` exist inside; the pam_tmpdir TMPDIR class of + * bugs is gone by construction. Git works inside the sandbox with the same layered + * mounts as the Docker runner, expressed as werkdock flags whose order is + * significant and preserved: read-only `.git`, tmpfs mask over `.git/werkator`, + * read-write worktree admin dir — see [gitMetadataMounts]. The returned [Process] + * is the attached `werkdock run`, whose `bwrap` child dies with it + * (`--die-with-parent`), so log streaming and cancellation work exactly like + * native builds. */ @Component class BwrapBuildRunner( @@ -31,7 +41,7 @@ class BwrapBuildRunner( ) : BuildRunner { private val log = LoggerFactory.getLogger(BwrapBuildRunner::class.java) - /** Replaceable process launcher so unit tests can capture the assembled `bwrap` argv. */ + /** Replaceable process launcher so unit tests can capture the assembled `werkdock` argv. */ internal var processStarter: (List, Path) -> Process = { command, dir -> ProcessBuilder(command).directory(dir.toFile()).start() } @@ -46,76 +56,37 @@ class BwrapBuildRunner( ): Process { val bwrap = branchConfig.bwrap require(bwrap.rootfs.isNotBlank()) { "branches..bwrap.rootfs must be set when bwrap.enabled is true" } - val buildEnvRoot = buildEnvRoot(repoDir) - val envKey = envKey(bwrap.rootfs) - val rootfsDir = buildEnvRoot.resolve(envKey).resolve(ROOTFS_DIR) - ensureRootfs(bwrap, rootfsDir, repoDir, onAuxProcess) - val homeDir = buildEnvRoot.resolve(HOME_DIR) + val werkdock = bwrap.werkdock.ifBlank { "werkdock" } + val image = imageName(bwrap.rootfs) + ensureImage(werkdock, image, bwrap, repoDir, onAuxProcess) + val homeDir = repoDir.resolve(BUILDENV_DIR).resolve(HOME_DIR) Files.createDirectories(homeDir) - val args = - invocation(command, workingDir, environment, repoDir, bwrap, rootfsDir, homeDir) - ensureMountpoints(rootfsDir, args) + val args = invocation(command, workingDir, environment, repoDir, bwrap, werkdock, image, homeDir) return processStarter(args, repoDir) } /** - * bwrap creates mountpoint directories for bind destinations inside the sandbox — - * against the read-only rootfs bind that fails with "Can't mkdir parents ... - * Read-only file system" for every destination that does not exist in the rootfs - * (the workspace under the repo, for example). The rootfs directory itself is a - * plain host directory, so we pre-create the mountpoints there; bwrap then finds - * them and has nothing left to mkdir. + * Loads the rootfs archive into the werkdock image store once per source. + * `werkdock images` answers existence through the CLI, like `docker image + * inspect` does for the Docker runner. */ - private fun ensureMountpoints( - rootfsDir: Path, - args: List, - ) { - var i = 0 - while (i < args.size) { - val arg = args[i] - if (arg == "--bind" || arg == "--ro-bind") { - val dest = args[i + 2] - val mountpoint = rootfsDir.resolve(dest.substring(1)) - // Skip anything that already exists in the rootfs (e.g. /etc/resolv.conf - // is a file the rootfs ships); only missing dirs are created. - if (dest.startsWith("/") && !Files.exists(mountpoint)) { - Files.createDirectories(mountpoint) - } - i += 3 - } else if (arg == "--proc" || arg == "--dev" || arg == "--tmpfs") { - // The rootfs archive ships no /proc, /dev (excluded when packed), so - // these mountpoints must exist too. - val dest = args[i + 1] - if (dest.startsWith("/") && !Files.exists(rootfsDir.resolve(dest.substring(1)))) { - Files.createDirectories(rootfsDir.resolve(dest.substring(1))) - } - i += 2 - } else { - i += 1 - } - } - } - - /** - * Unpacks the configured archive into [rootfsDir] once per environment version - * (identified by [envKey]). Missing means "not yet unpacked"; the environment is a - * cache like the Docker image and the Gradle volume, and stale ones are pruned with - * the rest of `.git/werkator`. - */ - private fun ensureRootfs( + private fun ensureImage( + werkdock: String, + image: String, bwrap: BwrapConfig, - rootfsDir: Path, repoDir: Path, onAuxProcess: (Process) -> Unit, ) { - if (Files.isDirectory(rootfsDir)) { + val loaded = commandRunner.runOrThrow(listOf(werkdock, "images"), repoDir, onProcess = onAuxProcess).lines() + if (image in loaded) { return } - Files.createDirectories(rootfsDir) - val archive = localArchive(bwrap.rootfs, rootfsDir.parent, repoDir, onAuxProcess) - log.info("unpacking build environment {} into {}", bwrap.rootfs, rootfsDir) + val envDir = repoDir.resolve(BUILDENV_DIR).resolve(sourceKey(bwrap.rootfs)) + Files.createDirectories(envDir) + val archive = localArchive(bwrap.rootfs, envDir, repoDir, onAuxProcess) + log.info("loading build environment {} as werkdock image {}", bwrap.rootfs, image) commandRunner.runOrThrow( - listOf("tar", "--no-same-owner", "-xf", archive, "-C", rootfsDir.toString()), + listOf(werkdock, "load", "-i", archive, "--name", image), repoDir, onProcess = onAuxProcess, ) @@ -123,9 +94,7 @@ class BwrapBuildRunner( /** * Resolves [BwrapConfig.rootfs] to a local archive path: a bare or `file:` path is - * used as-is; an `http(s)` URL is downloaded once into the buildenv root. GNU tar - * auto-detects the compression from the archive magic, so a `.tar.gz` or `.tar.zst` - * needs no extra flag. + * used as-is; an `http(s)` URL is downloaded once into the buildenv cache. */ private fun localArchive( rootfs: String, @@ -155,77 +124,48 @@ class BwrapBuildRunner( environment: Map, repoDir: Path, bwrap: BwrapConfig, - rootfsDir: Path, + werkdock: String, + image: String, homeDir: Path, ): List { - // bwrap creates mountpoints for bind destinations inside the sandbox; a - // relative workspace path would resolve there into the read-only rootfs - // ("Can't mkdir parents ...: Read-only file system"). Bind at absolute - // host paths instead — same contract as the Docker runner. Relative - // paths come from the CLI relative to the repo, so resolve them against - // repoDir, not against the process working directory. + // Mounts at absolute host paths — same contract as the Docker runner. + // Relative paths come from the CLI relative to the repo, so resolve them + // against repoDir, not against the process working directory. val repoDirAbs = repoDir.toAbsolutePath().normalize() val workspaceAbs = if (workspace.isAbsolute) workspace.normalize() else repoDirAbs.resolve(workspace).normalize() val homeDirAbs = if (homeDir.isAbsolute) homeDir.normalize() else repoDirAbs.resolve(homeDir).normalize() - val args = - mutableListOf( - "bwrap", - "--unshare-user", - "--unshare-pid", - "--die-with-parent", - "--uid", - "0", - "--gid", - "0", - "--ro-bind", - rootfsDir.toString(), - "/", - ) - // Bind the repo read-write FIRST so bwrap can create the mountpoints of - // the later binds (workspace, worktree admin dir) inside it — creating - // them against the read-only rootfs fails with "Can't mkdir parents ... - // Read-only file system". The git metadata mounts below then layer the - // usual isolation on top: read-only .git, tmpfs mask over .git/werkator, - // read-write worktree admin dir. - args += listOf("--bind", "$repoDirAbs", "$repoDirAbs") - // Git metadata mounts BEFORE the workspace bind: the tmpfs mask over - // .git/werkator must not shadow the workspace, which lives under - // .git/werkator/worktrees — the later workspace bind shadows the mask - // at exactly its own path and nothing else. + val args = mutableListOf(werkdock, "run", "--rm") + // The repo read-write FIRST, as the base the later mountpoints (workspace, + // worktree admin dir) are created in; the git metadata mounts then layer + // the isolation on top, and the workspace bind last shadows the tmpfs mask + // at exactly its own path (it lives under .git/werkator/worktrees). + args += listOf("-v", "$repoDirAbs:$repoDirAbs") args += gitMetadataMounts(workspaceAbs, repoDir) - args += listOf("--bind", "$workspaceAbs", "$workspaceAbs") - args += listOf("--bind", "$homeDirAbs", "/root") - args += listOf("--ro-bind", "/etc/resolv.conf", "/etc/resolv.conf") - args += listOf("--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp") - args += listOf("--setenv", "HOME", "/root") - // The sandbox /tmp is a fresh tmpfs, but bwrap inherits the server's - // environment — on hosts with pam_tmpdir that includes - // TMPDIR=/tmp/user/, which does not exist inside and breaks every - // tool honoring it (go: "creating work dir: stat ...: no such file or - // directory"; the JVM ignores TMPDIR, so Gradle never noticed). Set - // both back to /tmp; explicit env below can still override. - args += listOf("--setenv", "TMPDIR", "/tmp") - args += listOf("--setenv", "TMP", "/tmp") + args += listOf("-v", "$workspaceAbs:$workspaceAbs") + args += listOf("-v", "$homeDirAbs:/root") for ((key, value) in environment) { - args += listOf("--setenv", key, value) + args += listOf("-e", "$key=$value") } for ((key, value) in bwrap.env) { - args += listOf("--setenv", key, value) + args += listOf("-e", "$key=$value") } - args += listOf("--chdir", "$workspaceAbs", "/bin/sh", "-c", command) + args += listOf("-w", "$workspaceAbs") + args += image + args += listOf("/bin/sh", "-c", command) return args } /** * Makes git work inside the sandbox without exposing Werkator's secrets — the same - * three layered mounts as the Docker runner, expressed in `bwrap` flags (bwrap nests - * mounts by target path like Docker): the primary `.git` read-only, an empty tmpfs - * masking `.git/werkator/` (machine config with `git.token`, control token, build - * state), and this worktree's admin directory read-write so index-refreshing commands - * keep working. Object and ref writes stay blocked by the read-only `.git` mount. - * No mounts are added when the workspace is not a worktree of [repoDir]. + * three layered mounts as the Docker runner, expressed as werkdock flags (werkdock + * preserves the -v/--tmpfs flag order, and bwrap nests mounts by target path): the + * primary `.git` read-only, an empty tmpfs masking `.git/werkator/` (machine config + * with `git.token`, control token, build state), and this worktree's admin directory + * read-write so index-refreshing commands keep working. Object and ref writes stay + * blocked by the read-only `.git` mount. No mounts are added when the workspace is + * not a worktree of [repoDir]. */ private fun gitMetadataMounts( workspace: Path, @@ -247,28 +187,27 @@ class BwrapBuildRunner( if (!adminDir.startsWith(gitDir) || !Files.isDirectory(adminDir)) { return emptyList() } - val args = mutableListOf("--ro-bind", "$gitDir", "$gitDir") + val args = mutableListOf("-v", "$gitDir:$gitDir:ro") val werkatorDir = gitDir.resolve("werkator") if (Files.isDirectory(werkatorDir)) { args += listOf("--tmpfs", "$werkatorDir") } - args += listOf("--bind", "$adminDir", "$adminDir") + args += listOf("-v", "$adminDir:$adminDir") return args } - private fun buildEnvRoot(repoDir: Path): Path = repoDir.resolve(BUILDENV_DIR) - - /** A short hash of the archive source, so a changed source unpacks a fresh rootfs. */ - private fun envKey(rootfs: String): String = + /** A short hash of the archive source, so a changed source loads a fresh image. */ + private fun sourceKey(rootfs: String): String = MessageDigest .getInstance("SHA-256") .digest(rootfs.toByteArray()) .joinToString("") { "%02x".format(it) } .take(12) + private fun imageName(rootfs: String): String = "werkator-buildenv-${sourceKey(rootfs)}" + companion object { const val BUILDENV_DIR = ".git/werkator/buildenv" - const val ROOTFS_DIR = "rootfs" const val HOME_DIR = "home" } } diff --git a/src/main/kotlin/de/hoennig/werkator/commands/InitCommand.kt b/src/main/kotlin/de/hoennig/werkator/commands/InitCommand.kt index ac78d34..139490b 100644 --- a/src/main/kotlin/de/hoennig/werkator/commands/InitCommand.kt +++ b/src/main/kotlin/de/hoennig/werkator/commands/InitCommand.kt @@ -226,6 +226,7 @@ class InitCommand( bwrap: enabled: false # run clean/build in a bwrap sandbox instead of natively (pinned) rootfs: "" # prepared rootfs archive (path or URL); required when enabled (pinned) + werkdock: werkdock # the werkdock CLI executing the sandbox; default resolves via PATH (pinned) env: {} # additional environment variables set inside the sandbox # Gitea check this build reports as; empty uses gitea.statusContext. # Two builds of one commit under the same context overwrite each other. diff --git a/src/main/kotlin/de/hoennig/werkator/config/BuildDefinition.kt b/src/main/kotlin/de/hoennig/werkator/config/BuildDefinition.kt index 0c64dad..b02da40 100644 --- a/src/main/kotlin/de/hoennig/werkator/config/BuildDefinition.kt +++ b/src/main/kotlin/de/hoennig/werkator/config/BuildDefinition.kt @@ -68,6 +68,7 @@ data class BuildDefinition( branchConfig.bwrap.copy( enabled = bwrap?.enabled ?: branchConfig.bwrap.enabled, rootfs = bwrap?.rootfs ?: branchConfig.bwrap.rootfs, + werkdock = bwrap?.werkdock ?: branchConfig.bwrap.werkdock, env = bwrap?.env ?: branchConfig.bwrap.env, ), ) @@ -174,5 +175,7 @@ data class BwrapOverrides( val enabled: Boolean? = null, /** Rootfs archive source. Pinned — a branch must not substitute a foreign rootfs. */ val rootfs: String? = null, + /** The werkdock CLI executing the sandbox. Pinned — a branch must not substitute the executing binary. */ + val werkdock: String? = null, val env: Map? = null, ) diff --git a/src/main/kotlin/de/hoennig/werkator/config/ConfigLoader.kt b/src/main/kotlin/de/hoennig/werkator/config/ConfigLoader.kt index ca59590..88c909f 100644 --- a/src/main/kotlin/de/hoennig/werkator/config/ConfigLoader.kt +++ b/src/main/kotlin/de/hoennig/werkator/config/ConfigLoader.kt @@ -403,8 +403,8 @@ class ConfigLoader( /** `docker` keys a branch must never override: the sandbox policy. */ private val PINNED_DOCKER_KEYS = setOf("enabled", "network") - /** `bwrap` keys a branch must never override: the sandbox policy (Step 17). */ - private val PINNED_BWRAP_KEYS = setOf("enabled", "rootfs") + /** `bwrap` keys a branch must never override: the sandbox policy (Step 17) and its executing binary. */ + private val PINNED_BWRAP_KEYS = setOf("enabled", "rootfs", "werkdock") /** * The one key of a build definition that says *when* and *for which branches* it diff --git a/src/main/kotlin/de/hoennig/werkator/config/WerkatorConfig.kt b/src/main/kotlin/de/hoennig/werkator/config/WerkatorConfig.kt index 38eb09c..9045828 100644 --- a/src/main/kotlin/de/hoennig/werkator/config/WerkatorConfig.kt +++ b/src/main/kotlin/de/hoennig/werkator/config/WerkatorConfig.kt @@ -197,6 +197,11 @@ data class BwrapConfig( * Pinned — a branch must not substitute a foreign rootfs via its committed config. */ val rootfs: String = "", + /** + * The werkdock CLI executing the sandbox (step 21 session C); empty or the default + * resolves via PATH. Pinned — a branch must not substitute the executing binary. + */ + val werkdock: String = "werkdock", /** Additional environment variables set inside the sandbox. */ val env: Map = emptyMap(), ) diff --git a/src/test/kotlin/de/hoennig/werkator/build/BwrapBuildRunnerTest.kt b/src/test/kotlin/de/hoennig/werkator/build/BwrapBuildRunnerTest.kt index 3e8b988..7a77516 100644 --- a/src/test/kotlin/de/hoennig/werkator/build/BwrapBuildRunnerTest.kt +++ b/src/test/kotlin/de/hoennig/werkator/build/BwrapBuildRunnerTest.kt @@ -35,11 +35,18 @@ class BwrapBuildRunnerTest : FunSpec() { ), ) - private fun rootfsUnpacked(rootfs: String = "/srv/buildenv.tar.zst"): Path = - repoDir - .resolve(BwrapBuildRunner.BUILDENV_DIR) - .resolve(rootfs.sha12()) - .resolve(BwrapBuildRunner.ROOTFS_DIR) + private fun imageName(rootfs: String = "/srv/buildenv.tar.zst"): String = "werkator-buildenv-${rootfs.sha12()}" + + /** The image is already loaded: `werkdock images` lists it, so no load runs. */ + private fun givenImageLoaded(rootfs: String = "/srv/buildenv.tar.zst") { + every { commandRunner.runOrThrow(listOf("werkdock", "images"), repoDir, any(), any()) } returns + GitCommandResult(0, imageName(rootfs) + "\n", "") + } + + private fun givenImageMissing() { + every { commandRunner.runOrThrow(listOf("werkdock", "images"), repoDir, any(), any()) } returns + GitCommandResult(0, "some-other-image\n", "") + } init { beforeEach { @@ -54,110 +61,92 @@ class BwrapBuildRunnerTest : FunSpec() { } } - test("unpacks the rootfs on demand and assembles the exact bwrap command") { - every { - commandRunner.runOrThrow( - listOf("tar", "--no-same-owner", "-xf", "/srv/buildenv.tar.zst", "-C", rootfsUnpacked().toString()), - repoDir, - any(), - any(), - ) - } returns - GitCommandResult(0, "", "") + test("assembles the exact werkdock run command for a loaded image") { + givenImageLoaded() runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) - val args = captured.single() - val rootfsDir = args[args.indexOf("--ro-bind") + 1] - args shouldBe + captured.single() shouldBe listOf( - "bwrap", - "--unshare-user", - "--unshare-pid", - "--die-with-parent", - "--uid", - "0", - "--gid", - "0", - "--ro-bind", - rootfsUnpacked().toString(), - "/", - "--bind", - repoDir.toString(), - repoDir.toString(), - "--bind", - workspace.toString(), - workspace.toString(), - "--bind", - repoDir.resolve(".git/werkator/buildenv/home").toString(), - "/root", - "--ro-bind", - "/etc/resolv.conf", - "/etc/resolv.conf", - "--proc", - "/proc", - "--dev", - "/dev", - "--tmpfs", - "/tmp", - "--setenv", - "HOME", - "/root", - "--setenv", - "TMPDIR", - "/tmp", - "--setenv", - "TMP", - "/tmp", - "--setenv", - "branch", - "main", - "--chdir", + "werkdock", + "run", + "--rm", + "-v", + "$repoDir:$repoDir", + "-v", + "$workspace:$workspace", + "-v", + "${repoDir.resolve(".git/werkator/buildenv/home")}:/root", + "-e", + "branch=main", + "-w", workspace.toString(), + imageName(), "/bin/sh", "-c", "./gradlew test", ) - Files.isDirectory(rootfsUnpacked()) shouldBe true } - test("binds a relative workspace path at its absolute location") { - // bwrap creates mountpoints for bind destinations inside the sandbox; - // a relative path would land in the read-only rootfs and fail with - // "Can't mkdir parents ...: Read-only file system" (seen on the webspace). + test("loads the image once when werkdock does not know it yet") { + givenImageMissing() every { commandRunner.runOrThrow( - listOf("tar", "--no-same-owner", "-xf", "/srv/buildenv.tar.zst", "-C", rootfsUnpacked().toString()), + listOf("werkdock", "load", "-i", "/srv/buildenv.tar.zst", "--name", imageName()), repoDir, any(), any(), ) - } returns - GitCommandResult(0, "", "") + } returns GitCommandResult(0, "", "") + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) + + verify { + commandRunner.runOrThrow( + listOf("werkdock", "load", "-i", "/srv/buildenv.tar.zst", "--name", imageName()), + repoDir, + any(), + any(), + ) + } + } + + test("does not load an image werkdock already has") { + givenImageLoaded() + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) + + verify(exactly = 0) { commandRunner.runOrThrow(match { "load" in it }, any(), any(), any()) } + } + + test("uses the configured werkdock binary path") { + every { commandRunner.runOrThrow(listOf("/opt/bin/werkdock", "images"), repoDir, any(), any()) } returns + GitCommandResult(0, imageName() + "\n", "") + val branchConfig = + BranchConfig( + bwrap = BwrapConfig(enabled = true, rootfs = "/srv/buildenv.tar.zst", werkdock = "/opt/bin/werkdock"), + ) + + runner.start("./gradlew test", workspace, emptyMap(), repoDir, branchConfig) + + captured.single().first() shouldBe "/opt/bin/werkdock" + } + + test("mounts a relative workspace path at its absolute location") { + givenImageLoaded() val relativeWorkspace = repoDir.relativize(workspace) runner.start("./gradlew test", relativeWorkspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) val args = captured.single() val absolute = workspace.toAbsolutePath().normalize().toString() - val bindIdx = args.withIndex().filter { it.value == "--bind" }.map { it.index } - // first bind is the repo dir (mountpoint base), second is the workspace - args[bindIdx[1] + 1] shouldBe absolute - args[bindIdx[1] + 2] shouldBe absolute - args[args.indexOf("--chdir") + 1] shouldBe absolute + args shouldContainElement "-v" + args[args.indexOf("-w") + 1] shouldBe absolute + args.count { it == "$absolute:$absolute" } shouldBe 1 } - test("does not re-unpack an already prepared rootfs") { - Files.createDirectories(rootfsUnpacked()) - - runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) - - verify(exactly = 0) { commandRunner.runOrThrow(match { it.first() == "tar" }, any(), any(), any()) } - } - - test("adds bwrap env and passes the branch environment through") { - Files.createDirectories(rootfsUnpacked()) + test("adds bwrap env after the branch environment") { + givenImageLoaded() runner.start( "./gradlew test", @@ -168,39 +157,43 @@ class BwrapBuildRunnerTest : FunSpec() { ) val args = captured.single() - args[args.indexOf("branch") - 1] shouldBe "--setenv" - args[args.indexOf("branch") + 1] shouldBe "main" - args[args.indexOf("FOO") - 1] shouldBe "--setenv" - args[args.indexOf("FOO") + 1] shouldBe "bar" + args[args.indexOf("branch=main") - 1] shouldBe "-e" + args[args.indexOf("FOO=bar") - 1] shouldBe "-e" + args.indexOf("branch=main") shouldBe args.indexOf("FOO=bar") - 2 } - test("exposes git metadata read-only with the werkator dir masked for a worktree workspace") { + test("exposes git metadata read-only with the werkator dir masked, in mount order") { val gitDir = repoDir.resolve(".git") val adminDir = gitDir.resolve("worktrees/workspace") Files.createDirectories(adminDir) Files.createDirectories(gitDir.resolve("werkator")) Files.createDirectories(workspace) Files.writeString(workspace.resolve(".git"), "gitdir: $adminDir\n") - Files.createDirectories(rootfsUnpacked()) + givenImageLoaded() runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) val args = captured.single() - args[args.indexOf(gitDir.toString()) - 1] shouldBe "--ro-bind" + args[args.indexOf("$gitDir:$gitDir:ro") - 1] shouldBe "-v" args[args.indexOf("$gitDir/werkator") - 1] shouldBe "--tmpfs" - args[args.indexOf(adminDir.toString()) - 1] shouldBe "--bind" + args[args.indexOf("$adminDir:$adminDir") - 1] shouldBe "-v" + // order: ro .git, tmpfs mask, admin dir, then the workspace bind that + // shadows the mask at its own path + val roGit = args.indexOf("$gitDir:$gitDir:ro") + val mask = args.indexOf("$gitDir/werkator") + val admin = args.indexOf("$adminDir:$adminDir") + val workspaceBind = args.indexOf("$workspace:$workspace") + (roGit < mask && mask < admin && admin < workspaceBind) shouldBe true } test("mounts no git metadata when the workspace is not a worktree") { - Files.createDirectories(rootfsUnpacked()) + givenImageLoaded() Files.createDirectories(workspace) runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig()) val args = captured.single() val gitDir = repoDir.resolve(".git") - // the sandbox's own /tmp tmpfs is always present; the point is that no tmpfs - // masks .git/werkator and no worktree admin dir is bound args.none { it == "$gitDir/werkator" } shouldBe true args.none { it.contains("worktrees/") } shouldBe true } @@ -216,16 +209,17 @@ class BwrapBuildRunnerTest : FunSpec() { exception.message shouldContain "bwrap.rootfs" } - test("downloads a URL rootfs once before unpacking") { + test("downloads a URL rootfs once before loading it") { val url = "https://example.test/buildenv.tar.zst" val downloadTarget = repoDir .resolve(BwrapBuildRunner.BUILDENV_DIR) .resolve(url.sha12()) .resolve("buildenv.tar.zst") + givenImageMissing() every { commandRunner.runOrThrow(listOf("curl", "-fsSL", "-o", downloadTarget.toString(), url), repoDir, any(), any()) } returns GitCommandResult(0, "", "") - every { commandRunner.runOrThrow(match { it.first() == "tar" }, any(), any(), any()) } returns + every { commandRunner.runOrThrow(match { "load" in it }, any(), any(), any()) } returns GitCommandResult(0, "", "") runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, bwrapBranchConfig(rootfs = url)) @@ -233,10 +227,19 @@ class BwrapBuildRunnerTest : FunSpec() { verify { commandRunner.runOrThrow(listOf("curl", "-fsSL", "-o", downloadTarget.toString(), url), repoDir, any(), any()) } - verify { commandRunner.runOrThrow(match { it.first() == "tar" }, repoDir, any(), any()) } + verify { + commandRunner.runOrThrow( + listOf("werkdock", "load", "-i", downloadTarget.toString(), "--name", "werkator-buildenv-${url.sha12()}"), + repoDir, + any(), + any(), + ) + } } } + private infix fun List.shouldContainElement(element: String) = (element in this) shouldBe true + private fun String.sha12(): String = java.security.MessageDigest .getInstance("SHA-256")