Step 21 session C: BwrapBuildRunner delegates to the werkdock CLI
The runner no longer assembles raw bwrap argv: it checks image existence via 'werkdock images', loads the rootfs archive once per source as image werkator-buildenv-<hash> into werkdock's store (shared across every repository of the OS user — resolving step 22's buildenv-sharing question), and runs builds through 'werkdock run --rm' with the git-metadata mask expressed as ordered -v/--tmpfs flags. New pinned config key bwrap.werkdock (the executing binary, default via PATH) — a branch must not substitute the executing binary; synced in WerkatorConfig, BwrapOverrides, the pinned-key strip, the init template, docs/configuration.md, and AGENTS.md. werkdock's --clearenv means the server environment no longer leaks into builds; the TMPDIR workaround is gone with the raw invocation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
mhoennig
co-authored by
Claude Fable 5
Claude Fable 5
parent
f2d685932e
commit
823c8adc36
@@ -10,20 +10,30 @@ import java.nio.file.Path
|
||||
import java.security.MessageDigest
|
||||
|
||||
/**
|
||||
* Runs build commands inside a bubblewrap user-namespace sandbox (Step 17 / ADR 0007),
|
||||
* Runs build commands inside a bubblewrap user-namespace sandbox (Step 17 / ADR 0008),
|
||||
* for hosts without root and without a Docker daemon (e.g. Hostsharing managed
|
||||
* webspaces). Shells out to the `bwrap` CLI via the generic [GitCommandRunner] process
|
||||
* wrapper — no library, consistent with git and docker.
|
||||
* webspaces). Since step 21 session C it no longer assembles the raw `bwrap` argv:
|
||||
* it shells out to the `werkdock` CLI (`bwrap.werkdock`, default via PATH) — the same
|
||||
* pattern as git and docker, CLI, no library.
|
||||
*
|
||||
* The prepared rootfs (a Debian-base archive built elsewhere, since `debootstrap` is not
|
||||
* available on the target) is unpacked on demand into `.git/werkator/buildenv/<envKey>/rootfs`,
|
||||
* shared across all branch worktrees like the Docker gradle cache volume; `<envKey>` derives
|
||||
* from a hash of the archive source, so a changed source unpacks a fresh rootfs and stale
|
||||
* ones can be pruned. The returned [Process] is the attached `bwrap` process, so log
|
||||
* streaming and cancellation work exactly like native builds (`--die-with-parent` plus
|
||||
* `--unshare-pid` tear down the whole tree on cancel). Git works inside the sandbox with
|
||||
* the same layered mounts as the Docker runner: the primary `.git` read-only with
|
||||
* `.git/werkator/` masked, see [gitMetadataMounts].
|
||||
* The rootfs archive becomes a werkdock *image*, loaded once per source
|
||||
* (`werkator-buildenv-<hash>`, the hash over the source string, so a changed source
|
||||
* loads a fresh image) into werkdock's own store (`$WERKDOCK_HOME`, default
|
||||
* `~/.werkdock`) — shared by every repository of this OS user, unlike the old
|
||||
* per-repo unpack. Only the download cache for URL sources and the persistent
|
||||
* toolchain home (bound to `/root` for Gradle/Go caches) stay under
|
||||
* `.git/werkator/buildenv/`.
|
||||
*
|
||||
* Werkdock clears the environment inside the sandbox (docker semantics), so the
|
||||
* server's environment no longer leaks in — only the explicit `-e` variables below
|
||||
* plus werkdock's own `HOME`/`PATH` exist inside; the pam_tmpdir TMPDIR class of
|
||||
* bugs is gone by construction. Git works inside the sandbox with the same layered
|
||||
* mounts as the Docker runner, expressed as werkdock flags whose order is
|
||||
* significant and preserved: read-only `.git`, tmpfs mask over `.git/werkator`,
|
||||
* read-write worktree admin dir — see [gitMetadataMounts]. The returned [Process]
|
||||
* is the attached `werkdock run`, whose `bwrap` child dies with it
|
||||
* (`--die-with-parent`), so log streaming and cancellation work exactly like
|
||||
* native builds.
|
||||
*/
|
||||
@Component
|
||||
class BwrapBuildRunner(
|
||||
@@ -31,7 +41,7 @@ class BwrapBuildRunner(
|
||||
) : BuildRunner {
|
||||
private val log = LoggerFactory.getLogger(BwrapBuildRunner::class.java)
|
||||
|
||||
/** Replaceable process launcher so unit tests can capture the assembled `bwrap` argv. */
|
||||
/** Replaceable process launcher so unit tests can capture the assembled `werkdock` argv. */
|
||||
internal var processStarter: (List<String>, Path) -> Process = { command, dir ->
|
||||
ProcessBuilder(command).directory(dir.toFile()).start()
|
||||
}
|
||||
@@ -46,76 +56,37 @@ class BwrapBuildRunner(
|
||||
): Process {
|
||||
val bwrap = branchConfig.bwrap
|
||||
require(bwrap.rootfs.isNotBlank()) { "branches.<name>.bwrap.rootfs must be set when bwrap.enabled is true" }
|
||||
val buildEnvRoot = buildEnvRoot(repoDir)
|
||||
val envKey = envKey(bwrap.rootfs)
|
||||
val rootfsDir = buildEnvRoot.resolve(envKey).resolve(ROOTFS_DIR)
|
||||
ensureRootfs(bwrap, rootfsDir, repoDir, onAuxProcess)
|
||||
val homeDir = buildEnvRoot.resolve(HOME_DIR)
|
||||
val werkdock = bwrap.werkdock.ifBlank { "werkdock" }
|
||||
val image = imageName(bwrap.rootfs)
|
||||
ensureImage(werkdock, image, bwrap, repoDir, onAuxProcess)
|
||||
val homeDir = repoDir.resolve(BUILDENV_DIR).resolve(HOME_DIR)
|
||||
Files.createDirectories(homeDir)
|
||||
val args =
|
||||
invocation(command, workingDir, environment, repoDir, bwrap, rootfsDir, homeDir)
|
||||
ensureMountpoints(rootfsDir, args)
|
||||
val args = invocation(command, workingDir, environment, repoDir, bwrap, werkdock, image, homeDir)
|
||||
return processStarter(args, repoDir)
|
||||
}
|
||||
|
||||
/**
|
||||
* bwrap creates mountpoint directories for bind destinations inside the sandbox —
|
||||
* against the read-only rootfs bind that fails with "Can't mkdir parents ...
|
||||
* Read-only file system" for every destination that does not exist in the rootfs
|
||||
* (the workspace under the repo, for example). The rootfs directory itself is a
|
||||
* plain host directory, so we pre-create the mountpoints there; bwrap then finds
|
||||
* them and has nothing left to mkdir.
|
||||
* Loads the rootfs archive into the werkdock image store once per source.
|
||||
* `werkdock images` answers existence through the CLI, like `docker image
|
||||
* inspect` does for the Docker runner.
|
||||
*/
|
||||
private fun ensureMountpoints(
|
||||
rootfsDir: Path,
|
||||
args: List<String>,
|
||||
) {
|
||||
var i = 0
|
||||
while (i < args.size) {
|
||||
val arg = args[i]
|
||||
if (arg == "--bind" || arg == "--ro-bind") {
|
||||
val dest = args[i + 2]
|
||||
val mountpoint = rootfsDir.resolve(dest.substring(1))
|
||||
// Skip anything that already exists in the rootfs (e.g. /etc/resolv.conf
|
||||
// is a file the rootfs ships); only missing dirs are created.
|
||||
if (dest.startsWith("/") && !Files.exists(mountpoint)) {
|
||||
Files.createDirectories(mountpoint)
|
||||
}
|
||||
i += 3
|
||||
} else if (arg == "--proc" || arg == "--dev" || arg == "--tmpfs") {
|
||||
// The rootfs archive ships no /proc, /dev (excluded when packed), so
|
||||
// these mountpoints must exist too.
|
||||
val dest = args[i + 1]
|
||||
if (dest.startsWith("/") && !Files.exists(rootfsDir.resolve(dest.substring(1)))) {
|
||||
Files.createDirectories(rootfsDir.resolve(dest.substring(1)))
|
||||
}
|
||||
i += 2
|
||||
} else {
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unpacks the configured archive into [rootfsDir] once per environment version
|
||||
* (identified by [envKey]). Missing means "not yet unpacked"; the environment is a
|
||||
* cache like the Docker image and the Gradle volume, and stale ones are pruned with
|
||||
* the rest of `.git/werkator`.
|
||||
*/
|
||||
private fun ensureRootfs(
|
||||
private fun ensureImage(
|
||||
werkdock: String,
|
||||
image: String,
|
||||
bwrap: BwrapConfig,
|
||||
rootfsDir: Path,
|
||||
repoDir: Path,
|
||||
onAuxProcess: (Process) -> Unit,
|
||||
) {
|
||||
if (Files.isDirectory(rootfsDir)) {
|
||||
val loaded = commandRunner.runOrThrow(listOf(werkdock, "images"), repoDir, onProcess = onAuxProcess).lines()
|
||||
if (image in loaded) {
|
||||
return
|
||||
}
|
||||
Files.createDirectories(rootfsDir)
|
||||
val archive = localArchive(bwrap.rootfs, rootfsDir.parent, repoDir, onAuxProcess)
|
||||
log.info("unpacking build environment {} into {}", bwrap.rootfs, rootfsDir)
|
||||
val envDir = repoDir.resolve(BUILDENV_DIR).resolve(sourceKey(bwrap.rootfs))
|
||||
Files.createDirectories(envDir)
|
||||
val archive = localArchive(bwrap.rootfs, envDir, repoDir, onAuxProcess)
|
||||
log.info("loading build environment {} as werkdock image {}", bwrap.rootfs, image)
|
||||
commandRunner.runOrThrow(
|
||||
listOf("tar", "--no-same-owner", "-xf", archive, "-C", rootfsDir.toString()),
|
||||
listOf(werkdock, "load", "-i", archive, "--name", image),
|
||||
repoDir,
|
||||
onProcess = onAuxProcess,
|
||||
)
|
||||
@@ -123,9 +94,7 @@ class BwrapBuildRunner(
|
||||
|
||||
/**
|
||||
* Resolves [BwrapConfig.rootfs] to a local archive path: a bare or `file:` path is
|
||||
* used as-is; an `http(s)` URL is downloaded once into the buildenv root. GNU tar
|
||||
* auto-detects the compression from the archive magic, so a `.tar.gz` or `.tar.zst`
|
||||
* needs no extra flag.
|
||||
* used as-is; an `http(s)` URL is downloaded once into the buildenv cache.
|
||||
*/
|
||||
private fun localArchive(
|
||||
rootfs: String,
|
||||
@@ -155,77 +124,48 @@ class BwrapBuildRunner(
|
||||
environment: Map<String, String>,
|
||||
repoDir: Path,
|
||||
bwrap: BwrapConfig,
|
||||
rootfsDir: Path,
|
||||
werkdock: String,
|
||||
image: String,
|
||||
homeDir: Path,
|
||||
): List<String> {
|
||||
// bwrap creates mountpoints for bind destinations inside the sandbox; a
|
||||
// relative workspace path would resolve there into the read-only rootfs
|
||||
// ("Can't mkdir parents ...: Read-only file system"). Bind at absolute
|
||||
// host paths instead — same contract as the Docker runner. Relative
|
||||
// paths come from the CLI relative to the repo, so resolve them against
|
||||
// repoDir, not against the process working directory.
|
||||
// Mounts at absolute host paths — same contract as the Docker runner.
|
||||
// Relative paths come from the CLI relative to the repo, so resolve them
|
||||
// against repoDir, not against the process working directory.
|
||||
val repoDirAbs = repoDir.toAbsolutePath().normalize()
|
||||
val workspaceAbs =
|
||||
if (workspace.isAbsolute) workspace.normalize() else repoDirAbs.resolve(workspace).normalize()
|
||||
val homeDirAbs =
|
||||
if (homeDir.isAbsolute) homeDir.normalize() else repoDirAbs.resolve(homeDir).normalize()
|
||||
val args =
|
||||
mutableListOf(
|
||||
"bwrap",
|
||||
"--unshare-user",
|
||||
"--unshare-pid",
|
||||
"--die-with-parent",
|
||||
"--uid",
|
||||
"0",
|
||||
"--gid",
|
||||
"0",
|
||||
"--ro-bind",
|
||||
rootfsDir.toString(),
|
||||
"/",
|
||||
)
|
||||
// Bind the repo read-write FIRST so bwrap can create the mountpoints of
|
||||
// the later binds (workspace, worktree admin dir) inside it — creating
|
||||
// them against the read-only rootfs fails with "Can't mkdir parents ...
|
||||
// Read-only file system". The git metadata mounts below then layer the
|
||||
// usual isolation on top: read-only .git, tmpfs mask over .git/werkator,
|
||||
// read-write worktree admin dir.
|
||||
args += listOf("--bind", "$repoDirAbs", "$repoDirAbs")
|
||||
// Git metadata mounts BEFORE the workspace bind: the tmpfs mask over
|
||||
// .git/werkator must not shadow the workspace, which lives under
|
||||
// .git/werkator/worktrees — the later workspace bind shadows the mask
|
||||
// at exactly its own path and nothing else.
|
||||
val args = mutableListOf(werkdock, "run", "--rm")
|
||||
// The repo read-write FIRST, as the base the later mountpoints (workspace,
|
||||
// worktree admin dir) are created in; the git metadata mounts then layer
|
||||
// the isolation on top, and the workspace bind last shadows the tmpfs mask
|
||||
// at exactly its own path (it lives under .git/werkator/worktrees).
|
||||
args += listOf("-v", "$repoDirAbs:$repoDirAbs")
|
||||
args += gitMetadataMounts(workspaceAbs, repoDir)
|
||||
args += listOf("--bind", "$workspaceAbs", "$workspaceAbs")
|
||||
args += listOf("--bind", "$homeDirAbs", "/root")
|
||||
args += listOf("--ro-bind", "/etc/resolv.conf", "/etc/resolv.conf")
|
||||
args += listOf("--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp")
|
||||
args += listOf("--setenv", "HOME", "/root")
|
||||
// The sandbox /tmp is a fresh tmpfs, but bwrap inherits the server's
|
||||
// environment — on hosts with pam_tmpdir that includes
|
||||
// TMPDIR=/tmp/user/<uid>, which does not exist inside and breaks every
|
||||
// tool honoring it (go: "creating work dir: stat ...: no such file or
|
||||
// directory"; the JVM ignores TMPDIR, so Gradle never noticed). Set
|
||||
// both back to /tmp; explicit env below can still override.
|
||||
args += listOf("--setenv", "TMPDIR", "/tmp")
|
||||
args += listOf("--setenv", "TMP", "/tmp")
|
||||
args += listOf("-v", "$workspaceAbs:$workspaceAbs")
|
||||
args += listOf("-v", "$homeDirAbs:/root")
|
||||
for ((key, value) in environment) {
|
||||
args += listOf("--setenv", key, value)
|
||||
args += listOf("-e", "$key=$value")
|
||||
}
|
||||
for ((key, value) in bwrap.env) {
|
||||
args += listOf("--setenv", key, value)
|
||||
args += listOf("-e", "$key=$value")
|
||||
}
|
||||
args += listOf("--chdir", "$workspaceAbs", "/bin/sh", "-c", command)
|
||||
args += listOf("-w", "$workspaceAbs")
|
||||
args += image
|
||||
args += listOf("/bin/sh", "-c", command)
|
||||
return args
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes git work inside the sandbox without exposing Werkator's secrets — the same
|
||||
* three layered mounts as the Docker runner, expressed in `bwrap` flags (bwrap nests
|
||||
* mounts by target path like Docker): the primary `.git` read-only, an empty tmpfs
|
||||
* masking `.git/werkator/` (machine config with `git.token`, control token, build
|
||||
* state), and this worktree's admin directory read-write so index-refreshing commands
|
||||
* keep working. Object and ref writes stay blocked by the read-only `.git` mount.
|
||||
* No mounts are added when the workspace is not a worktree of [repoDir].
|
||||
* three layered mounts as the Docker runner, expressed as werkdock flags (werkdock
|
||||
* preserves the -v/--tmpfs flag order, and bwrap nests mounts by target path): the
|
||||
* primary `.git` read-only, an empty tmpfs masking `.git/werkator/` (machine config
|
||||
* with `git.token`, control token, build state), and this worktree's admin directory
|
||||
* read-write so index-refreshing commands keep working. Object and ref writes stay
|
||||
* blocked by the read-only `.git` mount. No mounts are added when the workspace is
|
||||
* not a worktree of [repoDir].
|
||||
*/
|
||||
private fun gitMetadataMounts(
|
||||
workspace: Path,
|
||||
@@ -247,28 +187,27 @@ class BwrapBuildRunner(
|
||||
if (!adminDir.startsWith(gitDir) || !Files.isDirectory(adminDir)) {
|
||||
return emptyList()
|
||||
}
|
||||
val args = mutableListOf("--ro-bind", "$gitDir", "$gitDir")
|
||||
val args = mutableListOf("-v", "$gitDir:$gitDir:ro")
|
||||
val werkatorDir = gitDir.resolve("werkator")
|
||||
if (Files.isDirectory(werkatorDir)) {
|
||||
args += listOf("--tmpfs", "$werkatorDir")
|
||||
}
|
||||
args += listOf("--bind", "$adminDir", "$adminDir")
|
||||
args += listOf("-v", "$adminDir:$adminDir")
|
||||
return args
|
||||
}
|
||||
|
||||
private fun buildEnvRoot(repoDir: Path): Path = repoDir.resolve(BUILDENV_DIR)
|
||||
|
||||
/** A short hash of the archive source, so a changed source unpacks a fresh rootfs. */
|
||||
private fun envKey(rootfs: String): String =
|
||||
/** A short hash of the archive source, so a changed source loads a fresh image. */
|
||||
private fun sourceKey(rootfs: String): String =
|
||||
MessageDigest
|
||||
.getInstance("SHA-256")
|
||||
.digest(rootfs.toByteArray())
|
||||
.joinToString("") { "%02x".format(it) }
|
||||
.take(12)
|
||||
|
||||
private fun imageName(rootfs: String): String = "werkator-buildenv-${sourceKey(rootfs)}"
|
||||
|
||||
companion object {
|
||||
const val BUILDENV_DIR = ".git/werkator/buildenv"
|
||||
const val ROOTFS_DIR = "rootfs"
|
||||
const val HOME_DIR = "home"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,7 @@ class InitCommand(
|
||||
bwrap:
|
||||
enabled: false # run clean/build in a bwrap sandbox instead of natively (pinned)
|
||||
rootfs: "" # prepared rootfs archive (path or URL); required when enabled (pinned)
|
||||
werkdock: werkdock # the werkdock CLI executing the sandbox; default resolves via PATH (pinned)
|
||||
env: {} # additional environment variables set inside the sandbox
|
||||
# Gitea check this build reports as; empty uses gitea.statusContext.
|
||||
# Two builds of one commit under the same context overwrite each other.
|
||||
|
||||
@@ -68,6 +68,7 @@ data class BuildDefinition(
|
||||
branchConfig.bwrap.copy(
|
||||
enabled = bwrap?.enabled ?: branchConfig.bwrap.enabled,
|
||||
rootfs = bwrap?.rootfs ?: branchConfig.bwrap.rootfs,
|
||||
werkdock = bwrap?.werkdock ?: branchConfig.bwrap.werkdock,
|
||||
env = bwrap?.env ?: branchConfig.bwrap.env,
|
||||
),
|
||||
)
|
||||
@@ -174,5 +175,7 @@ data class BwrapOverrides(
|
||||
val enabled: Boolean? = null,
|
||||
/** Rootfs archive source. Pinned — a branch must not substitute a foreign rootfs. */
|
||||
val rootfs: String? = null,
|
||||
/** The werkdock CLI executing the sandbox. Pinned — a branch must not substitute the executing binary. */
|
||||
val werkdock: String? = null,
|
||||
val env: Map<String, String>? = null,
|
||||
)
|
||||
|
||||
@@ -403,8 +403,8 @@ class ConfigLoader(
|
||||
/** `docker` keys a branch must never override: the sandbox policy. */
|
||||
private val PINNED_DOCKER_KEYS = setOf("enabled", "network")
|
||||
|
||||
/** `bwrap` keys a branch must never override: the sandbox policy (Step 17). */
|
||||
private val PINNED_BWRAP_KEYS = setOf("enabled", "rootfs")
|
||||
/** `bwrap` keys a branch must never override: the sandbox policy (Step 17) and its executing binary. */
|
||||
private val PINNED_BWRAP_KEYS = setOf("enabled", "rootfs", "werkdock")
|
||||
|
||||
/**
|
||||
* The one key of a build definition that says *when* and *for which branches* it
|
||||
|
||||
@@ -197,6 +197,11 @@ data class BwrapConfig(
|
||||
* Pinned — a branch must not substitute a foreign rootfs via its committed config.
|
||||
*/
|
||||
val rootfs: String = "",
|
||||
/**
|
||||
* The werkdock CLI executing the sandbox (step 21 session C); empty or the default
|
||||
* resolves via PATH. Pinned — a branch must not substitute the executing binary.
|
||||
*/
|
||||
val werkdock: String = "werkdock",
|
||||
/** Additional environment variables set inside the sandbox. */
|
||||
val env: Map<String, String> = emptyMap(),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user