From e869b46cbf7ddf8731070869f832883dfd582885 Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Tue, 7 Jul 2026 13:49:08 +0200 Subject: [PATCH] implemented 11-docker-build-runtime.md: added optional Docker-based build execution with configuration, per-branch runtime selection, image rebuild on input changes, Gradle cache volume, and ownership repair; updated docs and configuration --- CLAUDE.md | 2 + docs/configuration.md | 25 ++ docs/plan/11-docker-build-runtime.md | 36 +++ docs/plan/README.md | 2 +- .../hoennig/gittally/build/BuildExecutor.kt | 15 +- .../de/hoennig/gittally/build/BuildRunner.kt | 31 +- .../gittally/build/DockerBuildRunner.kt | 292 ++++++++++++++++++ .../gittally/build/DockerImageInputs.kt | 28 ++ .../gittally/build/DockerSocketLocator.kt | 46 +++ .../hoennig/gittally/commands/InitCommand.kt | 7 + .../hoennig/gittally/config/GitTallyConfig.kt | 16 + .../build/DispatchingBuildRunnerTest.kt | 42 +++ .../gittally/build/DockerBuildRunnerTest.kt | 248 +++++++++++++++ .../gittally/build/DockerImageInputsTest.kt | 51 +++ 14 files changed, 836 insertions(+), 5 deletions(-) create mode 100644 src/main/kotlin/de/hoennig/gittally/build/DockerBuildRunner.kt create mode 100644 src/main/kotlin/de/hoennig/gittally/build/DockerImageInputs.kt create mode 100644 src/main/kotlin/de/hoennig/gittally/build/DockerSocketLocator.kt create mode 100644 src/test/kotlin/de/hoennig/gittally/build/DispatchingBuildRunnerTest.kt create mode 100644 src/test/kotlin/de/hoennig/gittally/build/DockerBuildRunnerTest.kt create mode 100644 src/test/kotlin/de/hoennig/gittally/build/DockerImageInputsTest.kt diff --git a/CLAUDE.md b/CLAUDE.md index f8e7974..7249a1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,6 +73,8 @@ Three places must stay in sync when config keys change: the `GitTallyConfig` dat `BuildExecutor` runs builds asynchronously: up to `builds.maxConcurrent` branches concurrently (default 1), but never more than one build per branch at a time. Each branch builds in its own reusable git worktree at `.git/gittally/worktrees/` (`BranchWorkspaces`), checked out detached at the requested commit — the primary checkout is never used for builds. Status transitions are persisted via `BuildResultRepository` (JSON file under `.git/gittally/`), published to Gitea non-fatally, and emitted as `BuildStatusChangedEvent`s. Cancellation addresses a build by artifact key and terminates the whole process tree. Future code (watcher, server, UI) must not assume a single running build. +The runtime is selected per branch behind the `BuildRunner` interface: `DispatchingBuildRunner` (`@Primary`) routes to native `ProcessBuildRunner` (the default) or to `DockerBuildRunner` when `branches..docker.enabled`. The Docker runner shells out to the `docker` CLI (no SDK): it (re)builds the configured image when the Dockerfile inputs changed (tracked via the `org.gittally.build-inputs-sha256` image label), maintains a per-repo Gradle cache volume, mounts the worktree and the Docker socket into a labelled (`org.hoennig.gittally`) `--rm --init` container, and repairs workspace ownership in-container after each command. The returned `Process` is the attached `docker run` client, so log streaming and termination work exactly like native builds. + ### Watcher `Watcher` replaces the legacy blocking main loop with a non-blocking fixed-delay poll cycle: fetch origin, enqueue due branches (changed local, recent new origin, due auto-build slots) via `BuildExecutor`, then prune results, artifacts, and stale worktrees. Nothing is scheduled until `Watcher.start()` is called explicitly (server/watch mode) — CLI commands and tests never start the loop. "Already built" is tracked via the result repository, not by moving local branch refs. Auto-build slot state lives in `.git/gittally/auto-builds.json`; watcher health is exposed via `Watcher.state()`. diff --git a/docs/configuration.md b/docs/configuration.md index 74e7912..9823976 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -83,6 +83,21 @@ branches: autoBuild: enabled: false # whether to rebuild on schedule times: ["01:00"] # UTC times HH:MM for scheduled builds + # Optional Docker build runtime; when enabled, the clean and build commands + # run inside a container instead of natively (see notes below). + docker: + # run clean/build commands in a Docker container + enabled: false + # image for the build container; required when enabled + image: "" + # Dockerfile to (re)build the image from when it is missing or stale; empty pulls the image as-is + dockerfile: "" + # Docker build context used with dockerfile + context: "." + # Docker network mode for the build container; empty = Docker default + network: "" + # additional environment variables set inside the build container + env: {} main: autoBuild: @@ -100,6 +115,16 @@ branches: - "04:00" ``` +### Notes on `branches..docker` + +With `docker.enabled`, GitTally shells out to the `docker` CLI; the `docker` command must be on the `PATH`. +When `dockerfile` is set, the image is (re)built whenever the Dockerfile content, its path, or the context path changed. +Staleness is tracked via the image label `org.gittally.build-inputs-sha256`. +A Gradle cache volume `gittally-gradle-` is created per repository and mounted as `GRADLE_USER_HOME`. +The build worktree is bind-mounted into the container; after each command the ownership of `build/` and `.gradle/` is repaired to the host user. +The Docker socket is mounted into the container and `DOCKER_HOST`/`TESTCONTAINERS_*` variables are set, so Testcontainers-based builds work inside the container. +All GitTally containers carry `org.hoennig.gittally` labels; stale build containers of the repository are removed before the first Docker build after a restart. + ## `.git/gittally/.gittally.yml` (not committed) ```yaml diff --git a/docs/plan/11-docker-build-runtime.md b/docs/plan/11-docker-build-runtime.md index de2a25f..8f87a51 100644 --- a/docs/plan/11-docker-build-runtime.md +++ b/docs/plan/11-docker-build-runtime.md @@ -37,3 +37,39 @@ Update `GitTallyConfig`, `InitCommand` templates, and `docs/configuration.md` to - `./gradlew ktlintFormat` then `./gradlew build` is green with Docker absent. - Native execution path (step 04) is unchanged and remains the default. + +## Implementation Notes (2026-07-07) + +Implemented as designed: `DockerBuildRunner` (in `build/`) implements `BuildRunner` and shells out to the `docker` CLI via the generic `GitCommandRunner` process wrapper. +The runtime is selected per branch by `DispatchingBuildRunner` (`@Primary`), so `BuildExecutor` keeps a single `BuildRunner` dependency and the native `ProcessBuildRunner` stays the default. +No test needs Docker; the main `docker run` argv is asserted exactly through an injectable process launcher, everything else through the mocked command runner. + +Ported from legacy: + +- Image ensure (`ensure_docker_build_image`): rebuild when the image is missing or the `org.gittally.build-inputs-sha256` label no longer matches; all four `org.gittally.*` labels are set. + Without a configured `dockerfile`, the image is used as-is and pulled by `docker run` on demand. +- Gradle cache volume `gittally-gradle-`, created and chowned to the host uid/gid with the legacy container script. +- Build container: workspace bind mount, `branch` env var, configured extra env, network mode, docker socket mount with `DOCKER_HOST`/`TESTCONTAINERS_*` for Testcontainers-based builds, `--add-host host.docker.internal:host-gateway` off host network. +- Ownership repair of `build/` and `.gradle/` (`repair_docker_workspace_ownership`). +- `org.hoennig.gittally` labels (role `build`) and stale-container cleanup. + +Deviations and decisions: + +- The `BuildRunner` interface gained `repoDir` and `branchConfig` parameters (with defaults), because runner selection and Docker settings are per branch and the per-repo volume/container names need the repository path — a worktree cannot resolve the uncommitted config layer. +- The hsadmin-ng-specific legacy options were not ported, as the step suggests: no preflight command, no `JAVA_TOOL_OPTIONS` injection, no `.testcontainers.properties` generation, no `HSADMINNG_*` env passthrough — `TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE`/`TESTCONTAINERS_HOST_OVERRIDE`/`DOCKER_HOST` cover modern Testcontainers; anything else fits `docker.env`. +- Ownership repair runs inside the same build container (wrapped around the command, preserving its exit code) instead of a follow-up root container; the separate `prepare_docker_workspace_build_dir` step became unnecessary because the clean command already runs in the container. +- Container names are per branch (`gittally-build--`), not per repository, because builds of different branches may run concurrently. +- Containers run with `--init`, so termination signals from build cancellation reach the build process inside the container. +- Stale labelled containers are removed before the first Docker build of the process, not at daemon startup, so installations that never build in Docker never invoke docker. +- The Gradle volume is prepared once per process and image, not before every build. +- A missing unix socket skips the socket mount instead of failing the build (legacy errored); a tcp:// `DOCKER_HOST` also skips it. +- `docker.network` defaults to Docker's default network, not to `host` like legacy (host mode was an hsadmin-ng-ism); `network: host` switches the Testcontainers host override to `localhost` exactly like legacy. +- Image-input checksums are computed in-process (`DockerImageInputs`), not via `sha256sum`, with the same input format as legacy. + +Manual smoke test (2026-07-07, scratch repo, Rancher Desktop 27.3.1): + +- A scratch repo with `docker.enabled`, a two-line Dockerfile, and a build command writing `id -u` into `build/who.txt`: `gittally build` built the image with all four labels, created the `gittally-gradle-` volume, streamed the container output live, and exited 0 (`success after 0:14`). +- A second run reused the image (inputs label matched, no rebuild; `success after 0:04`). +- The command ran as uid 0 inside the container while `build/who.txt` ended up owned by the host user — the in-container ownership repair works. +- No labelled containers were left behind after the builds. +- Caveat found while testing (environmental, not GitTally): with a VM-based Docker (Rancher Desktop/Lima), workspace bind mounts only work for paths shared into the VM (e.g. `$HOME`); a repo under an unshared `/tmp` builds against an empty VM-side directory. diff --git a/docs/plan/README.md b/docs/plan/README.md index 18b0f44..242d363 100644 --- a/docs/plan/README.md +++ b/docs/plan/README.md @@ -61,7 +61,7 @@ Server and UI: Completion: - [x] `10-cli-commands.md` — CLI build/status commands -- [ ] `11-docker-build-runtime.md` — optional Docker build execution +- [x] `11-docker-build-runtime.md` — optional Docker build execution - [ ] `12-deployment.md` — systemd service, migration from legacy, docs Steps 01–03 are independent of each other. diff --git a/src/main/kotlin/de/hoennig/gittally/build/BuildExecutor.kt b/src/main/kotlin/de/hoennig/gittally/build/BuildExecutor.kt index 745e808..fb51402 100644 --- a/src/main/kotlin/de/hoennig/gittally/build/BuildExecutor.kt +++ b/src/main/kotlin/de/hoennig/gittally/build/BuildExecutor.kt @@ -185,7 +185,8 @@ class BuildExecutor( Files.newOutputStream(build.runningBuild.liveLogFile).use { liveLog -> writeLiveLogHeader(liveLog, build.runningBuild, branchConfig, workspace) if (branchConfig.cleanCommand.isNotBlank()) { - val cleanExitCode = runCommand(build, branchConfig.cleanCommand, workspace, stdoutLog, stderrLog, liveLog) + val cleanExitCode = + runCommand(build, branchConfig, branchConfig.cleanCommand, workspace, stdoutLog, stderrLog, liveLog) if (cleanExitCode != 0) { return cleanExitCode } @@ -193,7 +194,7 @@ class BuildExecutor( if (build.cancelled.get()) { return CANCELLED_EXIT_CODE } - return runCommand(build, branchConfig.buildCommand, workspace, stdoutLog, stderrLog, liveLog) + return runCommand(build, branchConfig, branchConfig.buildCommand, workspace, stdoutLog, stderrLog, liveLog) } } } @@ -201,13 +202,21 @@ class BuildExecutor( private fun runCommand( build: ActiveBuild, + branchConfig: BranchConfig, command: String, workspace: Path, stdoutLog: OutputStream, stderrLog: OutputStream, liveLog: OutputStream, ): Int { - val process = buildRunner.start(command, workspace, mapOf("branch" to build.runningBuild.branch)) + val process = + buildRunner.start( + command = command, + workingDir = workspace, + environment = mapOf("branch" to build.runningBuild.branch), + repoDir = build.workingDir, + branchConfig = branchConfig, + ) build.process = process if (build.cancelled.get()) { destroyProcessTree(process) diff --git a/src/main/kotlin/de/hoennig/gittally/build/BuildRunner.kt b/src/main/kotlin/de/hoennig/gittally/build/BuildRunner.kt index d4253c4..8039ef5 100644 --- a/src/main/kotlin/de/hoennig/gittally/build/BuildRunner.kt +++ b/src/main/kotlin/de/hoennig/gittally/build/BuildRunner.kt @@ -1,18 +1,23 @@ package de.hoennig.gittally.build +import de.hoennig.gittally.config.BranchConfig +import org.springframework.context.annotation.Primary import org.springframework.stereotype.Component import java.nio.file.Path /** * Starts a single build or clean command and hands the [Process] back to the caller, * which owns log streaming and process-tree termination. - * Native shell execution for now; a Docker runner can plug in later (step 11). + * [repoDir] and [branchConfig] let implementations derive per-repository resources + * and per-branch settings (step 11: Docker runtime). */ interface BuildRunner { fun start( command: String, workingDir: Path, environment: Map, + repoDir: Path = workingDir, + branchConfig: BranchConfig = BranchConfig(), ): Process } @@ -22,6 +27,8 @@ class ProcessBuildRunner : BuildRunner { command: String, workingDir: Path, environment: Map, + repoDir: Path, + branchConfig: BranchConfig, ): Process { val processBuilder = ProcessBuilder("bash", "-c", command) processBuilder.directory(workingDir.toFile()) @@ -29,3 +36,25 @@ class ProcessBuildRunner : BuildRunner { return processBuilder.start() } } + +/** + * Selects the runtime per branch: Docker when `branches..docker.enabled`, + * native shell execution otherwise (the unchanged default). + */ +@Primary +@Component +class DispatchingBuildRunner( + private val processBuildRunner: ProcessBuildRunner, + private val dockerBuildRunner: DockerBuildRunner, +) : BuildRunner { + override fun start( + command: String, + workingDir: Path, + environment: Map, + repoDir: Path, + branchConfig: BranchConfig, + ): Process { + val runner = if (branchConfig.docker.enabled) dockerBuildRunner else processBuildRunner + return runner.start(command, workingDir, environment, repoDir, branchConfig) + } +} diff --git a/src/main/kotlin/de/hoennig/gittally/build/DockerBuildRunner.kt b/src/main/kotlin/de/hoennig/gittally/build/DockerBuildRunner.kt new file mode 100644 index 0000000..0ccc3e1 --- /dev/null +++ b/src/main/kotlin/de/hoennig/gittally/build/DockerBuildRunner.kt @@ -0,0 +1,292 @@ +package de.hoennig.gittally.build + +import de.hoennig.gittally.config.BranchConfig +import de.hoennig.gittally.config.DockerConfig +import de.hoennig.gittally.git.GitCommandRunner +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Component +import java.nio.file.Path + +/** + * Runs build commands inside a Docker container (legacy `--docker` mode), shelling + * out to the `docker` CLI via the generic [GitCommandRunner] process wrapper — + * consistent with the git gateway; no Docker SDK dependency. + * + * Auxiliary steps run synchronously before each command: stale container cleanup + * (once per process), image ensure (rebuild when the Dockerfile inputs changed, + * tracked via the [DockerImageInputs.INPUTS_LABEL] image label), and the per-repo + * Gradle cache volume. The returned [Process] is the attached `docker run` client + * (`--init`, so termination signals reach the build inside the container); the + * executor streams and terminates it like a native build. Ownership of `build/` + * and `.gradle/` is repaired to the host user inside the same container run. + */ +@Component +class DockerBuildRunner( + private val commandRunner: GitCommandRunner, + private val socketLocator: DockerSocketLocator, +) : BuildRunner { + private val log = LoggerFactory.getLogger(DockerBuildRunner::class.java) + + /** Volumes already prepared by this process, keyed by volume and image. */ + private val preparedVolumes = mutableSetOf() + + private var staleContainersCleaned = false + + /** Replaceable process launcher so unit tests can capture the assembled `docker run` argv. */ + internal var processStarter: (List, Path) -> Process = { command, dir -> + ProcessBuilder(command).directory(dir.toFile()).start() + } + + override fun start( + command: String, + workingDir: Path, + environment: Map, + repoDir: Path, + branchConfig: BranchConfig, + ): Process { + val docker = branchConfig.docker + require(docker.image.isNotBlank()) { "branches..docker.image must be set when docker.enabled is true" } + val repoKey = ArtifactKeys.repoKey(repoDir) + cleanupStaleContainersOnce(repoKey, repoDir) + ensureImage(docker, repoDir) + val uid = id("-u", repoDir) + val gid = id("-g", repoDir) + val volume = gradleVolumeName(repoKey) + prepareGradleVolume(volume, docker.image, uid, gid, repoDir) + val containerName = containerName(repoKey, environment["branch"]) + removeContainer(containerName, repoDir) + val runCommand = + runArgs( + command = command, + workspace = workingDir.toAbsolutePath().normalize(), + environment = environment, + docker = docker, + repoKey = repoKey, + containerName = containerName, + volume = volume, + uid = uid, + gid = gid, + ) + return processStarter(runCommand, repoDir) + } + + /** + * Legacy `ensure_docker_build_image`: without a configured Dockerfile the image + * is used as-is (pulled by `docker run` on demand); otherwise it is (re)built + * whenever it is missing or its build-inputs label no longer matches. + */ + private fun ensureImage( + docker: DockerConfig, + repoDir: Path, + ) { + if (docker.dockerfile.isBlank()) { + return + } + val dockerfileHash = DockerImageInputs.dockerfileSha256(repoDir.resolve(docker.dockerfile)) + val inputsHash = DockerImageInputs.inputsSha256(dockerfileHash, docker.dockerfile, docker.context) + val inspect = + commandRunner.run( + listOf("docker", "image", "inspect", docker.image, "--format", INSPECT_INPUTS_LABEL_FORMAT), + repoDir, + ) + if (inspect.isSuccess && inspect.stdout.trim() == inputsHash) { + return + } + log.info("building Docker image {} from {}", docker.image, docker.dockerfile) + commandRunner.runOrThrow( + listOf( + "docker", + "build", + "--label", + "org.gittally.dockerfile=${docker.dockerfile}", + "--label", + "org.gittally.dockerfile-sha256=$dockerfileHash", + "--label", + "org.gittally.build-context=${docker.context}", + "--label", + "${DockerImageInputs.INPUTS_LABEL}=$inputsHash", + "-t", + docker.image, + "-f", + docker.dockerfile, + docker.context, + ), + repoDir, + ) + } + + /** + * Legacy `prepare_docker_gradle_volume`: create the per-repo Gradle cache volume + * and chown it to the host user — once per process and image. + */ + @Synchronized + private fun prepareGradleVolume( + volume: String, + image: String, + uid: String, + gid: String, + repoDir: Path, + ) { + val key = "$volume@$image" + if (key in preparedVolumes) { + return + } + commandRunner.runOrThrow(listOf("docker", "volume", "create", volume), repoDir) + commandRunner.runOrThrow( + listOf( + "docker", + "run", + "--rm", + "--user", + "0", + "--volume", + "$volume:/gradle-user-home", + image, + "sh", + "-c", + VOLUME_PREPARE_SCRIPT, + "sh", + uid, + gid, + ), + repoDir, + ) + preparedVolumes += key + } + + /** + * Legacy `cleanup_stale_build_runtime`, run before the first Docker build of this + * process instead of at daemon startup, so hosts that never build in Docker never + * see a docker call: force-remove all leftover build containers of this repository. + */ + @Synchronized + private fun cleanupStaleContainersOnce( + repoKey: String, + repoDir: Path, + ) { + if (staleContainersCleaned) { + return + } + staleContainersCleaned = true + val listing = + commandRunner.run( + listOf( + "docker", + "ps", + "-aq", + "--filter", + "label=$GITTALLY_LABEL=true", + "--filter", + "label=$GITTALLY_LABEL.repository=$repoKey", + "--filter", + "label=$GITTALLY_LABEL.role=build", + ), + repoDir, + ) + if (!listing.isSuccess) { + log.warn("could not list stale build containers: {}", listing.stderr.trim()) + return + } + val containers = listing.lines() + if (containers.isEmpty()) { + return + } + log.info("removing {} stale build container(s)", containers.size) + commandRunner.run(listOf("docker", "rm", "-f") + containers, repoDir) + } + + private fun removeContainer( + containerName: String, + repoDir: Path, + ) { + commandRunner.run(listOf("docker", "rm", "-f", containerName), repoDir) + } + + private fun runArgs( + command: String, + workspace: Path, + environment: Map, + docker: DockerConfig, + repoKey: String, + containerName: String, + volume: String, + uid: String, + gid: String, + ): List { + val socket = socketLocator.locate(uid) + val args = mutableListOf("docker", "run", "--rm", "--init", "--name", containerName) + args += + listOf( + "--label", + "$GITTALLY_LABEL=true", + "--label", + "$GITTALLY_LABEL.repository=$repoKey", + "--label", + "$GITTALLY_LABEL.role=build", + ) + args += listOf("--workdir", "$workspace", "--volume", "$workspace:$workspace") + args += listOf("--volume", "$volume:/gradle-user-home") + args += listOf("--env", "HOME=/tmp/docker-home", "--env", "GRADLE_USER_HOME=/gradle-user-home") + for ((key, value) in environment) { + args += listOf("--env", "$key=$value") + } + for ((key, value) in docker.env) { + args += listOf("--env", "$key=$value") + } + if (docker.network.isNotBlank()) { + args += listOf("--network", docker.network) + } + if (socket != null) { + args += listOf("--volume", "${socket.path}:/var/run/docker.sock") + args += listOf("--env", "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock") + args += listOf("--env", "DOCKER_HOST=unix:///var/run/docker.sock") + if (!socket.rootless && socket.gid != null) { + args += listOf("--group-add", socket.gid.toString()) + } + } + args += listOf("--user", if (socket?.rootless == true) uid else "0") + val hostOverride = if (docker.network == "host") "localhost" else "host.docker.internal" + args += listOf("--env", "TESTCONTAINERS_HOST_OVERRIDE=$hostOverride") + if (docker.network != "host") { + args += listOf("--add-host", "host.docker.internal:host-gateway") + } + args += docker.image + args += listOf("sh", "-c", OWNERSHIP_REPAIR_SCRIPT, "sh", uid, gid, command) + return args + } + + private fun id( + flag: String, + repoDir: Path, + ): String = commandRunner.runOrThrow(listOf("id", flag), repoDir).stdout.trim() + + companion object { + /** Container label namespace; legacy used `org.hostsharing.gittally`. */ + const val GITTALLY_LABEL = "org.hoennig.gittally" + + fun gradleVolumeName(repoKey: String): String = "gittally-gradle-$repoKey" + + fun containerName( + repoKey: String, + branch: String?, + ): String = "gittally-build-$repoKey" + (branch?.let { "-${ArtifactKeys.branchKey(it)}" } ?: "") + + private val INSPECT_INPUTS_LABEL_FORMAT = """{{ index .Config.Labels "${DockerImageInputs.INPUTS_LABEL}" }}""" + + /** Legacy `prepare_docker_gradle_volume` container script, verbatim. */ + private const val VOLUME_PREPARE_SCRIPT = + "mkdir -p /gradle-user-home/wrapper/dists && chown -R \"\$1:\$2\" /gradle-user-home && chmod -R u+rwX /gradle-user-home" + + /** + * Runs the build command and then repairs the workspace ownership (legacy + * `repair_docker_workspace_ownership`) inside the same container, preserving + * the build's exit code. + */ + private const val OWNERSHIP_REPAIR_SCRIPT = + "bash -c \"\$3\"; build_exit=\$?; " + + "for path in build .gradle; do " + + "if [ -e \"\$path\" ]; then chown -R \"\$1:\$2\" \"\$path\" && chmod -R u+rwX \"\$path\"; fi; " + + "done; " + + "exit \$build_exit" + } +} diff --git a/src/main/kotlin/de/hoennig/gittally/build/DockerImageInputs.kt b/src/main/kotlin/de/hoennig/gittally/build/DockerImageInputs.kt new file mode 100644 index 0000000..279b473 --- /dev/null +++ b/src/main/kotlin/de/hoennig/gittally/build/DockerImageInputs.kt @@ -0,0 +1,28 @@ +package de.hoennig.gittally.build + +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +/** + * Staleness checksum of the Docker image build inputs, compatible with legacy + * `docker_build_inputs_sha256`: the SHA-256 of the Dockerfile contents combined + * with the configured Dockerfile and context paths, stored as an image label. + */ +object DockerImageInputs { + const val INPUTS_LABEL = "org.gittally.build-inputs-sha256" + + fun dockerfileSha256(dockerfile: Path): String = sha256Hex(Files.readAllBytes(dockerfile)) + + fun inputsSha256( + dockerfileHash: String, + dockerfile: String, + context: String, + ): String = sha256Hex("$dockerfileHash\n$dockerfile\n$context\n".toByteArray()) + + private fun sha256Hex(bytes: ByteArray): String = + MessageDigest + .getInstance("SHA-256") + .digest(bytes) + .joinToString("") { "%02x".format(it) } +} diff --git a/src/main/kotlin/de/hoennig/gittally/build/DockerSocketLocator.kt b/src/main/kotlin/de/hoennig/gittally/build/DockerSocketLocator.kt new file mode 100644 index 0000000..788a7d7 --- /dev/null +++ b/src/main/kotlin/de/hoennig/gittally/build/DockerSocketLocator.kt @@ -0,0 +1,46 @@ +package de.hoennig.gittally.build + +import org.springframework.stereotype.Component +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +/** The host-side Docker socket a build container should talk to. */ +data class DockerSocket( + val path: Path, + /** True for a rootless daemon socket (`/run/user//docker.sock`); the container then runs as the host user. */ + val rootless: Boolean, + /** Group id owning the socket, for `--group-add` on rootful daemons; null when unavailable. */ + val gid: Long?, +) + +/** + * Port of the legacy socket detection: a unix `DOCKER_HOST` wins, then the rootless + * user socket, then the system socket. Returns null when no unix socket exists + * (e.g. a tcp:// daemon); the build container then runs without a socket mount. + */ +@Component +class DockerSocketLocator { + fun locate(uid: String): DockerSocket? { + val dockerHost = System.getenv("DOCKER_HOST").orEmpty() + val rootlessPath = Paths.get("/run/user/$uid/docker.sock") + val path = + when { + dockerHost.startsWith("unix://") -> Paths.get(dockerHost.removePrefix("unix://")) + dockerHost.isNotEmpty() -> return null + Files.exists(rootlessPath) -> rootlessPath + else -> Paths.get("/var/run/docker.sock") + } + if (!Files.exists(path)) { + return null + } + return DockerSocket(path = path, rootless = path == rootlessPath, gid = socketGid(path)) + } + + private fun socketGid(path: Path): Long? = + try { + (Files.getAttribute(path, "unix:gid") as? Number)?.toLong() + } catch (_: Exception) { + null + } +} diff --git a/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt b/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt index f0b7dcb..c11bab3 100644 --- a/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt +++ b/src/main/kotlin/de/hoennig/gittally/commands/InitCommand.kt @@ -153,6 +153,13 @@ class InitCommand( autoBuild: enabled: false # whether to rebuild on schedule times: ["01:00"] # UTC times HH:MM for scheduled builds + docker: + enabled: false # run clean/build commands in a Docker container instead of natively + image: "" # image for the build container; required when enabled + dockerfile: "" # Dockerfile to (re)build the image from when missing or stale; empty pulls the image as-is + context: "." # Docker build context used with dockerfile + network: "" # Docker network mode for the build container; empty = Docker default + env: {} # additional environment variables set inside the build container """.trimIndent() file.toFile().writeText(content + "\n") println("created ${file.toFile().relativeTo(normalizedWorkingDir.toFile())}") diff --git a/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt b/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt index 8259327..aaf0924 100644 --- a/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt +++ b/src/main/kotlin/de/hoennig/gittally/config/GitTallyConfig.kt @@ -58,6 +58,22 @@ data class BranchConfig( val stdoutLog: String = "build.stdout.log", val stderrLog: String = "build.stderr.log", val autoBuild: AutoBuildConfig = AutoBuildConfig(), + val docker: DockerConfig = DockerConfig(), +) + +data class DockerConfig( + /** Run the clean and build commands in a Docker container instead of natively. */ + val enabled: Boolean = false, + /** Image for the build container; required when [enabled]. */ + val image: String = "", + /** Dockerfile to build [image] from when it is missing or stale; empty uses [image] as-is (pulled on demand). */ + val dockerfile: String = "", + /** Docker build context used with [dockerfile]. */ + val context: String = ".", + /** Docker network mode for the build container; empty uses Docker's default network. */ + val network: String = "", + /** Additional environment variables set inside the build container. */ + val env: Map = emptyMap(), ) data class AutoBuildConfig( diff --git a/src/test/kotlin/de/hoennig/gittally/build/DispatchingBuildRunnerTest.kt b/src/test/kotlin/de/hoennig/gittally/build/DispatchingBuildRunnerTest.kt new file mode 100644 index 0000000..ac1dddb --- /dev/null +++ b/src/test/kotlin/de/hoennig/gittally/build/DispatchingBuildRunnerTest.kt @@ -0,0 +1,42 @@ +package de.hoennig.gittally.build + +import de.hoennig.gittally.config.BranchConfig +import de.hoennig.gittally.config.DockerConfig +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.mockk.Called +import io.mockk.clearMocks +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import java.nio.file.Paths + +class DispatchingBuildRunnerTest : FunSpec() { + private val processBuildRunner = mockk() + private val dockerBuildRunner = mockk() + private val dispatcher = DispatchingBuildRunner(processBuildRunner, dockerBuildRunner) + private val process = mockk() + private val dir = Paths.get(".") + + init { + beforeEach { clearMocks(processBuildRunner, dockerBuildRunner) } + + test("runs natively by default") { + val branchConfig = BranchConfig() + every { processBuildRunner.start("cmd", dir, emptyMap(), dir, branchConfig) } returns process + + dispatcher.start("cmd", dir, emptyMap(), dir, branchConfig) shouldBe process + + verify { dockerBuildRunner wasNot Called } + } + + test("runs in Docker when the branch enables it") { + val branchConfig = BranchConfig(docker = DockerConfig(enabled = true, image = "build-env:latest")) + every { dockerBuildRunner.start("cmd", dir, emptyMap(), dir, branchConfig) } returns process + + dispatcher.start("cmd", dir, emptyMap(), dir, branchConfig) shouldBe process + + verify { processBuildRunner wasNot Called } + } + } +} diff --git a/src/test/kotlin/de/hoennig/gittally/build/DockerBuildRunnerTest.kt b/src/test/kotlin/de/hoennig/gittally/build/DockerBuildRunnerTest.kt new file mode 100644 index 0000000..a38d6d9 --- /dev/null +++ b/src/test/kotlin/de/hoennig/gittally/build/DockerBuildRunnerTest.kt @@ -0,0 +1,248 @@ +package de.hoennig.gittally.build + +import de.hoennig.gittally.config.BranchConfig +import de.hoennig.gittally.config.DockerConfig +import de.hoennig.gittally.git.GitCommandResult +import de.hoennig.gittally.git.GitCommandRunner +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldContain +import io.kotest.matchers.collections.shouldNotContain +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.mockk.clearMocks +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class DockerBuildRunnerTest : FunSpec() { + private val commandRunner = mockk() + private val socketLocator = mockk() + private lateinit var runner: DockerBuildRunner + private lateinit var repoDir: Path + private lateinit var workspace: Path + private val captured = mutableListOf>() + + private fun dockerBranchConfig( + image: String = "build-env:latest", + dockerfile: String = "", + network: String = "", + env: Map = emptyMap(), + ): BranchConfig = + BranchConfig( + docker = + DockerConfig( + enabled = true, + image = image, + dockerfile = dockerfile, + network = network, + env = env, + ), + ) + + init { + beforeEach { + clearMocks(commandRunner, socketLocator) + captured.clear() + repoDir = Files.createTempDirectory("gittally-docker-runner") + workspace = repoDir.resolve("workspace") + every { commandRunner.run(any(), any(), any()) } returns GitCommandResult(0, "", "") + every { commandRunner.runOrThrow(any(), any(), any()) } returns GitCommandResult(0, "", "") + every { commandRunner.runOrThrow(listOf("id", "-u"), any(), any()) } returns GitCommandResult(0, "1000\n", "") + every { commandRunner.runOrThrow(listOf("id", "-g"), any(), any()) } returns GitCommandResult(0, "1001\n", "") + every { socketLocator.locate("1000") } returns + DockerSocket(Paths.get("/var/run/docker.sock"), rootless = false, gid = 999L) + runner = DockerBuildRunner(commandRunner, socketLocator) + runner.processStarter = { command, _ -> + captured += command + ProcessBuilder("true").start() + } + } + + test("assembles the exact docker run command (rootful socket, default network)") { + val branchConfig = dockerBranchConfig(env = mapOf("FOO" to "bar")) + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, branchConfig) + + val repoKey = ArtifactKeys.repoKey(repoDir) + val args = captured.single() + val script = args[args.size - 5] + args shouldBe + listOf( + "docker", + "run", + "--rm", + "--init", + "--name", + "gittally-build-$repoKey-${ArtifactKeys.branchKey("main")}", + "--label", + "org.hoennig.gittally=true", + "--label", + "org.hoennig.gittally.repository=$repoKey", + "--label", + "org.hoennig.gittally.role=build", + "--workdir", + "$workspace", + "--volume", + "$workspace:$workspace", + "--volume", + "gittally-gradle-$repoKey:/gradle-user-home", + "--env", + "HOME=/tmp/docker-home", + "--env", + "GRADLE_USER_HOME=/gradle-user-home", + "--env", + "branch=main", + "--env", + "FOO=bar", + "--volume", + "/var/run/docker.sock:/var/run/docker.sock", + "--env", + "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock", + "--env", + "DOCKER_HOST=unix:///var/run/docker.sock", + "--group-add", + "999", + "--user", + "0", + "--env", + "TESTCONTAINERS_HOST_OVERRIDE=host.docker.internal", + "--add-host", + "host.docker.internal:host-gateway", + "build-env:latest", + "sh", + "-c", + script, + "sh", + "1000", + "1001", + "./gradlew test", + ) + script shouldContain "bash -c \"\$3\"" + script shouldContain "chown -R \"\$1:\$2\"" + script shouldContain "exit \$build_exit" + } + + test("a rootless socket runs the container as the host user without group-add") { + every { socketLocator.locate("1000") } returns + DockerSocket(Paths.get("/run/user/1000/docker.sock"), rootless = true, gid = 998L) + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig()) + + val args = captured.single() + args shouldContain "/run/user/1000/docker.sock:/var/run/docker.sock" + args[args.indexOf("--user") + 1] shouldBe "1000" + args shouldNotContain "--group-add" + } + + test("host network keeps Testcontainers on localhost without add-host") { + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig(network = "host")) + + val args = captured.single() + args[args.indexOf("--network") + 1] shouldBe "host" + args shouldContain "TESTCONTAINERS_HOST_OVERRIDE=localhost" + args shouldNotContain "--add-host" + } + + test("without a docker socket the container runs without socket mount as root") { + every { socketLocator.locate("1000") } returns null + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig()) + + val args = captured.single() + args shouldNotContain "/var/run/docker.sock:/var/run/docker.sock" + args shouldNotContain "DOCKER_HOST=unix:///var/run/docker.sock" + args[args.indexOf("--user") + 1] shouldBe "0" + } + + test("builds a missing image from the Dockerfile with the input labels") { + Files.writeString(repoDir.resolve("Dockerfile"), "FROM eclipse-temurin:21\n") + every { commandRunner.run(match { it.take(3) == listOf("docker", "image", "inspect") }, any(), any()) } returns + GitCommandResult(1, "", "no such image") + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig(dockerfile = "Dockerfile")) + + val dockerfileHash = DockerImageInputs.dockerfileSha256(repoDir.resolve("Dockerfile")) + val inputsHash = DockerImageInputs.inputsSha256(dockerfileHash, "Dockerfile", ".") + verify { + commandRunner.runOrThrow( + listOf( + "docker", + "build", + "--label", + "org.gittally.dockerfile=Dockerfile", + "--label", + "org.gittally.dockerfile-sha256=$dockerfileHash", + "--label", + "org.gittally.build-context=.", + "--label", + "org.gittally.build-inputs-sha256=$inputsHash", + "-t", + "build-env:latest", + "-f", + "Dockerfile", + ".", + ), + repoDir, + ) + } + } + + test("skips the image build when the build-inputs label still matches") { + Files.writeString(repoDir.resolve("Dockerfile"), "FROM eclipse-temurin:21\n") + val dockerfileHash = DockerImageInputs.dockerfileSha256(repoDir.resolve("Dockerfile")) + val inputsHash = DockerImageInputs.inputsSha256(dockerfileHash, "Dockerfile", ".") + every { commandRunner.run(match { it.take(3) == listOf("docker", "image", "inspect") }, any(), any()) } returns + GitCommandResult(0, "$inputsHash\n", "") + + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig(dockerfile = "Dockerfile")) + + verify(exactly = 0) { + commandRunner.runOrThrow(match { it.take(2) == listOf("docker", "build") }, any(), any()) + } + } + + test("prepares the gradle cache volume once but removes the container before every command") { + val branchConfig = dockerBranchConfig() + + runner.start("./gradlew clean", workspace, mapOf("branch" to "main"), repoDir, branchConfig) + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, branchConfig) + + val repoKey = ArtifactKeys.repoKey(repoDir) + verify(exactly = 1) { + commandRunner.runOrThrow(listOf("docker", "volume", "create", "gittally-gradle-$repoKey"), repoDir) + } + verify(exactly = 2) { + commandRunner.run( + listOf("docker", "rm", "-f", "gittally-build-$repoKey-${ArtifactKeys.branchKey("main")}"), + repoDir, + ) + } + } + + test("removes stale labelled build containers once, before the first docker build") { + every { commandRunner.run(match { it.take(3) == listOf("docker", "ps", "-aq") }, any(), any()) } returns + GitCommandResult(0, "abc\ndef\n", "") + + runner.start("./gradlew clean", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig()) + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, dockerBranchConfig()) + + verify(exactly = 1) { commandRunner.run(match { it.take(3) == listOf("docker", "ps", "-aq") }, any(), any()) } + verify { commandRunner.run(listOf("docker", "rm", "-f", "abc", "def"), repoDir) } + } + + test("fails without a configured image") { + val branchConfig = BranchConfig(docker = DockerConfig(enabled = true)) + + val exception = + shouldThrow { + runner.start("./gradlew test", workspace, mapOf("branch" to "main"), repoDir, branchConfig) + } + + exception.message shouldContain "docker.image" + } + } +} diff --git a/src/test/kotlin/de/hoennig/gittally/build/DockerImageInputsTest.kt b/src/test/kotlin/de/hoennig/gittally/build/DockerImageInputsTest.kt new file mode 100644 index 0000000..cc72aa3 --- /dev/null +++ b/src/test/kotlin/de/hoennig/gittally/build/DockerImageInputsTest.kt @@ -0,0 +1,51 @@ +package de.hoennig.gittally.build + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.string.shouldMatch +import java.nio.file.Files + +class DockerImageInputsTest : FunSpec() { + init { + test("dockerfileSha256 is a stable hex checksum of the file contents") { + val dir = Files.createTempDirectory("gittally-docker-inputs") + val dockerfile = dir.resolve("Dockerfile") + Files.writeString(dockerfile, "FROM eclipse-temurin:21\n") + + val hash = DockerImageInputs.dockerfileSha256(dockerfile) + + hash shouldMatch Regex("[0-9a-f]{64}") + DockerImageInputs.dockerfileSha256(dockerfile) shouldBe hash + } + + test("inputs checksum changes when the Dockerfile contents change") { + val dir = Files.createTempDirectory("gittally-docker-inputs") + val dockerfile = dir.resolve("Dockerfile") + Files.writeString(dockerfile, "FROM eclipse-temurin:21\n") + val before = + DockerImageInputs.inputsSha256( + DockerImageInputs.dockerfileSha256(dockerfile), + "Dockerfile", + ".", + ) + + Files.writeString(dockerfile, "FROM eclipse-temurin:22\n") + + DockerImageInputs.inputsSha256( + DockerImageInputs.dockerfileSha256(dockerfile), + "Dockerfile", + ".", + ) shouldNotBe before + } + + test("inputs checksum changes when the Dockerfile path or the context change") { + val dockerfileHash = "0".repeat(64) + val base = DockerImageInputs.inputsSha256(dockerfileHash, "ci/Dockerfile", "ci") + + DockerImageInputs.inputsSha256(dockerfileHash, "other/Dockerfile", "ci") shouldNotBe base + DockerImageInputs.inputsSha256(dockerfileHash, "ci/Dockerfile", "other") shouldNotBe base + DockerImageInputs.inputsSha256(dockerfileHash, "ci/Dockerfile", "ci") shouldBe base + } + } +}