implemented 05-artifact-store.md: filesystem artifact store with legacy-compatible naming, atomic persist from concurrent builds, retention pruning, and artifacts.rootDir config
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Hoennig
co-authored by
Claude Fable 5
Claude Fable 5
parent
b379bc0a6b
commit
3c9eeda5da
@@ -0,0 +1,17 @@
|
||||
package de.hoennig.gittally.artifacts
|
||||
|
||||
import de.hoennig.gittally.build.ArtifactStore
|
||||
import de.hoennig.gittally.config.ConfigLoader
|
||||
import org.springframework.context.annotation.Bean
|
||||
import org.springframework.context.annotation.Configuration
|
||||
|
||||
@Configuration
|
||||
class ArtifactsConfiguration {
|
||||
/**
|
||||
* Store relative to the working directory, matching how `ConfigLoader` and the
|
||||
* `BuildResultRepository` bean resolve their files. Nothing is touched until the
|
||||
* first build persists, so the bean is safe outside a git repository.
|
||||
*/
|
||||
@Bean
|
||||
fun artifactStore(configLoader: ConfigLoader): ArtifactStore = FileArtifactStore(configLoader)
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package de.hoennig.gittally.artifacts
|
||||
|
||||
import de.hoennig.gittally.build.ArtifactKeys
|
||||
import de.hoennig.gittally.build.ArtifactStore
|
||||
import de.hoennig.gittally.build.BuildResult
|
||||
import de.hoennig.gittally.config.BranchConfig
|
||||
import de.hoennig.gittally.config.ConfigLoader
|
||||
import org.slf4j.LoggerFactory
|
||||
import java.io.IOException
|
||||
import java.nio.file.FileVisitResult
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.LinkOption
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.SimpleFileVisitor
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||
import kotlin.concurrent.read
|
||||
import kotlin.concurrent.write
|
||||
|
||||
/**
|
||||
* Stores build artifacts on the filesystem under `<root>/branches/<artifactKey>/`.
|
||||
* The root is `artifacts.rootDir` from the config, or the platform default
|
||||
* `XDG_STATE_HOME` (falling back to `~/.local/state`) plus `/gittally/artifacts/<repo-key>` when unset —
|
||||
* deliberately not `/tmp` like legacy, where artifacts vanished on reboot.
|
||||
*
|
||||
* Each build is assembled in a temporary directory next to its target and moved
|
||||
* into place atomically, so a crashed persist never leaves a partial artifact dir.
|
||||
* Builds persist concurrently (their keys are unique) under a shared read lock;
|
||||
* [prune] takes the write lock so it never deletes a directory mid-persist.
|
||||
*/
|
||||
class FileArtifactStore(
|
||||
private val configLoader: ConfigLoader,
|
||||
private val workingDir: Path = Paths.get("."),
|
||||
private val env: (String) -> String? = System::getenv,
|
||||
) : ArtifactStore {
|
||||
private val log = LoggerFactory.getLogger(FileArtifactStore::class.java)
|
||||
|
||||
private val lock = ReentrantReadWriteLock()
|
||||
|
||||
override fun persist(
|
||||
build: BuildResult,
|
||||
stagingDir: Path,
|
||||
workspace: Path?,
|
||||
) {
|
||||
lock.read {
|
||||
val branchesDir = Files.createDirectories(branchesDir())
|
||||
val target = branchesDir.resolve(build.artifactKey)
|
||||
val incoming = branchesDir.resolve(INCOMING_PREFIX + build.artifactKey)
|
||||
try {
|
||||
deleteRecursively(incoming)
|
||||
Files.createDirectories(incoming)
|
||||
copyChildren(stagingDir, incoming)
|
||||
copyArtifactDirs(build, workspace, incoming)
|
||||
deleteRecursively(target)
|
||||
Files.move(incoming, target, StandardCopyOption.ATOMIC_MOVE)
|
||||
} catch (e: Exception) {
|
||||
deleteRecursively(incoming)
|
||||
throw e
|
||||
}
|
||||
deleteRecursively(stagingDir)
|
||||
log.info("persisted build artifacts of {} to {}", build.artifactKey, target)
|
||||
}
|
||||
}
|
||||
|
||||
override fun prune(keptResults: Collection<BuildResult>): List<String> {
|
||||
lock.write {
|
||||
val branchesDir = branchesDir()
|
||||
if (!Files.isDirectory(branchesDir, LinkOption.NOFOLLOW_LINKS)) {
|
||||
return emptyList()
|
||||
}
|
||||
val keptKeys = keptResults.map { it.artifactKey }.toSet()
|
||||
val removed = mutableListOf<String>()
|
||||
Files.list(branchesDir).use { children ->
|
||||
children.forEach { child ->
|
||||
val key = child.fileName.toString()
|
||||
if (key !in keptKeys) {
|
||||
deleteRecursively(child)
|
||||
removed += key
|
||||
}
|
||||
}
|
||||
}
|
||||
if (removed.isNotEmpty()) {
|
||||
log.info("pruned {} artifact dir(s): {}", removed.size, removed)
|
||||
}
|
||||
return removed
|
||||
}
|
||||
}
|
||||
|
||||
override fun artifactDir(artifactKey: String): Path? {
|
||||
if (!ARTIFACT_KEY_PATTERN.matches(artifactKey)) {
|
||||
return null
|
||||
}
|
||||
val branchesDir = branchesDir()
|
||||
val dir = branchesDir.resolve(artifactKey).normalize()
|
||||
if (dir.parent != branchesDir) {
|
||||
return null
|
||||
}
|
||||
return dir.takeIf { Files.isDirectory(it, LinkOption.NOFOLLOW_LINKS) }
|
||||
}
|
||||
|
||||
private fun branchesDir(): Path = rootDir().resolve("branches")
|
||||
|
||||
private fun rootDir(): Path {
|
||||
val configured =
|
||||
configLoader
|
||||
.load(workingDir)
|
||||
.artifacts.rootDir
|
||||
.trim()
|
||||
if (configured.isNotEmpty()) {
|
||||
return workingDir.resolve(expandHome(configured)).toAbsolutePath().normalize()
|
||||
}
|
||||
val stateHome =
|
||||
env("XDG_STATE_HOME")?.takeIf { it.isNotBlank() }?.let { Paths.get(it) }
|
||||
?: Paths.get(System.getProperty("user.home"), ".local", "state")
|
||||
return stateHome
|
||||
.resolve("gittally")
|
||||
.resolve("artifacts")
|
||||
.resolve(ArtifactKeys.repoKey(workingDir))
|
||||
.toAbsolutePath()
|
||||
.normalize()
|
||||
}
|
||||
|
||||
private fun expandHome(path: String): Path =
|
||||
if (path == "~" || path.startsWith("~/")) {
|
||||
Paths.get(System.getProperty("user.home"), path.removePrefix("~"))
|
||||
} else {
|
||||
Paths.get(path)
|
||||
}
|
||||
|
||||
private fun copyArtifactDirs(
|
||||
build: BuildResult,
|
||||
workspace: Path?,
|
||||
targetDir: Path,
|
||||
) {
|
||||
if (workspace == null) {
|
||||
log.warn("build {} has no workspace; storing only its logs", build.artifactKey)
|
||||
return
|
||||
}
|
||||
for (artifactDir in branchConfig(build.branch).artifactDirs) {
|
||||
if (artifactDir.isBlank()) {
|
||||
continue
|
||||
}
|
||||
val source = workspace.resolve(artifactDir)
|
||||
if (!Files.isDirectory(source)) {
|
||||
log.info("build {} did not produce artifact directory {}; skipped", build.artifactKey, artifactDir)
|
||||
continue
|
||||
}
|
||||
copyRecursively(source, targetDir.resolve(archivedPath(artifactDir)))
|
||||
}
|
||||
}
|
||||
|
||||
/** Legacy `archived_artefact_dir_path`: `build/reports` archives as `reports/`, everything else below `reports/<dir>`. */
|
||||
private fun archivedPath(artifactDir: String): String =
|
||||
if (artifactDir == "build/reports") {
|
||||
"reports"
|
||||
} else {
|
||||
"reports/$artifactDir"
|
||||
}
|
||||
|
||||
private fun branchConfig(branch: String): BranchConfig {
|
||||
val branches = configLoader.load(workingDir).branches
|
||||
return branches[branch] ?: branches["default"] ?: BranchConfig()
|
||||
}
|
||||
|
||||
private fun copyChildren(
|
||||
sourceDir: Path,
|
||||
targetDir: Path,
|
||||
) {
|
||||
if (Files.isDirectory(sourceDir)) {
|
||||
copyRecursively(sourceDir, targetDir)
|
||||
}
|
||||
}
|
||||
|
||||
private fun copyRecursively(
|
||||
source: Path,
|
||||
target: Path,
|
||||
) {
|
||||
Files.walkFileTree(
|
||||
source,
|
||||
object : SimpleFileVisitor<Path>() {
|
||||
override fun preVisitDirectory(
|
||||
dir: Path,
|
||||
attrs: BasicFileAttributes,
|
||||
): FileVisitResult {
|
||||
Files.createDirectories(target.resolve(source.relativize(dir)))
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun visitFile(
|
||||
file: Path,
|
||||
attrs: BasicFileAttributes,
|
||||
): FileVisitResult {
|
||||
Files.copy(file, target.resolve(source.relativize(file)), StandardCopyOption.REPLACE_EXISTING)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/** Does not follow symlinks — a link is deleted, its target is never touched. */
|
||||
private fun deleteRecursively(path: Path) {
|
||||
if (!Files.exists(path, LinkOption.NOFOLLOW_LINKS)) {
|
||||
return
|
||||
}
|
||||
Files.walkFileTree(
|
||||
path,
|
||||
object : SimpleFileVisitor<Path>() {
|
||||
override fun visitFile(
|
||||
file: Path,
|
||||
attrs: BasicFileAttributes,
|
||||
): FileVisitResult {
|
||||
Files.delete(file)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun postVisitDirectory(
|
||||
dir: Path,
|
||||
exc: IOException?,
|
||||
): FileVisitResult {
|
||||
if (exc != null) {
|
||||
throw exc
|
||||
}
|
||||
Files.delete(dir)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** Work-in-progress dir next to the target; unreferenced leftovers are cleaned up by [prune]. */
|
||||
private const val INCOMING_PREFIX = ".incoming-"
|
||||
|
||||
/** Same character set the key sanitization produces; everything else is rejected on lookup. */
|
||||
private val ARTIFACT_KEY_PATTERN = Regex("[A-Za-z0-9._-]+")
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package de.hoennig.gittally.build
|
||||
|
||||
import java.nio.file.Path
|
||||
import java.security.MessageDigest
|
||||
import java.time.Instant
|
||||
|
||||
@@ -16,6 +17,9 @@ object ArtifactKeys {
|
||||
startedAt: Instant,
|
||||
): String = "${branchKey(branch)}-${sanitize(startedAt.toString())}-${sha256Prefix("$branch\t$startedAt")}"
|
||||
|
||||
/** Legacy `repository_key`: the sanitized absolute repository path. */
|
||||
fun repoKey(repoDir: Path): String = sanitize(repoDir.toAbsolutePath().normalize().toString())
|
||||
|
||||
private fun sanitize(value: String): String = value.replace(Regex("[^A-Za-z0-9._-]"), "_")
|
||||
|
||||
private fun sha256Prefix(value: String): String =
|
||||
|
||||
@@ -1,29 +1,31 @@
|
||||
package de.hoennig.gittally.build
|
||||
|
||||
import org.slf4j.LoggerFactory
|
||||
import org.springframework.stereotype.Component
|
||||
import java.nio.file.Path
|
||||
|
||||
/**
|
||||
* Takes over the staging directory of a finished build.
|
||||
* The real store (naming, retention, serving) arrives in step 05.
|
||||
* Persists the artifacts of finished builds (logs plus configured report directories)
|
||||
* and prunes them together with the result retention.
|
||||
* Implemented by `de.hoennig.gittally.artifacts.FileArtifactStore`.
|
||||
*/
|
||||
interface ArtifactStore {
|
||||
/**
|
||||
* Takes over the staging directory of the finished [build] (log files) plus the
|
||||
* configured `artifactDirs` from [workspace] and stores them under the build's
|
||||
* artifact key. [workspace] is null when the build crashed before its workspace
|
||||
* was prepared; only the logs are stored then.
|
||||
*/
|
||||
fun persist(
|
||||
build: BuildResult,
|
||||
stagingDir: Path,
|
||||
workspace: Path?,
|
||||
)
|
||||
}
|
||||
|
||||
/** Placeholder until step 05: logs and leaves the staging directory untouched. */
|
||||
@Component
|
||||
class NoOpArtifactStore : ArtifactStore {
|
||||
private val log = LoggerFactory.getLogger(NoOpArtifactStore::class.java)
|
||||
/**
|
||||
* Deletes all stored artifact directories whose keys are not in [keptResults];
|
||||
* call after repository retention pruning. Returns the removed artifact keys.
|
||||
*/
|
||||
fun prune(keptResults: Collection<BuildResult>): List<String>
|
||||
|
||||
override fun persist(
|
||||
build: BuildResult,
|
||||
stagingDir: Path,
|
||||
) {
|
||||
log.info("artifact store not implemented yet; leaving build output of {} in {}", build.artifactKey, stagingDir)
|
||||
}
|
||||
/** The stored artifact directory for [artifactKey], or null if none exists. */
|
||||
fun artifactDir(artifactKey: String): Path?
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ class BuildExecutor(
|
||||
private fun execute(build: ActiveBuild) {
|
||||
var slot: Semaphore? = null
|
||||
var finalStatus = BuildStatus.FAILED
|
||||
var workspace: Path? = null
|
||||
try {
|
||||
slot = slotsFor(build.workingDir)
|
||||
slot.acquire()
|
||||
@@ -120,13 +121,14 @@ class BuildExecutor(
|
||||
}
|
||||
build.running = true
|
||||
transition(build, BuildStatus.RUNNING, duration = null)
|
||||
val workspace =
|
||||
val preparedWorkspace =
|
||||
workspaces.prepare(
|
||||
branch = build.runningBuild.branch,
|
||||
commit = build.runningBuild.commit,
|
||||
repoDir = build.workingDir,
|
||||
)
|
||||
val exitCode = runBuildCommands(build, workspace)
|
||||
workspace = preparedWorkspace
|
||||
val exitCode = runBuildCommands(build, preparedWorkspace)
|
||||
finalStatus =
|
||||
when {
|
||||
build.cancelled.get() -> BuildStatus.CANCELLED
|
||||
@@ -141,7 +143,7 @@ class BuildExecutor(
|
||||
val duration = Duration.between(build.runningBuild.startedAt, Instant.now())
|
||||
val result = transition(build, finalStatus, duration)
|
||||
try {
|
||||
artifactStore.persist(result, build.runningBuild.stagingDir)
|
||||
artifactStore.persist(result, build.runningBuild.stagingDir, workspace)
|
||||
} catch (e: Exception) {
|
||||
log.warn("could not persist artifacts of {}: {}", result.artifactKey, e.message)
|
||||
}
|
||||
|
||||
@@ -117,8 +117,10 @@ class InitCommand(
|
||||
# how many branches may build at the same time (at most one build per branch regardless)
|
||||
maxConcurrent: 1
|
||||
|
||||
# Build artifact retention.
|
||||
# Build artifact storage and retention.
|
||||
artifacts:
|
||||
# root directory for stored artifacts; empty = XDG_STATE_HOME (or ~/.local/state) + /gittally/artifacts/<repo-key>
|
||||
rootDir: ""
|
||||
# number of builds to keep per branch
|
||||
retentionPerBranch: 3
|
||||
|
||||
|
||||
@@ -33,6 +33,11 @@ data class BuildsConfig(
|
||||
|
||||
data class ArtifactsConfig(
|
||||
val retentionPerBranch: Int = 3,
|
||||
/**
|
||||
* Root directory for stored build artifacts; empty means the platform default
|
||||
* `XDG_STATE_HOME` (or `~/.local/state`) + `/gittally/artifacts/<repo-key>`.
|
||||
*/
|
||||
val rootDir: String = "",
|
||||
)
|
||||
|
||||
data class WatcherConfig(
|
||||
|
||||
Reference in New Issue
Block a user