werkator/tools/self-deploy

#!/usr/bin/env bash
#
# Installs a Werkator runtime bundle into the instance that is running this very
# script, and asks systemd — from outside the service — to restart it (RFC 0005,
# step 1).
#
# This is the `buildCommand` of the repository's `deploy` build: a follow-up of
# the test build on main, run natively (never in the sandbox, it has to reach the
# installation directory). Werkator is the one repository its own instance builds
# but cannot deploy the ordinary way, because the deploy build is a child of the
# process it replaces: the stop ends the build, the build is recorded INTERRUPTED,
# and the startup recovery re-runs it — a loop, not a glitch. So the build only
# stages, verifies and swaps the runtime, and schedules the restart as a transient
# systemd unit. That unit lives outside the service's control group, fires after
# the build is recorded SUCCESS, and the recovery therefore leaves it alone.
#
# Since step 2 the instance ends itself: a build with `restartsInstance` drains its
# queue and closes its context once nothing executes any more, and it exports
# WERKATOR_RESTARTS_INSTANCE=true into this script's environment so it knows. The
# transient unit is then no longer what stops the service — it waits for the JVM to
# go, swaps into the gap that `RestartSec` leaves, and starts the new version. The
# unit stays either way, because the rollback needs it: an instance that does not
# start cannot put the previous runtime back itself.
#
# Usage:
#   tools/self-deploy                        deploy build: stage and verify, then let the
#                                            transient unit swap and restart
#   tools/self-deploy --no-restart           stage, verify and swap at once, leaving the
#                                            restart to the caller
#   tools/self-deploy --install-dir DIR      installation directory to swap in (default:
#                                            derived from the running unit's ExecStart)
#   tools/self-deploy --self-restart-pid PID  (transient unit only) wait for that pid to
#                                            go instead of stopping the instance
#   tools/self-deploy --bundle SRC           the runtime bundle, as the directory
#                                            build/runtime-bundle/werkator or as a
#                                            tarball (default: the directory in this
#                                            worktree, else the tarball)
#
# Given both an installation directory and a bundle it swaps and nothing else —
# no git worktree, no systemd, no running instance to ask. That is how tools/remote
# installs a bundle from a developer machine, where the service is stopped and
# started around the swap by that script itself.
#
# Everything else it derives from where it runs:
#   - the primary checkout from `git rev-parse --git-common-dir`, and from its
#     basename the unit name exactly as SystemdServiceFiles.unitName does;
#   - the installation directory from that unit's ExecStart, whose jar sits in
#     <install>/werkator/lib/;
#   - the instance's port from <install>/werkator/bin/werkator config:print.
#
# The build never installs anything. It puts the new runtime beside the running one
# as `werkator.new` and runs its launcher — which is what catches a jlinked JRE that
# does not start on this host — and there it stops. The transient unit does the rest
# while the service is down: stop, swap, start. That order matters, because replacing
# the directory under a running JVM leaves it pointing at a foreign `jspawnhelper`,
# and from that moment it cannot start a single child process — a watcher that fetches
# nothing and a build that runs nothing (PR#64).
#
# The runtime that is replaced stays as `werkator.prev`: the last known good version,
# and what the unit puts back when the restarted service does not answer.
#
# A self-deployment takes the bundle as a directory, because it was built on this
# very host: packing 64 MiB and unpacking it seconds later would be work for
# nobody. A tarball is what a transfer from another machine needs, and both are
# accepted here.

set -euo pipefail

# Only the head start the transient unit needs to come up. It does not have to cover
# the deploy build any more: the unit waits for the instance to go idle, and a build
# is gone from the running list only after its status and artifacts are written — the
# executor guarantees that order. What used to be a guess of thirty seconds is now a
# condition, and it is met as soon as it is true (PR#64).
RESTART_DELAY_SECONDS=5
# How long the unit waits for the instance to go idle before restarting anyway. A
# build interrupted then is re-enqueued by the startup recovery, so the deployment
# is never lost — but it is not free either, hence the generous bound.
IDLE_TIMEOUT_SECONDS=600
IDLE_INTERVAL_SECONDS=5
# How long the watchdog waits for the restarted instance to answer before it
# considers the new version broken and puts the previous one back.
HEALTH_TIMEOUT_SECONDS=120
HEALTH_INTERVAL_SECONDS=5
BUNDLE_DIR_RELATIVE_PATH="build/runtime-bundle/werkator"
BUNDLE_TARBALL_RELATIVE_PATH="build/distributions/werkator-runtime-linux-x64.tar.gz"
# The copy of this script that the transient unit runs; it lives in the
# installation, which is per instance, so the name needs no suffix.
WATCHDOG_NAME="werkator-self-restart"

die() {
    echo "ERROR: $*" >&2
    exit 1
}

usage() {
    awk 'NR > 2 && !/^#/ { exit } NR > 2 { sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"
    exit 2
}

# systemctl --user and systemd-run need the user manager's bus. The deploy build
# is a child of the service, so it normally inherits both; a manual run from a
# non-login shell (or a cron-like context) does not, and guessing the default
# paths is what every systemd user unit does anyway.
ensure_user_bus() {
    : "${XDG_RUNTIME_DIR:=/run/user/$(id -u)}"
    export XDG_RUNTIME_DIR
    if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then
        export DBUS_SESSION_BUS_ADDRESS="unix:path=$XDG_RUNTIME_DIR/bus"
    fi
    command -v systemctl >/dev/null 2>&1 ||
        die "no systemctl on this host — self-deployment needs a systemd user manager (RFC 0005 step 1)"
    systemctl --user show --property=Version >/dev/null 2>&1 ||
        die "no systemd user manager reachable (XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR) — self-deployment needs one"
}

# Mirrors SystemdServiceFiles.unitName: the repository directory's name, every
# character outside [A-Za-z0-9_.-] replaced by a dash.
unit_name_for() {
    echo "werkator-$(basename "$1" | sed 's/[^A-Za-z0-9_.-]/-/g').service"
}

# The transient unit that restarts and verifies, named after the service it
# restarts: transient unit names are global to the user manager, and two instances
# under one account deploying at the same time must not collide on one name.
restart_unit_for() {
    echo "werkator-self-restart-$(basename "$1" .service | sed 's/^werkator-//')"
}

# The primary checkout of the repository this build belongs to — the directory the
# service watches, and the one that names its unit. In a build worktree the common
# git dir is the primary's `.git`; in the primary checkout itself it is `.git` too,
# which is why the caller checks for a linked worktree separately.
primary_checkout() {
    local common_dir
    common_dir="$(git rev-parse --git-common-dir 2>/dev/null)" ||
        die "not inside a git repository — run this as the deploy build's command"
    case "$common_dir" in
        /*) ;;
        *) common_dir="$PWD/$common_dir" ;;
    esac
    dirname "$(cd "$common_dir" && pwd)"
}

# A build runs in .git/werkator/worktrees/<branchKey>, never in the primary
# checkout — and a deployment that swapped the runtime out from under the
# repository it was started from would be exactly the accident worktrees exist to
# prevent.
require_linked_worktree() {
    local git_dir common_dir
    git_dir="$(git rev-parse --absolute-git-dir 2>/dev/null)" ||
        die "not inside a git repository"
    common_dir="$(git rev-parse --git-common-dir 2>/dev/null)"
    case "$common_dir" in
        /*) ;;
        *) common_dir="$PWD/$common_dir" ;;
    esac
    common_dir="$(cd "$common_dir" && pwd)"
    [ "$git_dir" != "$common_dir" ] ||
        die "this is the primary checkout — the deploy build runs in a worktree under .git/werkator/worktrees/"
}

# The installation directory, read from the unit that is actually running: its
# ExecStart names the jar, which sits in <install>/werkator/lib/werkator.jar.
install_dir_from_unit() {
    local unit="$1" exec_start jar bundle_dir
    exec_start="$(systemctl --user show -p ExecStart --value "$unit" 2>/dev/null || true)"
    [ -n "$exec_start" ] ||
        die "the unit $unit is not known to the user manager — is this the host the instance runs on?"
    jar="$(printf '%s\n' "$exec_start" | tr ' ' '\n' | grep -m1 '\.jar$' || true)"
    [ -n "$jar" ] || die "cannot read a jar path from the ExecStart of $unit: $exec_start"
    bundle_dir="$(dirname "$(dirname "$jar")")"
    [ "$(basename "$bundle_dir")" = "werkator" ] &&
        [ -x "$bundle_dir/bin/werkator" ] &&
        [ -d "$bundle_dir/jre" ] ||
        die "$unit does not run a runtime bundle (jar: $jar) — self-deployment installs bundles only, see docs/deployment.md"
    dirname "$bundle_dir"
}

# The port the instance serves on, straight from the installed binary: config:print
# is the single answer to "what is configured", wherever the key lives.
port_of_instance() {
    local install_dir="$1" repo_dir="$2" port
    port="$(cd "$repo_dir" && "$install_dir/werkator/bin/werkator" config:print 2>/dev/null |
        awk '/^server:/{f=1;next} f && /^  port:/{print $2; exit}' | tr -d '"' || true)"
    [ -n "$port" ] || die "cannot determine the server port from $install_dir/werkator/bin/werkator config:print"
    echo "$port"
}

# Unpacks beside the running runtime and proves the new launcher starts on this host.
# Nothing of the installation is touched: what this leaves behind is `werkator.new`.
stage_and_verify() {
    local install_dir="$1" bundle="$2" staging version
    staging="$install_dir/.werkator-stage"
    echo "==> Staging $bundle in $install_dir"
    rm -rf "$staging" "$install_dir/werkator.new"
    if [ -d "$bundle" ]; then
        # already unpacked, built on this host by the preceding build
        cp -a "$bundle" "$install_dir/werkator.new"
    else
        mkdir -p "$staging"
        tar xzf "$bundle" -C "$staging"
        [ -d "$staging/werkator" ] ||
            die "the bundle does not contain a top-level werkator/ directory: $bundle"
        mv "$staging/werkator" "$install_dir/werkator.new"
        rmdir "$staging"
    fi

    echo "==> Verifying the staged runtime natively"
    version="$("$install_dir/werkator.new/bin/werkator" --version 2>&1)" || {
        echo "$version" >&2
        rm -rf "$install_dir/werkator.new"
        die "the staged runtime does not run on this host — nothing was swapped"
    }
    echo "    $version"
}

# Puts the staged runtime in place. The unit's ExecStart keeps pointing at the same
# path, because only the directory's content changes, never its name — which is also
# why this must not happen while that JVM is running.
swap_staged() {
    local install_dir="$1"
    [ -d "$install_dir/werkator.new" ] || die "nothing staged in $install_dir — werkator.new is missing"
    echo "==> Swapping in the staged runtime (the replaced one stays as werkator.prev)"
    rm -rf "$install_dir/werkator.prev"
    [ ! -d "$install_dir/werkator" ] || mv "$install_dir/werkator" "$install_dir/werkator.prev"
    mv "$install_dir/werkator.new" "$install_dir/werkator"
}

# Answers once the instance serves again; the watcher endpoint is the cheapest
# route that proves the server context came up, not just the port.
instance_answers() {
    local port="$1"
    curl -fsS --max-time 5 "http://127.0.0.1:$port/api/watcher" >/dev/null 2>&1
}

await_instance() {
    local port="$1" waited=0
    while [ "$waited" -lt "$HEALTH_TIMEOUT_SECONDS" ]; do
        if instance_answers "$port"; then
            return 0
        fi
        sleep "$HEALTH_INTERVAL_SECONDS"
        waited=$((waited + HEALTH_INTERVAL_SECONDS))
    done
    return 1
}

# True while the instance has a build running or queued, over all of its repositories.
# An unreachable instance runs nothing, so it counts as idle.
instance_busy() {
    local port="$1" body
    body="$(curl -fsS --max-time 10 "http://127.0.0.1:$port/api/repos" 2>/dev/null)" || return 1
    printf '%s' "$body" | grep -qE '"(running|pending)":[1-9]'
}

# Waits for the deploy build that scheduled this unit to be recorded, and for
# anything else to finish with it. A build leaves the running list only after its
# status and its artifacts are written — that order is an invariant of the executor —
# so an idle instance means the deployment is safely in the history and the restart
# cannot turn it into an interrupted build that deploys all over again.
await_idle() {
    local port="$1" waited=0
    while [ "$waited" -lt "$IDLE_TIMEOUT_SECONDS" ]; do
        if ! instance_busy "$port"; then
            [ "$waited" -eq 0 ] || echo "instance idle after ${waited}s"
            return 0
        fi
        sleep "$IDLE_INTERVAL_SECONDS"
        waited=$((waited + IDLE_INTERVAL_SECONDS))
    done
    echo "instance still busy after ${IDLE_TIMEOUT_SECONDS}s — restarting anyway; a running build is re-enqueued by the recovery" >&2
    return 0
}

# The pid of the running instance, 0 while the service is down. Passed to the
# transient unit at schedule time, so that waiting for "this instance is gone" is a
# comparison and not a guess: the pid read afterwards is either still the one that
# scheduled the restart, 0 while it is down, or already the next one.
main_pid_of() {
    systemctl --user show -p MainPID --value "$1" 2>/dev/null || echo 0
}

# Waits for the instance that scheduled this unit to end itself (RFC 0005, step 2).
# Nothing is stopped here: the executor drains its queue, records every build that was
# still executing, and only then closes its context — so the moment the pid changes,
# the deployment is in the history and no build is being cut short. `RestartSec=30`
# then leaves the gap the swap needs.
await_instance_exit() {
    local unit="$1" was="$2" waited=0
    while [ "$waited" -lt "$IDLE_TIMEOUT_SECONDS" ]; do
        if [ "$(main_pid_of "$unit")" != "$was" ]; then
            echo "the instance ended itself after ${waited}s (was pid $was)"
            return 0
        fi
        sleep "$IDLE_INTERVAL_SECONDS"
        waited=$((waited + IDLE_INTERVAL_SECONDS))
    done
    echo "instance $was did not end itself within ${IDLE_TIMEOUT_SECONDS}s — stopping it; its running build is re-enqueued by the recovery" >&2
    return 0
}

# Runs as the transient unit, once the build that staged the runtime has ended:
# wait for the instance to go idle, stop it, swap, start, and put the previous
# runtime back when the new one does not answer. Everything it prints lands in the
# journal of that unit. The swap happens here, between stop and start, so the JVM
# being replaced never sees its own runtime change underneath it.
restart_and_verify() {
    local unit="$1" install_dir="$2" port="$3" self_restart_pid="${4:-}"
    ensure_user_bus
    if [ -n "$self_restart_pid" ]; then
        await_instance_exit "$unit" "$self_restart_pid"
    else
        await_idle "$port"
    fi
    # also after the instance ended itself: `Restart=always` has the old runtime
    # queued to come back, and swapping under it is what PR#64 was about
    echo "stopping $unit to swap in the staged runtime"
    systemctl --user stop "$unit" || echo "systemctl stop reported a failure; swapping anyway" >&2
    swap_staged "$install_dir"
    echo "starting $unit into the newly installed runtime"
    # a failed start command must not skip the health check: whether the
    # instance answers is what decides, not what systemctl reported
    systemctl --user start "$unit" || echo "systemctl start reported a failure; checking the instance anyway" >&2
    if await_instance "$port"; then
        echo "the new runtime answers on port $port"
        return 0
    fi
    echo "the new runtime did not answer on port $port within ${HEALTH_TIMEOUT_SECONDS}s — rolling back to werkator.prev" >&2
    if [ ! -d "$install_dir/werkator.prev" ]; then
        echo "no werkator.prev to roll back to; leaving the installation as it is" >&2
        return 1
    fi
    rm -rf "$install_dir/werkator.failed"
    mv "$install_dir/werkator" "$install_dir/werkator.failed"
    mv "$install_dir/werkator.prev" "$install_dir/werkator"
    systemctl --user restart "$unit"
    if await_instance "$port"; then
        echo "rolled back; the previous runtime answers again (the failed one is werkator.failed)" >&2
    else
        echo "rolled back, but the previous runtime does not answer either — manual intervention needed" >&2
    fi
    return 1
}

# Schedules the restart to happen after this build has ended. The transient unit is
# started by the user manager, so stopping the service does not end it — which is
# the whole point: a restart triggered from inside the build would kill the build.
schedule_restart() {
    local unit="$1" install_dir="$2" port="$3" watchdog restart_unit
    local -a self_restart=()
    watchdog="$install_dir/bin/$WATCHDOG_NAME"
    restart_unit="$(restart_unit_for "$unit")"
    # read here, while the instance that runs this build is still the running one
    if [ "${WERKATOR_RESTARTS_INSTANCE:-}" = "true" ]; then
        self_restart=(--self-restart-pid "$(main_pid_of "$unit")")
    fi
    mkdir -p "$install_dir/bin"
    # from the installation, not from the worktree: the worktree is reused by the
    # next build of this branch and may hold another commit by the time this runs
    install -m 755 "$(readlink -f "${BASH_SOURCE[0]}")" "$watchdog"
    systemctl --user stop "$restart_unit.timer" "$restart_unit.service" >/dev/null 2>&1 || true
    if [ "${#self_restart[@]}" -gt 0 ]; then
        echo "==> This instance ends itself after this build; the transient unit $restart_unit swaps and starts the new version"
    else
        echo "==> Scheduling the restart in ${RESTART_DELAY_SECONDS}s (transient unit $restart_unit)"
    fi
    systemd-run --user --collect --quiet \
        --on-active="$RESTART_DELAY_SECONDS" \
        --unit="$restart_unit" \
        "$watchdog" --restart-and-verify --unit "$unit" --install-dir "$install_dir" --port "$port" \
        ${self_restart[@]+"${self_restart[@]}"}
    echo "    journalctl --user -u $restart_unit -f    follows the restart and its health check"
}

main() {
    local mode="deploy" install_dir="" bundle="" unit="" port="" self_restart_pid=""
    while [ $# -gt 0 ]; do
        case "$1" in
            --no-restart) mode="swap-only"; shift ;;
            --restart-and-verify) mode="watchdog"; shift ;;
            --self-restart-pid) self_restart_pid="${2:?missing value for --self-restart-pid}"; shift 2 ;;
            --install-dir) install_dir="${2:?missing value for --install-dir}"; shift 2 ;;
            --bundle) bundle="${2:?missing value for --bundle}"; shift 2 ;;
            --unit) unit="${2:?missing value for --unit}"; shift 2 ;;
            --port) port="${2:?missing value for --port}"; shift 2 ;;
            -h|--help) usage ;;
            *) die "unknown argument: $1 (try --help)" ;;
        esac
    done

    if [ "$mode" = "watchdog" ]; then
        [ -n "$unit" ] && [ -n "$install_dir" ] && [ -n "$port" ] ||
            die "--restart-and-verify needs --unit, --install-dir and --port"
        restart_and_verify "$unit" "$install_dir" "$port" "$self_restart_pid"
        return
    fi

    # Called by tools/remote with both of them (and --no-restart, which says at the
    # call site what this path does anyway): no git worktree, no systemd, no running
    # instance to ask — just the staged swap, the part the two must not implement twice.
    if [ -n "$install_dir" ] && [ -n "$bundle" ]; then
        [ -e "$bundle" ] || die "runtime bundle not found: $bundle"
        [ -d "$install_dir" ] || die "installation directory not found: $install_dir"
        stage_and_verify "$install_dir" "$bundle"
        swap_staged "$install_dir"
        return
    fi

    local repo_dir
    require_linked_worktree
    repo_dir="$(primary_checkout)"
    local worktree
    worktree="$(git rev-parse --show-toplevel)"
    # the directory first: the preceding build produced it here, and a bundle built
    # on this host has no reason to be packed and unpacked again
    if [ -z "$bundle" ]; then
        if [ -d "$worktree/$BUNDLE_DIR_RELATIVE_PATH" ]; then
            bundle="$worktree/$BUNDLE_DIR_RELATIVE_PATH"
        else
            bundle="$worktree/$BUNDLE_TARBALL_RELATIVE_PATH"
        fi
    fi
    # A restart of `deploy` alone runs without its predecessor, so the worktree
    # holds no bundle. Installing whatever happens to lie around would deploy an
    # older version than the one the History says was deployed.
    [ -e "$bundle" ] ||
        die "no runtime bundle at $worktree/$BUNDLE_DIR_RELATIVE_PATH — the deploy build runs after a green build that built it; restart that one"

    ensure_user_bus
    # the watchdog decides by asking the instance; without curl it would find every
    # deployment broken and roll back a perfectly good one
    command -v curl >/dev/null 2>&1 ||
        die "no curl on this host — the restart watchdog needs it to tell a started instance from a broken one"
    [ -n "$unit" ] || unit="$(unit_name_for "$repo_dir")"
    local working_dir
    working_dir="$(systemctl --user show -p WorkingDirectory --value "$unit" 2>/dev/null || true)"
    # compared resolved: a webspace reaches the same directory under more than one
    # path, and a symlink is no reason to refuse the deployment
    [ -z "$working_dir" ] || [ "$(readlink -f "$working_dir")" = "$(readlink -f "$repo_dir")" ] ||
        die "$unit watches $working_dir, but this build belongs to $repo_dir — refusing to deploy into a foreign instance"
    [ -n "$install_dir" ] || install_dir="$(install_dir_from_unit "$unit")"
    echo "==> Deploying into $install_dir (unit $unit, repository $repo_dir)"

    port="$(port_of_instance "$install_dir" "$repo_dir")"
    stage_and_verify "$install_dir" "$bundle"

    if [ "$mode" = "swap-only" ]; then
        # the caller stops and starts the service itself, so it owns the moment
        swap_staged "$install_dir"
        echo "==> Swapped; no restart scheduled (--no-restart)"
        return
    fi
    # staged and proven, installed by nobody yet: the transient unit swaps it in
    # while the service is down, so the running JVM keeps its own runtime
    schedule_restart "$unit" "$install_dir" "$port"
}

main "$@"