werkator/tools/remote

#!/usr/bin/env bash
#
# Central control script for remote Werkator operations (same pattern as the
# `remote` scripts in the other repos): the first argument is the repo selector,
# the second the command. All connection and deployment values come from the
# `.env` file in the repository root — never as command line parameters.
#
# Commands name their role (step 21 session D): `instance-*` manages the
# BUILDER — the installed Werkator instance and its werkdock sandbox tool —
# while `repo-*` acts on the BUILT, the repository the instance watches.
# Werkator is never built on the target: the instance is installed from the
# locally built runtime bundle (ADR 0006), and builds of the watched
# repository are the running instance's job (or `bin/werkator build` on the
# host — the werkator CLI, not this script).
#
# Usage (step 23: `--env-file` selects the target instance, default `.env`):
#   tools/remote [--env-file FILE] werkator check-prerequisites   werkdock doctor on the host
#   tools/remote [--env-file FILE] werkator instance-install      first-time: upload + unpack bundle and werkdock
#   tools/remote [--env-file FILE] werkator instance-update       redeploy bundle + werkdock, restart the service
#   tools/remote [--env-file FILE] werkator instance-start        apply fragment, Apache proxy, systemd unit
#   tools/remote [--env-file FILE] werkator repo-init             clone the watched repo, init --apply, rootfs
#   tools/remote [--env-file FILE] werkator repo-add URL [NAME]   clone and init ANOTHER repository for the
#                                   registry, then print the entry to add to ~/.werkator.yml
#   tools/remote [--env-file FILE] werkator control-token
#   tools/remote [--env-file FILE] port-forward start             background tunnel to the Werkator UI
#   tools/remote [--env-file FILE] port-forward stop
#
# The env file carries TRANSPORT values only; everything that is Werkator
# configuration travels as a YAML fragment in the config schema, named by
# WERKATOR_INIT_CONFIG and installed remotely via `werkator init --apply`
# (docs/plan/23-init-owns-the-files.md). Pair the files per instance, e.g.
# `.env.mih34` + `.env.mih34.yml` (both gitignored).
#
# Required in the env file:
#   WERKATOR_REMOTE   user@host to operate on, e.g. mih34-werkator@mih34.hostsharing.net
#   WERKATOR_PATH     target directory on that host, e.g. /home/storage/mih34/users/werkator
#
# Required for `instance-start`:
#   WERKATOR_DOMAIN        the domain served by the managed Apache (docroot location for the
#                          generated .htaccess); the port lives in the fragment (server.port)
#
# Required for `port-forward`:
#   WERKATOR_LOCAL_PORT    the local port the browser uses
# Optional in the env file:
#   WERKATOR_INIT_CONFIG   the init fragment to apply (repo-init, instance-start)
#   WERKATOR_REPO_URL      https clone URL of the watched repository
#                          (default: https://github.com/mhoennig/werkator.git)
#   WERKDOCK_REPO          checkout of the werkdock repository, whose binary the
#                          instance runs (default: <repo>/../werkdock)
#   WERKDOCK_BINARY        the built werkdock binary (default: $WERKDOCK_REPO/dist/werkdock)
#   WERKATOR_ROOTFS        rootfs archive path for repo-init
#                          (default: <repo>/build/werkator-buildenv-trixie-java-go-node.tar.zst)
#
# Install layout on the host:
#   $WERKATOR_PATH/werkator/            the watched repository (clone)
#   $WERKATOR_PATH/.werkator/werkator/  the unpacked runtime bundle
#   $WERKATOR_PATH/.werkator/bin/       the werkdock binary
#   $WERKATOR_PATH/.werkator/*.tar.*    uploaded bundle and rootfs archives
#

set -euo pipefail

die() { echo "ERROR: $*" >&2; exit 1; }

ENV_FILE=""
if [ "${1:-}" = "--env-file" ]; then
    ENV_FILE="${2:?missing value for --env-file}"
    shift 2
fi

REPO="${1:-}"
COMMAND="${2:-}"

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUNTIME_BUNDLE="$REPO_ROOT/build/distributions/werkator-runtime-linux-x64.tar.gz"
# Werkdock lives in its own repository since step 21 session E. Its binary is
# built there, not here: WERKDOCK_REPO names the checkout (default: a sibling
# of this repository), WERKDOCK_BINARY the built binary within it.
WERKDOCK_REPO="${WERKDOCK_REPO:-$REPO_ROOT/../werkdock}"
WERKDOCK_BINARY="${WERKDOCK_BINARY:-$WERKDOCK_REPO/dist/werkdock}"
PID_FILE="/tmp/werkator-port-forward-$(id -u).pid"
LOG_FILE="/tmp/werkator-port-forward-$(id -u).log"

usage() {
    awk 'NR > 2 && !/^#/ { exit } NR > 2 { sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"
    exit 2
}

require_env() {
    local missing=0
    for name in "$@"; do
        if [ -z "${!name:-}" ]; then
            echo "ERROR: $name is not set — define it in $ENV_FILE" >&2
            missing=1
        fi
    done
    [ "$missing" -eq 0 ] || exit 1
}

[ -n "$REPO" ] && [ -n "$COMMAND" ] || usage

# Load the transport values. The selected env file (default: the .env in the
# repository root) wins over the environment — `set -a; source` assigns
# unconditionally; pick the target with --env-file rather than by exporting
# single values.
ENV_FILE="${ENV_FILE:-$REPO_ROOT/.env}"
set -a
[ -f "$ENV_FILE" ] && source "$ENV_FILE"
set +a

require_env WERKATOR_REMOTE WERKATOR_PATH
HOST="$WERKATOR_REMOTE"
TARGET_DIR="$WERKATOR_PATH"
ROOTFS="${WERKATOR_ROOTFS:-$REPO_ROOT/build/werkator-buildenv-trixie-java-go-node.tar.zst}"
REPO_URL="${WERKATOR_REPO_URL:-https://github.com/mhoennig/werkator.git}"
MACHINE_CONFIG="$TARGET_DIR/werkator/.git/werkator/.werkator.yml"
WERKATOR_BIN="$TARGET_DIR/.werkator/werkator/bin/werkator"
UNIT="werkator-werkator.service"

ssh_present() {
    ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null
}

ensure_ssh() {
    if ssh_present; then
        echo "==> SSH access to $HOST: ok"
    else
        echo "==> No key-based SSH access yet; running ssh-copy-id (password prompt expected)"
        ssh-copy-id "$HOST"
        ssh_present || die "SSH access still not working after ssh-copy-id"
    fi
}

# Werkdock owns the host checks (`werkdock doctor` ports the old prerequisites
# script); the binary is uploaded first, so the check works pre-install.
check_prerequisites() {
    ensure_werkdock_binary
    echo "==> Uploading werkdock and running its doctor on $HOST (target dir: $TARGET_DIR)"
    ssh "$HOST" "mkdir -p '$TARGET_DIR/.werkator/bin'"
    scp -q "$WERKDOCK_BINARY" "$HOST:$TARGET_DIR/.werkator/bin/werkdock.new"
    ssh "$HOST" "mv '$TARGET_DIR/.werkator/bin/werkdock.new' '$TARGET_DIR/.werkator/bin/werkdock' && chmod 755 '$TARGET_DIR/.werkator/bin/werkdock'"
    if ! ssh "$HOST" "'$TARGET_DIR/.werkator/bin/werkdock' doctor '$TARGET_DIR'"; then
        die "werkdock doctor failed on $HOST — install aborted"
    fi
}

ensure_werkdock_binary() {
    if [ ! -f "$WERKDOCK_BINARY" ]; then
        [ -f "$WERKDOCK_REPO/go.mod" ] || die \
            "werkdock binary missing: $WERKDOCK_BINARY — clone https://git.javagil.de/mi/werkdock.git \
next to this repository, or point WERKDOCK_REPO/WERKDOCK_BINARY at your checkout"
        echo "==> werkdock binary not found; building it in $WERKDOCK_REPO (go build)"
        (cd "$WERKDOCK_REPO" && CGO_ENABLED=0 go build -o dist/werkdock .)
    fi
    [ -f "$WERKDOCK_BINARY" ] || die "werkdock binary missing: $WERKDOCK_BINARY"
}

# Uploads the init fragment named by WERKATOR_INIT_CONFIG and echoes its remote
# path; empty when no fragment is configured.
upload_fragment() {
    [ -n "${WERKATOR_INIT_CONFIG:-}" ] || { echo ""; return 0; }
    [ -f "$WERKATOR_INIT_CONFIG" ] || die "init fragment missing: $WERKATOR_INIT_CONFIG"
    local remote="$TARGET_DIR/.werkator/$(basename "$WERKATOR_INIT_CONFIG")"
    scp -q "$WERKATOR_INIT_CONFIG" "$HOST:$remote"
    echo "$remote"
}

# The instance artifacts are built locally (ADR 0006): the runtime bundle via
# Gradle, the werkdock binary via the Go toolchain. Both are rebuilt when
# missing, never on the target.
ensure_instance_artifacts() {
    if [ ! -f "$RUNTIME_BUNDLE" ]; then
        echo "==> Runtime bundle not found; building it locally (./gradlew runtimeBundle)"
        (cd "$REPO_ROOT" && ./gradlew runtimeBundle --console=plain -q)
    fi
    [ -f "$RUNTIME_BUNDLE" ] || die "runtime bundle missing: $RUNTIME_BUNDLE"
    ensure_werkdock_binary
}

# Uploads and unpacks the instance artifacts. The previous runtime stays as
# werkator.prev for one deployment as the rollback asset.
deploy_instance() {
    echo "==> Uploading runtime bundle and werkdock binary"
    ssh "$HOST" "mkdir -p '$TARGET_DIR/.werkator/bin'"
    scp -q "$RUNTIME_BUNDLE" "$HOST:$TARGET_DIR/.werkator/"
    scp -q "$WERKDOCK_BINARY" "$HOST:$TARGET_DIR/.werkator/bin/werkdock.new"
    echo "==> Unpacking"
    ssh "$HOST" "set -e
        cd '$TARGET_DIR/.werkator'
        mv bin/werkdock.new bin/werkdock && chmod 755 bin/werkdock
        rm -rf werkator.prev
        [ ! -d werkator ] || mv werkator werkator.prev
        tar xzf '$(basename "$RUNTIME_BUNDLE")'
        './werkator/bin/werkator' --version
        './bin/werkdock' version"
}

instance_install() {
    ensure_ssh
    check_prerequisites
    ensure_instance_artifacts
    deploy_instance
    echo
    echo "==> Instance installed."
    echo "    Runtime:  $WERKATOR_BIN"
    echo "    werkdock: $TARGET_DIR/.werkator/bin/werkdock"
    echo "    Next:     tools/remote werkator repo-init, then instance-start"
}

# Refuse to swap the runtime under a running build; FORCE=1 overrides.
require_idle() {
    local port
    port="$(ssh "$HOST" "cd '$TARGET_DIR/werkator' 2>/dev/null && '$WERKATOR_BIN' config:print 2>/dev/null" | awk '/^server:/{f=1;next} f && /^  port:/{print $2; exit}' | tr -d '"' || true)"
    [ -n "$port" ] || return 0
    local current
    current="$(ssh "$HOST" "curl -s --max-time 5 http://127.0.0.1:$port/api/builds/current" || true)"
    if [ -n "$current" ] && [ "$current" != "[]" ]; then
        [ "${FORCE:-}" = "1" ] || die "a build is running on $HOST — retry when idle, or FORCE=1 to override"
        echo "==> WARNING: deploying although a build is running (FORCE=1)"
    fi
}

instance_update() {
    ensure_ssh
    ensure_instance_artifacts
    require_idle
    local was_active=0
    if ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user is-active --quiet '$UNIT'"; then
        was_active=1
    fi
    if [ "$was_active" = "1" ]; then
        echo "==> Stopping $UNIT"
        ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user stop '$UNIT'"
    fi
    deploy_instance
    if [ "$was_active" = "1" ]; then
        echo "==> Starting $UNIT"
        ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user start '$UNIT' && sleep 3 && systemctl --user is-active '$UNIT'"
    else
        echo "==> Service was not running; not started (use instance-start for the first start)"
    fi
    echo "==> Instance updated."
}

# Sets up the WATCHED repository: an anonymous https clone (a private origin
# gets its credentials via git.account/git.token in the machine config that
# `werkator init` creates), the werkator init with the instance fragment
# applied, and the rootfs archive for the sandbox builds. All configuration
# writing is init's — this script transports and invokes (step 23).
repo_init() {
    ensure_ssh
    [ -f "$ROOTFS" ] || die "rootfs archive missing: $ROOTFS — build it with tools/build-bwrap-rootfs.sh or set WERKATOR_ROOTFS"
    ssh "$HOST" "test -x '$WERKATOR_BIN'" || die "no instance on $HOST — run instance-install first"

    echo "==> Cloning the watched repository"
    if ssh "$HOST" "test -d '$TARGET_DIR/werkator/.git'"; then
        echo "    (already cloned, skipping)"
    else
        ssh "$HOST" "git clone '$REPO_URL' '$TARGET_DIR/werkator'"
    fi

    echo "==> Uploading the rootfs archive (skipped when unchanged)"
    local rootfs_remote="$TARGET_DIR/.werkator/$(basename "$ROOTFS")"
    local local_sha remote_sha
    local_sha="$(sha256sum "$ROOTFS" | cut -d' ' -f1)"
    remote_sha="$(ssh "$HOST" "sha256sum '$rootfs_remote' 2>/dev/null | cut -d' ' -f1" || true)"
    if [ "$local_sha" = "$remote_sha" ]; then
        echo "    (already on the host, skipping)"
    else
        scp -q "$ROOTFS" "$HOST:$rootfs_remote"
        remote_sha="$(ssh "$HOST" "sha256sum '$rootfs_remote' | cut -d' ' -f1")"
        [ "$local_sha" = "$remote_sha" ] || die "rootfs upload checksum mismatch"
    fi

    echo "==> Running werkator init${WERKATOR_INIT_CONFIG:+ --apply $(basename "${WERKATOR_INIT_CONFIG}")}"
    local fragment_remote
    fragment_remote="$(upload_fragment)"
    ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' init ${fragment_remote:+--apply '$fragment_remote'}"

    echo "==> Verifying the effective configuration"
    ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' config:print 2>/dev/null | grep -A4 'bwrap:' | head -5"

    echo
    echo "==> Repository ready."
    echo "    Repo: $TARGET_DIR/werkator"
    echo "    Next: fill git.account/git.token in $MACHINE_CONFIG if the origin is private,"
    echo "          then tools/remote werkator instance-start"
}

# Prepare a SECOND (third, …) repository for the registry of an installed
# instance (ADR 0009): clone it next to the others and run `init` in it, so it
# has its own machine config. The registry entry itself is only PRINTED, never
# written: `~/.werkator.yml` is the instance's own file — it carries the port,
# the global concurrency and possibly shared credentials, and a script that
# edits it in place would rewrite the operator's own configuration behind their
# back. Cloning and initialising is mechanical, registering is a decision.
repo_add() {
    local url="${1:-}"
    [ -n "$url" ] || die "usage: tools/remote [--env-file FILE] werkator repo-add <clone-url> [name]"
    local name="${2:-$(basename "$url" .git)}"
    case "$name" in
        */*|"") die "the repository name is one path segment (it becomes the route segment /repos/<name>)" ;;
    esac

    ensure_ssh
    ssh "$HOST" "test -x '$WERKATOR_BIN'" || die "no instance on $HOST — run instance-install first"

    echo "==> Cloning $url as '$name'"
    if ssh "$HOST" "test -d '$TARGET_DIR/$name/.git'"; then
        echo "    (already cloned, skipping)"
    else
        ssh "$HOST" "git clone '$url' '$TARGET_DIR/$name'"
    fi

    # The instance fragment carries the sandbox policy (bwrap rootfs and werkdock
    # binary). Without it a watched repository builds on the bare host, where the
    # toolchains are not installed — the same --apply repo-init does.
    echo "==> Running werkator init in $name${WERKATOR_INIT_CONFIG:+ --apply $(basename "${WERKATOR_INIT_CONFIG}")}"
    local fragment_remote
    fragment_remote="$(upload_fragment)"
    ssh "$HOST" "cd '$TARGET_DIR/$name' && '$WERKATOR_BIN' init ${fragment_remote:+--apply '$fragment_remote'}"

    echo "==> Checking the registry"
    # Grepped locally: the entry may name the path absolute or as ~/<name>, and
    # matching both is easier without a second layer of remote shell quoting.
    if ssh "$HOST" "cat ~/.werkator.yml 2>/dev/null" |
        grep -qE "path: *(~|$TARGET_DIR)/$name[[:space:]]*$"; then
        echo "    (~/.werkator.yml already names this path)"
    else
        echo "    not registered yet — add this entry to ~/.werkator.yml on $HOST:"
        echo
        echo "        repositories:"
        echo "          - path: $TARGET_DIR/$name"
        echo "            name: $name"
        echo
    fi

    echo "==> Repository prepared."
    echo "    Repo: $TARGET_DIR/$name"
    echo "    Next: fill git.account/git.token in $TARGET_DIR/$name/.git/werkator/.werkator.yml if the origin is private"
    echo "          (or once for all repositories in the 'defaults' block of ~/.werkator.yml),"
    echo "          then restart the service — the registry is read at start."
}

# Start the server as a systemd user unit behind the managed Apache. All
# configuration comes from the instance fragment (server.port, publicBaseUrl,
# systemd limits); init generates the units AND the .htaccess — this script
# only places and activates them (step 23).
instance_start() {
    ensure_ssh
    require_env WERKATOR_DOMAIN

    echo "==> Applying the instance fragment and generating the host integration (init --systemd)"
    local fragment_remote
    fragment_remote="$(upload_fragment)"
    ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' init ${fragment_remote:+--apply '$fragment_remote'} --systemd"

    local htaccess_src="$TARGET_DIR/werkator/.git/werkator/werkator.htaccess"
    local htaccess="$TARGET_DIR/doms/$WERKATOR_DOMAIN/subs/www/.htaccess"
    if ssh "$HOST" "test -f '$htaccess_src'"; then
        echo "==> Placing the generated Apache reverse proxy at $htaccess"
        ssh "$HOST" "mkdir -p '$TARGET_DIR/doms/$WERKATOR_DOMAIN/subs/www' && cp '$htaccess_src' '$htaccess'"
    else
        echo "==> No werkator.htaccess generated (no server.publicBaseUrl configured) — skipping the Apache proxy"
    fi

    echo "==> Linking the units into ~/.config/systemd/user and enabling the service"
    ssh "$HOST" "mkdir -p ~/.config/systemd/user && \
      ln -sf '$TARGET_DIR/werkator/.git/werkator/$UNIT' ~/.config/systemd/user/ && \
      ln -sf '$TARGET_DIR/werkator/.git/werkator/werkator-docker-prune.service' ~/.config/systemd/user/ && \
      ln -sf '$TARGET_DIR/werkator/.git/werkator/werkator-docker-prune.timer' ~/.config/systemd/user/ && \
      XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user daemon-reload && \
      XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user restart '$UNIT' && \
      XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user status '$UNIT' --no-pager -l | head -12"

    echo
    echo "==> Server started. Verify: https://$WERKATOR_DOMAIN/"
    echo "    Logs: ssh $HOST -- systemctl --user status '$UNIT'"
}

# Background SSH tunnel to the Werkator server, so the browser reaches the UI
# at http://localhost:<WERKATOR_LOCAL_PORT> without keeping a terminal busy.
# `start` runs ssh -N -L detached with a pid file; `stop` kills it.
port_forward() {
    require_env WERKATOR_LOCAL_PORT
    # the effective port, wherever it is configured (machine config or applied
    # fragment) — config:print is the single answer, not this script's parser
    local remote_port
    remote_port="$(ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' config:print 2>/dev/null" | awk '/^server:/{f=1;next} f && /^  port:/{print $2; exit}' | tr -d '"')"
    [ -n "$remote_port" ] || die "no server.port configured — run 'tools/remote werkator instance-start' first"

    case "$COMMAND" in
        start)
            if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
                echo "==> Port-forward already running (pid $(cat "$PID_FILE")) — http://localhost:$WERKATOR_LOCAL_PORT"
                exit 0
            fi
            nohup ssh -N -L "$WERKATOR_LOCAL_PORT:127.0.0.1:$remote_port" "$HOST" \
                >"$LOG_FILE" 2>&1 &
            echo $! > "$PID_FILE"
            sleep 1
            if kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
                echo "==> Forwarding http://localhost:$WERKATOR_LOCAL_PORT -> $HOST:127.0.0.1:$remote_port (pid $(cat "$PID_FILE"))"
            else
                echo "ERROR: port-forward failed to start — see $LOG_FILE" >&2
                rm -f "$PID_FILE"
                exit 1
            fi
            ;;
        stop)
            if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
                kill "$(cat "$PID_FILE")"
                rm -f "$PID_FILE"
                echo "==> Port-forward stopped"
            else
                rm -f "$PID_FILE"
                echo "==> Port-forward is not running"
            fi
            ;;
        *)
            echo "ERROR: unknown port-forward command: $COMMAND (use start or stop)" >&2
            exit 2
            ;;
    esac
}

# Print the control token guarding the mutating build endpoints — the werkator
# CLI owns creation and format (step 23), this script only invokes it.
control_token() {
    ensure_ssh
    ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' control-token"
}

case "$REPO" in
    port-forward)
        port_forward
        ;;
    werkator)
        case "$COMMAND" in
            check-prerequisites)
                ensure_ssh
                check_prerequisites
                ;;
            instance-install)
                instance_install
                ;;
            instance-update)
                instance_update
                ;;
            instance-start)
                instance_start
                ;;
            repo-init)
                repo_init
                ;;
            repo-add)
                repo_add "${3:-}" "${4:-}"
                ;;
            control-token)
                control_token
                ;;
            install)
                die "'install' was the self-build prototype; use instance-install + repo-init (step 21 session D)"
                ;;
            build)
                die "'build' (the self-build) is retired; the instance builds pushes itself, or run '$WERKATOR_BIN build <branch>' on the host"
                ;;
            start)
                die "'start' is now 'instance-start' — commands name their role (builder vs built)"
                ;;
            *)
                echo "ERROR: unknown command: $COMMAND" >&2
                usage
                ;;
        esac
        ;;
    *)
        echo "ERROR: unknown repo selector: $REPO" >&2
        usage
        ;;
esac