#!/usr/bin/env bash # # Central control script for remote Werkator operations (same pattern as the # `remote` scripts in the other repos): the first argument is the repo selector, # the second the command. All connection and deployment values come from the # `.env` file in the repository root — never as command line parameters. # # Commands name their role (step 21 session D): `instance-*` manages the # BUILDER — the installed Werkator instance and its werkdock sandbox tool — # while `repo-*` acts on the BUILT, the repository the instance watches. # Werkator is never built on the target: the instance is installed from the # locally built runtime bundle (ADR 0006), and builds of the watched # repository are the running instance's job (or `bin/werkator build` on the # host — the werkator CLI, not this script). # # Usage: # tools/remote werkator check-prerequisites bwrap capability + disk/quota on the host # tools/remote werkator instance-install first-time: upload + unpack bundle and werkdock # tools/remote werkator instance-update redeploy bundle + werkdock, restart the service # tools/remote werkator instance-start server config, Apache proxy, systemd unit # tools/remote werkator repo-init clone the watched repo, init, rootfs, bwrap config # tools/remote werkator control-token # tools/remote port-forward start background tunnel to the Werkator UI # tools/remote port-forward stop # # Required in .env: # WERKATOR_REMOTE user@host to operate on, e.g. mih34-werkator@mih34.hostsharing.net # WERKATOR_PATH target directory on that host, e.g. /home/storage/mih34/users/werkator # # Required for `instance-start`: # WERKATOR_PORT the localhost port assigned by Hostsharing (eigener Serverdienst) # WERKATOR_DOMAIN the domain served by the managed Apache, e.g. ci.example.de # # Required for `port-forward`: # WERKATOR_LOCAL_PORT the local port the browser uses # Optional in .env: # WERKATOR_REPO_URL https clone URL of the watched repository # (default: https://github.com/mhoennig/werkator.git) # WERKATOR_MEMORY_MAX systemd MemoryMax for the unit, e.g. 1G (instance-start) # WERKATOR_TASKS_MAX systemd TasksMax for the unit, e.g. 512 (instance-start) # WERKATOR_ROOTFS rootfs archive path for repo-init # (default: /build/werkator-buildenv-trixie-java-go-node.tar.zst) # # Install layout on the host: # $WERKATOR_PATH/werkator/ the watched repository (clone) # $WERKATOR_PATH/.werkator/werkator/ the unpacked runtime bundle # $WERKATOR_PATH/.werkator/bin/ the werkdock binary # $WERKATOR_PATH/.werkator/*.tar.* uploaded bundle and rootfs archives # set -euo pipefail die() { echo "ERROR: $*" >&2; exit 1; } REPO="${1:-}" COMMAND="${2:-}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PREREQ_SCRIPT="$REPO_ROOT/tools/werkator-build-prerequisites.sh" RUNTIME_BUNDLE="$REPO_ROOT/build/distributions/werkator-runtime-linux-x64.tar.gz" WERKDOCK_BINARY="$REPO_ROOT/werkdock/dist/werkdock" PID_FILE="/tmp/werkator-port-forward-$(id -u).pid" LOG_FILE="/tmp/werkator-port-forward-$(id -u).log" usage() { awk 'NR > 2 && !/^#/ { exit } NR > 2 { sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}" exit 2 } require_env() { local missing=0 for name in "$@"; do if [ -z "${!name:-}" ]; then echo "ERROR: $name is not set — define it in $REPO_ROOT/.env" >&2 missing=1 fi done [ "$missing" -eq 0 ] || exit 1 } [ -n "$REPO" ] && [ -n "$COMMAND" ] || usage # Load the connection and deployment values; explicit environment wins, the # .env in the repository root fills the rest. set -a [ -f "$REPO_ROOT/.env" ] && source "$REPO_ROOT/.env" set +a require_env WERKATOR_REMOTE WERKATOR_PATH HOST="$WERKATOR_REMOTE" TARGET_DIR="$WERKATOR_PATH" ROOTFS="${WERKATOR_ROOTFS:-$REPO_ROOT/build/werkator-buildenv-trixie-java-go-node.tar.zst}" REPO_URL="${WERKATOR_REPO_URL:-https://github.com/mhoennig/werkator.git}" MACHINE_CONFIG="$TARGET_DIR/werkator/.git/werkator/.werkator.yml" WERKATOR_BIN="$TARGET_DIR/.werkator/werkator/bin/werkator" UNIT="werkator-werkator.service" ssh_present() { ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" true 2>/dev/null } ensure_ssh() { if ssh_present; then echo "==> SSH access to $HOST: ok" else echo "==> No key-based SSH access yet; running ssh-copy-id (password prompt expected)" ssh-copy-id "$HOST" ssh_present || die "SSH access still not working after ssh-copy-id" fi } # Run the prerequisites script remotely by piping it over stdin; TARGET_DIR and # ROOTFS_ARCHIVE are passed as arguments to `bash -s --`. check_prerequisites() { echo "==> Checking prerequisites on $HOST (target dir: $TARGET_DIR)" local rootfs_remote="$TARGET_DIR/.werkator/$(basename "$ROOTFS")" if ! ssh "$HOST" "WERKATOR_SSH_TARGET='$HOST' bash -s -- '$TARGET_DIR' '$rootfs_remote'" < "$PREREQ_SCRIPT"; then die "prerequisites failed on $HOST — install aborted" fi } # The instance artifacts are built locally (ADR 0006): the runtime bundle via # Gradle, the werkdock binary via the Go toolchain. Both are rebuilt when # missing, never on the target. ensure_instance_artifacts() { if [ ! -f "$RUNTIME_BUNDLE" ]; then echo "==> Runtime bundle not found; building it locally (./gradlew runtimeBundle)" (cd "$REPO_ROOT" && ./gradlew runtimeBundle --console=plain -q) fi [ -f "$RUNTIME_BUNDLE" ] || die "runtime bundle missing: $RUNTIME_BUNDLE" if [ ! -f "$WERKDOCK_BINARY" ]; then echo "==> werkdock binary not found; building it locally (go build)" (cd "$REPO_ROOT/werkdock" && CGO_ENABLED=0 go build -o dist/werkdock .) fi [ -f "$WERKDOCK_BINARY" ] || die "werkdock binary missing: $WERKDOCK_BINARY" } # Uploads and unpacks the instance artifacts. The previous runtime stays as # werkator.prev for one deployment as the rollback asset. deploy_instance() { echo "==> Uploading runtime bundle and werkdock binary" ssh "$HOST" "mkdir -p '$TARGET_DIR/.werkator/bin'" scp -q "$RUNTIME_BUNDLE" "$HOST:$TARGET_DIR/.werkator/" scp -q "$WERKDOCK_BINARY" "$HOST:$TARGET_DIR/.werkator/bin/werkdock.new" echo "==> Unpacking" ssh "$HOST" "set -e cd '$TARGET_DIR/.werkator' mv bin/werkdock.new bin/werkdock && chmod 755 bin/werkdock rm -rf werkator.prev [ ! -d werkator ] || mv werkator werkator.prev tar xzf '$(basename "$RUNTIME_BUNDLE")' './werkator/bin/werkator' --version './bin/werkdock' version" } instance_install() { ensure_ssh check_prerequisites ensure_instance_artifacts deploy_instance echo echo "==> Instance installed." echo " Runtime: $WERKATOR_BIN" echo " werkdock: $TARGET_DIR/.werkator/bin/werkdock" echo " Next: tools/remote werkator repo-init, then instance-start" } # Refuse to swap the runtime under a running build; FORCE=1 overrides. require_idle() { local port port="$(ssh "$HOST" "awk '/^server:/{f=1;next} f && /^ port:/{print \$2; exit}' '$MACHINE_CONFIG' 2>/dev/null" || true)" [ -n "$port" ] || return 0 local current current="$(ssh "$HOST" "curl -s --max-time 5 http://127.0.0.1:$port/api/builds/current" || true)" if [ -n "$current" ] && [ "$current" != "[]" ]; then [ "${FORCE:-}" = "1" ] || die "a build is running on $HOST — retry when idle, or FORCE=1 to override" echo "==> WARNING: deploying although a build is running (FORCE=1)" fi } instance_update() { ensure_ssh ensure_instance_artifacts require_idle local was_active=0 if ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user is-active --quiet '$UNIT'"; then was_active=1 fi if [ "$was_active" = "1" ]; then echo "==> Stopping $UNIT" ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user stop '$UNIT'" fi deploy_instance if [ "$was_active" = "1" ]; then echo "==> Starting $UNIT" ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user start '$UNIT' && sleep 3 && systemctl --user is-active '$UNIT'" else echo "==> Service was not running; not started (use instance-start for the first start)" fi echo "==> Instance updated." } # Sets up the WATCHED repository: an anonymous https clone (a private origin # gets its credentials via git.account/git.token in the machine config that # `werkator init` creates), the werkator init, the rootfs archive for the # sandbox builds, and the machine-local bwrap configuration. repo_init() { ensure_ssh [ -f "$ROOTFS" ] || die "rootfs archive missing: $ROOTFS — build it with tools/build-bwrap-rootfs.sh or set WERKATOR_ROOTFS" ssh "$HOST" "test -x '$WERKATOR_BIN'" || die "no instance on $HOST — run instance-install first" echo "==> Cloning the watched repository" if ssh "$HOST" "test -d '$TARGET_DIR/werkator/.git'"; then echo " (already cloned, skipping)" else ssh "$HOST" "git clone '$REPO_URL' '$TARGET_DIR/werkator'" fi echo "==> Running werkator init" ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' init" echo "==> Uploading the rootfs archive (skipped when unchanged)" local rootfs_remote="$TARGET_DIR/.werkator/$(basename "$ROOTFS")" local local_sha remote_sha local_sha="$(sha256sum "$ROOTFS" | cut -d' ' -f1)" remote_sha="$(ssh "$HOST" "sha256sum '$rootfs_remote' 2>/dev/null | cut -d' ' -f1" || true)" if [ "$local_sha" = "$remote_sha" ]; then echo " (already on the host, skipping)" else scp -q "$ROOTFS" "$HOST:$rootfs_remote" remote_sha="$(ssh "$HOST" "sha256sum '$rootfs_remote' | cut -d' ' -f1")" [ "$local_sha" = "$remote_sha" ] || die "rootfs upload checksum mismatch" fi echo "==> Writing the machine-local bwrap configuration" # NOTE: the guard must match the block's real indentation — a mismatch here # once appended the block on every run. if ssh "$HOST" "grep -q '^ bwrap:' '$MACHINE_CONFIG' 2>/dev/null"; then echo " (bwrap block present, skipping)" else ssh "$HOST" "cat >> '$MACHINE_CONFIG' <<'CFG' # Build in the bubblewrap sandbox instead of natively (ADR 0008), executed by # the werkdock CLI (step 21 session C). All three keys are pinned: read from # this machine config even if a branch sets its own values. builds: default: bwrap: enabled: true rootfs: $rootfs_remote werkdock: $TARGET_DIR/.werkator/bin/werkdock CFG" fi echo "==> Verifying the effective configuration" ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' config:print 2>/dev/null | grep -A4 'bwrap:' | head -5" echo echo "==> Repository ready." echo " Repo: $TARGET_DIR/werkator" echo " Next: fill git.account/git.token in $MACHINE_CONFIG if the origin is private," echo " then tools/remote werkator instance-start" } # Start the server as a systemd user unit behind the managed Apache. # WERKATOR_MEMORY_MAX / WERKATOR_TASKS_MAX (optional) are written into the # machine config so `init --systemd` bakes them into the unit. instance_start() { ensure_ssh require_env WERKATOR_PORT WERKATOR_DOMAIN local htaccess="$TARGET_DIR/doms/$WERKATOR_DOMAIN/subs/www/.htaccess" echo "==> Writing server settings to the machine config" if ssh "$HOST" "grep -q '^server:' '$MACHINE_CONFIG' 2>/dev/null"; then # re-run: update port and publicBaseUrl in place (systemd limits stay as written) ssh "$HOST" "sed -i 's/^ port: .*/ port: $WERKATOR_PORT/; s|^ publicBaseUrl: .*| publicBaseUrl: \"https://$WERKATOR_DOMAIN/\"|' '$MACHINE_CONFIG'" else ssh "$HOST" "cat >> '$MACHINE_CONFIG' <<'CFG' # Web access: the managed Apache terminates TLS and proxies to the localhost # port assigned by Hostsharing (eigener Serverdienst); TLS is the domain's # Let's Encrypt certificate, so Werkator itself stays on 127.0.0.1. server: port: $WERKATOR_PORT bindAddress: 127.0.0.1 publicBaseUrl: \"https://$WERKATOR_DOMAIN/\" nginx: enabled: false systemd: memoryMax: \"${WERKATOR_MEMORY_MAX:-}\" tasksMax: \"${WERKATOR_TASKS_MAX:-}\" CFG" fi echo "==> Writing the Apache reverse proxy to $htaccess" ssh "$HOST" "mkdir -p '$TARGET_DIR/doms/$WERKATOR_DOMAIN/subs/www' && cat > '$htaccess' <<'HT' DirectoryIndex disabled RewriteEngine On RewriteBase / RewriteRule .* http://127.0.0.1:$WERKATOR_PORT%{REQUEST_URI} [proxy] HT" echo "==> Generating the systemd user unit (init --systemd)" ssh "$HOST" "cd '$TARGET_DIR/werkator' && '$WERKATOR_BIN' init --systemd" echo "==> Linking the units into ~/.config/systemd/user and enabling the service" ssh "$HOST" "mkdir -p ~/.config/systemd/user && \ ln -sf '$TARGET_DIR/werkator/.git/werkator/$UNIT' ~/.config/systemd/user/ && \ ln -sf '$TARGET_DIR/werkator/.git/werkator/werkator-docker-prune.service' ~/.config/systemd/user/ && \ ln -sf '$TARGET_DIR/werkator/.git/werkator/werkator-docker-prune.timer' ~/.config/systemd/user/ && \ XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user daemon-reload && \ XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user restart '$UNIT' && \ XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user status '$UNIT' --no-pager -l | head -12" echo echo "==> Server started. Verify: https://$WERKATOR_DOMAIN/" echo " Logs: ssh $HOST -- systemctl --user status '$UNIT'" } # Background SSH tunnel to the Werkator server, so the browser reaches the UI # at http://localhost: without keeping a terminal busy. # `start` runs ssh -N -L detached with a pid file; `stop` kills it. port_forward() { require_env WERKATOR_LOCAL_PORT local remote_port remote_port="$(ssh "$HOST" "awk '/^server:/{f=1;next} f && /^ port:/{print \$2; exit}' '$MACHINE_CONFIG'")" [ -n "$remote_port" ] || die "no server.port in the machine config — run 'tools/remote werkator instance-start' first" case "$COMMAND" in start) if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then echo "==> Port-forward already running (pid $(cat "$PID_FILE")) — http://localhost:$WERKATOR_LOCAL_PORT" exit 0 fi nohup ssh -N -L "$WERKATOR_LOCAL_PORT:127.0.0.1:$remote_port" "$HOST" \ >"$LOG_FILE" 2>&1 & echo $! > "$PID_FILE" sleep 1 if kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then echo "==> Forwarding http://localhost:$WERKATOR_LOCAL_PORT -> $HOST:127.0.0.1:$remote_port (pid $(cat "$PID_FILE"))" else echo "ERROR: port-forward failed to start — see $LOG_FILE" >&2 rm -f "$PID_FILE" exit 1 fi ;; stop) if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then kill "$(cat "$PID_FILE")" rm -f "$PID_FILE" echo "==> Port-forward stopped" else rm -f "$PID_FILE" echo "==> Port-forward is not running" fi ;; *) echo "ERROR: unknown port-forward command: $COMMAND (use start or stop)" >&2 exit 2 ;; esac } # Print the control token guarding the mutating build endpoints. If the server # has not created it yet (it does so on first use), generate one in place — the # server reads the file lazily, so a pre-created token is equivalent. control_token() { ensure_ssh local token_file="$TARGET_DIR/werkator/.git/werkator/control-token" ssh "$HOST" "if [ -f '$token_file' ]; then cat '$token_file'; else \ umask 077 && head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \\n' > '$token_file' && cat '$token_file'; fi" } case "$REPO" in port-forward) port_forward ;; werkator) case "$COMMAND" in check-prerequisites) ensure_ssh check_prerequisites ;; instance-install) instance_install ;; instance-update) instance_update ;; instance-start) instance_start ;; repo-init) repo_init ;; control-token) control_token ;; install) die "'install' was the self-build prototype; use instance-install + repo-init (step 21 session D)" ;; build) die "'build' (the self-build) is retired; the instance builds pushes itself, or run '$WERKATOR_BIN build ' on the host" ;; start) die "'start' is now 'instance-start' — commands name their role (builder vs built)" ;; *) echo "ERROR: unknown command: $COMMAND" >&2 usage ;; esac ;; *) echo "ERROR: unknown repo selector: $REPO" >&2 usage ;; esac