diff --git a/tools/remote b/tools/remote index 1fd1ac0..e99d899 100755 --- a/tools/remote +++ b/tools/remote @@ -218,19 +218,54 @@ ensure_instance_artifacts() { [ "$SANDBOX" = "docker" ] || ensure_werkdock_binary } -# Uploads and unpacks the instance artifacts. The previous runtime stays as -# werkator.prev for one deployment as the rollback asset. -deploy_instance() { +# Uploads one file and verifies it arrived whole: a transfer that dies mid-way +# (scp: Connection closed) otherwise leaves a truncated archive that unpacks into +# a broken runtime. Retries twice, because a dropped WAN connection is not a reason +# to abort a deployment. +upload_verified() { + local src="$1" dest="$2" + local local_sha remote_sha attempt + local_sha="$(sha256sum "$src" | cut -d' ' -f1)" + remote_sha="$(ssh "$HOST" "sha256sum '$dest' 2>/dev/null | cut -d' ' -f1" || true)" + if [ "$local_sha" = "$remote_sha" ]; then + echo " $(basename "$src"): already on the host, skipping" + return 0 + fi + for attempt in 1 2 3; do + if scp -q "$src" "$HOST:$dest.part"; then + remote_sha="$(ssh "$HOST" "sha256sum '$dest.part' 2>/dev/null | cut -d' ' -f1" || true)" + if [ "$local_sha" = "$remote_sha" ]; then + ssh "$HOST" "mv '$dest.part' '$dest'" + return 0 + fi + echo " checksum mismatch after transfer $attempt of $(basename "$src")" >&2 + else + echo " transfer $attempt of $(basename "$src") failed" >&2 + fi + done + ssh "$HOST" "rm -f '$dest.part'" || true + die "cannot upload $src to $HOST:$dest — three attempts failed" +} + +# Uploads the instance artifacts, without touching the installed runtime: the +# service keeps running until swap_instance_runtime replaces it, so a failed +# transfer costs nothing but the transfer. +upload_instance_artifacts() { if [ "$SANDBOX" = "docker" ]; then echo "==> Uploading runtime bundle" else echo "==> Uploading runtime bundle and werkdock binary" fi ssh "$HOST" "mkdir -p '$INSTALL_DIR/bin'" - scp -q "$RUNTIME_BUNDLE" "$HOST:$INSTALL_DIR/" + upload_verified "$RUNTIME_BUNDLE" "$INSTALL_DIR/$(basename "$RUNTIME_BUNDLE")" if [ "$SANDBOX" != "docker" ]; then - scp -q "$WERKDOCK_BINARY" "$HOST:$INSTALL_DIR/bin/werkdock.new" + upload_verified "$WERKDOCK_BINARY" "$INSTALL_DIR/bin/werkdock.new" fi +} + +# Swaps in the uploaded artifacts. The previous runtime stays as werkator.prev +# for one deployment as the rollback asset. +swap_instance_runtime() { echo "==> Unpacking" ssh "$HOST" "set -e cd '$INSTALL_DIR' @@ -246,7 +281,8 @@ instance_install() { ensure_ssh check_prerequisites ensure_instance_artifacts - deploy_instance + upload_instance_artifacts + swap_instance_runtime echo echo "==> Instance installed." echo " Runtime: $WERKATOR_BIN" @@ -271,6 +307,9 @@ instance_update() { ensure_ssh ensure_instance_artifacts require_idle + # upload first, stop second: a transfer that fails must not leave the host + # without a running service (measured on vm4006, 2026-09-03) + upload_instance_artifacts local was_active=0 if ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user is-active --quiet '$UNIT'"; then was_active=1 @@ -279,7 +318,7 @@ instance_update() { echo "==> Stopping $UNIT" ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user stop '$UNIT'" fi - deploy_instance + swap_instance_runtime if [ "$was_active" = "1" ]; then echo "==> Starting $UNIT" ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user start '$UNIT' && sleep 3 && systemctl --user is-active '$UNIT'"