fix(remote): upload before stopping the service, and verify the transfer

instance-update stopped the unit and only then started the upload, so a transfer
that dies mid-way leaves the host with no running Werkator and nothing to start
again. That is not theoretical: deploying to vm4006 on 2026-09-03 failed with
"scp: Connection closed" with the service already stopped.

The upload now happens before the stop, and each artifact is transferred to a
.part file whose sha256 is compared with the local one before it is moved into
place, retrying twice. A truncated archive would otherwise unpack into a broken
runtime, which is worse than the failed transfer it came from.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tools/remote

@@ -218,19 +218,54 @@ ensure_instance_artifacts() {
     [ "$SANDBOX" = "docker" ] || ensure_werkdock_binary
 }
 
-# Uploads and unpacks the instance artifacts. The previous runtime stays as
-# werkator.prev for one deployment as the rollback asset.
-deploy_instance() {
+# Uploads one file and verifies it arrived whole: a transfer that dies mid-way
+# (scp: Connection closed) otherwise leaves a truncated archive that unpacks into
+# a broken runtime. Retries twice, because a dropped WAN connection is not a reason
+# to abort a deployment.
+upload_verified() {
+    local src="$1" dest="$2"
+    local local_sha remote_sha attempt
+    local_sha="$(sha256sum "$src" | cut -d' ' -f1)"
+    remote_sha="$(ssh "$HOST" "sha256sum '$dest' 2>/dev/null | cut -d' ' -f1" || true)"
+    if [ "$local_sha" = "$remote_sha" ]; then
+        echo "    $(basename "$src"): already on the host, skipping"
+        return 0
+    fi
+    for attempt in 1 2 3; do
+        if scp -q "$src" "$HOST:$dest.part"; then
+            remote_sha="$(ssh "$HOST" "sha256sum '$dest.part' 2>/dev/null | cut -d' ' -f1" || true)"
+            if [ "$local_sha" = "$remote_sha" ]; then
+                ssh "$HOST" "mv '$dest.part' '$dest'"
+                return 0
+            fi
+            echo "    checksum mismatch after transfer $attempt of $(basename "$src")" >&2
+        else
+            echo "    transfer $attempt of $(basename "$src") failed" >&2
+        fi
+    done
+    ssh "$HOST" "rm -f '$dest.part'" || true
+    die "cannot upload $src to $HOST:$dest — three attempts failed"
+}
+
+# Uploads the instance artifacts, without touching the installed runtime: the
+# service keeps running until swap_instance_runtime replaces it, so a failed
+# transfer costs nothing but the transfer.
+upload_instance_artifacts() {
     if [ "$SANDBOX" = "docker" ]; then
         echo "==> Uploading runtime bundle"
     else
         echo "==> Uploading runtime bundle and werkdock binary"
     fi
     ssh "$HOST" "mkdir -p '$INSTALL_DIR/bin'"
-    scp -q "$RUNTIME_BUNDLE" "$HOST:$INSTALL_DIR/"
+    upload_verified "$RUNTIME_BUNDLE" "$INSTALL_DIR/$(basename "$RUNTIME_BUNDLE")"
     if [ "$SANDBOX" != "docker" ]; then
-        scp -q "$WERKDOCK_BINARY" "$HOST:$INSTALL_DIR/bin/werkdock.new"
+        upload_verified "$WERKDOCK_BINARY" "$INSTALL_DIR/bin/werkdock.new"
     fi
+}
+
+# Swaps in the uploaded artifacts. The previous runtime stays as werkator.prev
+# for one deployment as the rollback asset.
+swap_instance_runtime() {
     echo "==> Unpacking"
     ssh "$HOST" "set -e
         cd '$INSTALL_DIR'
@@ -246,7 +281,8 @@ instance_install() {
     ensure_ssh
     check_prerequisites
     ensure_instance_artifacts
-    deploy_instance
+    upload_instance_artifacts
+    swap_instance_runtime
     echo
     echo "==> Instance installed."
     echo "    Runtime:  $WERKATOR_BIN"
@@ -271,6 +307,9 @@ instance_update() {
     ensure_ssh
     ensure_instance_artifacts
     require_idle
+    # upload first, stop second: a transfer that fails must not leave the host
+    # without a running service (measured on vm4006, 2026-09-03)
+    upload_instance_artifacts
     local was_active=0
     if ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user is-active --quiet '$UNIT'"; then
         was_active=1
@@ -279,7 +318,7 @@ instance_update() {
         echo "==> Stopping $UNIT"
         ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user stop '$UNIT'"
     fi
-    deploy_instance
+    swap_instance_runtime
     if [ "$was_active" = "1" ]; then
         echo "==> Starting $UNIT"
         ssh "$HOST" "XDG_RUNTIME_DIR=/run/user/\$(id -u) systemctl --user start '$UNIT' && sleep 3 && systemctl --user is-active '$UNIT'"