werkdock/internal/engine/bwrap.go

package engine

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

// Bwrap runs a RunSpec through the bwrap CLI — filesystem isolation
// only; network, uid mapping target, /proc, /dev, and /tmp come from
// the host by contract.
//
// The invocation is a port of Werkator's BwrapBuildRunner, including
// the parts hardened on a real Hostsharing webspace: bind mountpoints
// are pre-created inside the rootfs (a plain host directory), because
// bwrap cannot mkdir them against the read-only root bind.
type Bwrap struct {
	// Path of the bwrap binary; empty means "bwrap" via PATH.
	Path string
	// Stdio of the sandboxed command; nil fields default to the
	// werkdock process's own.
	Stdout io.Writer
	Stderr io.Writer
	Stdin  io.Reader
}

// DefaultPATH is the PATH inside the sandbox; the environment is
// cleared (docker semantics), so a sane default must be set explicitly.
const DefaultPATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

// Argv assembles the full bwrap command line for spec.
//
// The command is pid 1 of the sandbox (--as-pid-1), as in docker: a
// signal sent to it from outside reaches it if it handles the signal,
// which is what `stop` relies on — bwrap's own init would swallow
// SIGTERM. Zombies of orphaned children are the command's business,
// as they are in a container without --init. bwrap reports the
// command's host pid on fd 3 (--info-fd), which Start reads.
//
// Mount order: the rootfs first; then /proc, /dev, and the tmpfs
// mounts for /tmp and /root, BEFORE the user binds, so a bind whose
// destination lies below them lands inside instead of being shadowed;
// then the user binds in the given order.
func (b *Bwrap) Argv(spec RunSpec) ([]string, error) {
	if spec.RootFS == "" {
		return nil, errors.New("rootfs must be set")
	}
	if !filepath.IsAbs(spec.RootFS) {
		return nil, fmt.Errorf("rootfs must be an absolute path: %s", spec.RootFS)
	}
	if len(spec.Command) == 0 {
		return nil, errors.New("no command specified")
	}
	bin := b.Path
	if bin == "" {
		bin = "bwrap"
	}
	args := []string{
		bin,
		"--unshare-user",
		"--unshare-pid",
		"--as-pid-1",
		"--die-with-parent",
		"--info-fd", "3",
		"--uid", strconv.Itoa(spec.UID),
		"--gid", strconv.Itoa(spec.GID),
		"--ro-bind", spec.RootFS, "/",
		"--proc", "/proc",
		"--dev", "/dev",
		"--tmpfs", "/tmp",
		"--tmpfs", "/root",
		"--tmpfs", "/run",
	}
	for _, m := range spec.Mounts {
		if !filepath.IsAbs(m.Dest) {
			return nil, fmt.Errorf("mount destination must be an absolute path: %s", m.Dest)
		}
		switch m.Mode {
		case MountBind:
			args = append(args, "--bind", m.Source, m.Dest)
		case MountRoBind:
			args = append(args, "--ro-bind", m.Source, m.Dest)
		case MountTmpfs:
			args = append(args, "--tmpfs", m.Dest)
		default:
			return nil, fmt.Errorf("unknown mount mode %d for %s", m.Mode, m.Dest)
		}
	}
	home := spec.Home
	if home == "" {
		home = "/root"
	}
	args = append(args, "--clearenv")
	for _, e := range environment(home, spec.Env) {
		args = append(args, "--setenv", e.Key, e.Value)
	}
	workdir := spec.Workdir
	if workdir == "" {
		workdir = "/"
	}
	if WorkdirWritable(spec) {
		// docker creates a missing workdir; bwrap can only do so where
		// the sandbox is writable (--dir is mkdir -p, a no-op if present).
		args = append(args, "--dir", workdir)
	}
	args = append(args, "--chdir", workdir, "--")
	args = append(args, spec.Command...)
	return args, nil
}

// environment is HOME and PATH with werkdock's defaults, then the
// spec's variables in order; a later value for the same key replaces
// the earlier one in place, so each key is set exactly once.
func environment(home string, env []EnvVar) []EnvVar {
	all := []EnvVar{{Key: "HOME", Value: home}, {Key: "PATH", Value: DefaultPATH}}
	for _, e := range env {
		replaced := false
		for i := range all {
			if all[i].Key == e.Key {
				all[i].Value, replaced = e.Value, true
				break
			}
		}
		if !replaced {
			all = append(all, e)
		}
	}
	return all
}

// implicitMounts are the tmpfs mounts every sandbox carries, in front of
// the user mounts.
// /run holds runtime state by FHS definition (pid files, sockets, locks)
// and is /var/run's target in practically every image; services write
// there without declaring a VOLUME (RFC 0005).
var implicitMounts = []Mount{{Mode: MountTmpfs, Dest: "/tmp"}, {Mode: MountTmpfs, Dest: "/root"}, {Mode: MountTmpfs, Dest: "/run"}}

// coveringMount returns the mount whose destination is the longest
// prefix of path — the one the path is actually served from, since
// later and deeper mounts shadow earlier ones. ok is false when the
// path lies in the read-only rootfs itself.
func coveringMount(spec RunSpec, path string) (m Mount, ok bool) {
	best := -1
	for _, cand := range append(append([]Mount{}, implicitMounts...), spec.Mounts...) {
		dest := filepath.Clean(cand.Dest)
		if path == dest || strings.HasPrefix(path, strings.TrimSuffix(dest, "/")+"/") {
			if len(dest) >= best {
				best, m, ok = len(dest), cand, true
			}
		}
	}
	return m, ok
}

// WorkdirWritable reports whether spec.Workdir lies in a writable area
// of the sandbox: a tmpfs or a read-write bind.
func WorkdirWritable(spec RunSpec) bool {
	if spec.Workdir == "" {
		return false
	}
	m, ok := coveringMount(spec, filepath.Clean(spec.Workdir))
	return ok && m.Mode != MountRoBind
}

// CheckWorkdir verifies that a workdir outside the writable areas exists
// where the sandbox will see it — in the rootfs or the read-only bind
// covering it — so a typo fails with a clear message instead of bwrap's
// "Can't chdir".
func CheckWorkdir(spec RunSpec) error {
	if spec.Workdir == "" || WorkdirWritable(spec) {
		return nil
	}
	workdir := filepath.Clean(spec.Workdir)
	hostPath := ""
	if m, ok := coveringMount(spec, workdir); ok {
		hostPath = filepath.Join(m.Source, strings.TrimPrefix(workdir, filepath.Clean(m.Dest)))
	} else {
		var err error
		if hostPath, err = rootfsPath(spec.RootFS, workdir); err != nil {
			return err
		}
	}
	if fi, err := os.Stat(hostPath); err != nil || !fi.IsDir() {
		return fmt.Errorf("working directory %s does not exist in the image and lies outside the sandbox's writable areas (tmpfs and read-write binds)", spec.Workdir)
	}
	return nil
}

// EnsureMountpoints pre-creates the mountpoints of spec inside the
// rootfs directory. bwrap creates mountpoints against the sandbox view,
// which is the read-only rootfs bind — every destination missing from
// the rootfs fails with "Read-only file system". The rootfs directory
// itself is a plain host directory, so the mountpoints are created
// there; bwrap then finds them and has nothing left to mkdir.
//
// Anything that already exists in the rootfs is left alone (e.g.
// /etc/resolv.conf is a file many rootfs archives ship). A bind whose
// source is a regular file gets a file mountpoint, not a directory.
func EnsureMountpoints(spec RunSpec) error {
	for _, dest := range []string{"/proc", "/dev", "/tmp", "/root"} {
		if err := ensureDir(spec.RootFS, dest); err != nil {
			return err
		}
	}
	for _, m := range spec.Mounts {
		target, err := rootfsPath(spec.RootFS, m.Dest)
		if err != nil {
			return err
		}
		if _, err := os.Lstat(target); err == nil {
			continue
		}
		if m.Mode == MountTmpfs {
			if err := os.MkdirAll(target, 0o755); err != nil {
				return err
			}
			continue
		}
		src, err := os.Stat(m.Source)
		if err != nil {
			return fmt.Errorf("bind source %s: %w", m.Source, err)
		}
		if src.Mode().IsRegular() {
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return err
			}
			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o644)
			if err != nil {
				return err
			}
			if err := f.Close(); err != nil {
				return err
			}
			continue
		}
		if err := os.MkdirAll(target, 0o755); err != nil {
			return err
		}
	}
	return nil
}

func ensureDir(rootfs, dest string) error {
	target, err := rootfsPath(rootfs, dest)
	if err != nil {
		return err
	}
	if _, statErr := os.Lstat(target); statErr == nil {
		return nil
	}
	return os.MkdirAll(target, 0o755)
}

// rootfsPath resolves dest inside rootfs and refuses destinations that
// escape it — werkdock assembles mounts from user input, so this must
// hold even for hostile paths.
func rootfsPath(rootfs, dest string) (string, error) {
	root := filepath.Clean(rootfs)
	target := filepath.Join(root, dest)
	prefix := root
	if !strings.HasSuffix(prefix, string(filepath.Separator)) {
		prefix += string(filepath.Separator)
	}
	if target != root && !strings.HasPrefix(target, prefix) {
		return "", fmt.Errorf("bind destination escapes the rootfs: %s", dest)
	}
	return target, nil
}

// process is a running bwrap invocation.
type process struct {
	cmd      *exec.Cmd
	childPID int
}

func (p *process) ChildPID() int { return p.childPID }

func (p *process) Signal(sig os.Signal) error {
	s, ok := sig.(syscall.Signal)
	if !ok {
		return fmt.Errorf("unsupported signal %v", sig)
	}
	return syscall.Kill(p.childPID, s)
}

// Wait returns the command's exit code; bwrap propagates the child's
// code (128+signal for a signalled child), so it passes through.
func (p *process) Wait() (int, error) {
	err := p.cmd.Wait()
	if err == nil {
		return 0, nil
	}
	var exit *exec.ExitError
	if errors.As(err, &exit) {
		if exit.ExitCode() >= 0 {
			return exit.ExitCode(), nil
		}
		if ws, ok := exit.Sys().(syscall.WaitStatus); ok && ws.Signaled() {
			return 128 + int(ws.Signal()), nil
		}
	}
	return 0, err
}

// Start launches spec and returns once bwrap has reported the
// sandboxed command's host pid. bwrap runs in a process group of its
// own, so a terminal's Ctrl-C reaches werkdock, which forwards it as
// the supervisor sees fit, rather than killing bwrap outright.
func (b *Bwrap) Start(spec RunSpec) (Process, error) {
	argv, err := b.Argv(spec)
	if err != nil {
		return nil, err
	}
	if err := EnsureMountpoints(spec); err != nil {
		return nil, err
	}
	if err := CheckWorkdir(spec); err != nil {
		return nil, err
	}
	infoR, infoW, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	defer infoR.Close()
	cmd := exec.Command(argv[0], argv[1:]...)
	cmd.Stdout = b.Stdout
	if cmd.Stdout == nil {
		cmd.Stdout = os.Stdout
	}
	cmd.Stderr = b.Stderr
	if cmd.Stderr == nil {
		cmd.Stderr = os.Stderr
	}
	cmd.Stdin = b.Stdin
	cmd.ExtraFiles = []*os.File{infoW} // fd 3, see Argv
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
	if err := cmd.Start(); err != nil {
		infoW.Close()
		return nil, err
	}
	infoW.Close()
	var info struct {
		ChildPID int `json:"child-pid"`
	}
	if err := json.NewDecoder(infoR).Decode(&info); err != nil || info.ChildPID <= 0 {
		// bwrap failed before the sandbox existed; its message went
		// to stderr, its exit code is the only thing left to report.
		code, werr := (&process{cmd: cmd}).Wait()
		if werr != nil {
			return nil, werr
		}
		return nil, fmt.Errorf("bwrap exited with code %d before starting the command", code)
	}
	return &process{cmd: cmd, childPID: info.ChildPID}, nil
}

// Run executes spec and returns the command's exit code; bwrap
// propagates the child's code, so the caller can pass it through.
func (b *Bwrap) Run(spec RunSpec) (int, error) {
	p, err := b.Start(spec)
	if err != nil {
		return 0, err
	}
	return p.Wait()
}