Machine descriptors

Taskfile.yml

@@ -258,7 +258,8 @@ tasks:
       # a rebuilt QEMU with a device removed would otherwise be met with a cached pass.
       # -v because what was rewritten for the TCG binary, and what a host without
       # /dev/vhost-vsock left unchecked, is the part a reader has to see.
-      - SPIN_MACHINE_OUTPUT={{.OUTPUT_ABS}} go test ./machine -run TestQEMUAcceptsEveryArgument -count=1 -v
+      # And the chain over descriptors: what QEMU does with it, not only whether it parses.
+      - SPIN_MACHINE_OUTPUT={{.OUTPUT_ABS}} go test ./machine -run 'TestQEMUAcceptsEveryArgument|TestAChainOverDescriptors' -count=1 -v
 
   fingerprint:
     desc: >-

machine/machine.go

@@ -26,6 +26,7 @@ package machine
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -115,8 +116,17 @@ const memGrowthID = "mem.growth"
 
 // Disk is one virtio-blk device.
 type Disk struct {
-	// Path to the image file.
+	// Path to the image file. QEMU opens it, and every backing file its header
+	// names, by path. Set Path or Chain, not both.
 	Path string
+	// Chain is the image and every image under it, top first, each handed to
+	// QEMU as a descriptor set (Spec.FDSets): QEMU opens no image by path, and
+	// follows no backing file a header names — each image's backing is the next
+	// one here, and the last has none. So a QEMU that is not allowed to see the
+	// caller's filesystem at all still reads the whole chain, and reads nothing
+	// a header could point it at. Format and Cache do not apply: each image
+	// carries its own format, and the chain is cached as QEMU does by default.
+	Chain []Image
 	// Format as QEMU names it: qcow2 or raw. Required — it is not guessed
 	// from the file name, because a wrong guess is a guest that boots and finds
 	// a disk full of nothing, and because letting QEMU probe the format of a
@@ -161,12 +171,96 @@ type Disk struct {
 	DirectOverBacking bool
 }
 
+// chainArgs is disk i as -blockdev nodes over descriptor sets, the base first: each format
+// node is backed by the node below it, and the last by nothing, so QEMU never reads a backing
+// name from a header. The top node is blk<i>, as a -drive's id would be, and the rest
+// blk<i>-<depth>.
+//
+// The same options as the path form: aio=io_uring, discard=unmap, and the lock only where
+// asked for. Every image but the top is read-only, as a backing file is.
+//
+// JSON, not key=value: "no backing" is JSON null, which key=value cannot say — backing=null
+// names a node called null — and an opaque path needs no comma escaping.
+func (d Disk) chainArgs(i int) ([]string, error) {
+	var args []string
+	for j := len(d.Chain) - 1; j >= 0; j-- {
+		img := d.Chain[j]
+		node := fmt.Sprintf("blk%d", i)
+		if j > 0 {
+			node = fmt.Sprintf("blk%d-%d", i, j)
+		}
+		readOnly := d.Readonly || j > 0
+		file := map[string]any{
+			"driver": "file", "node-name": node + "-file",
+			"filename": fmt.Sprintf("/dev/fdset/%d", img.FDSet),
+			"aio":      "io_uring", "discard": "unmap", "read-only": readOnly,
+		}
+		if j == 0 && d.Locking {
+			file["locking"] = "on"
+		}
+		if d.DirectOverBacking {
+			file["cache"] = map[string]any{"direct": j == 0}
+		}
+		format := map[string]any{
+			"driver": img.Format, "node-name": node, "file": node + "-file", "read-only": readOnly,
+		}
+		switch {
+		case j < len(d.Chain)-1:
+			format["backing"] = fmt.Sprintf("blk%d-%d", i, j+1)
+		case img.Format != "raw":
+			// Or QEMU opens whatever the header names.
+			format["backing"] = nil
+		}
+		for _, n := range []map[string]any{file, format} {
+			b, err := json.Marshal(n)
+			if err != nil {
+				return nil, err
+			}
+			args = append(args, "-blockdev", string(b))
+		}
+	}
+	return args, nil
+}
+
+// Image is one image of a Disk's chain.
+type Image struct {
+	// FDSet is the id of the descriptor set holding the image (Spec.FDSets): one
+	// descriptor, opened read-write for the top of a writable disk and read-only
+	// for everything else. Sealing the top under a new overlay (blockdev-snapshot)
+	// needs nothing more — measured on QEMU 11.1.1, which keeps the old top's
+	// descriptor rather than reopening it read-only.
+	FDSet int
+	// Format as QEMU names it, qcow2 or raw, for the same reasons as Disk.Format.
+	// Only the last image of a chain may be raw: raw has no backing.
+	Format string
+}
+
+// FDSet is descriptors QEMU is handed open rather than a path to open: a path
+// of the form /dev/fdset/<ID> names the set wherever QEMU takes a file name.
+type FDSet struct {
+	ID  int
+	FDs []FD
+}
+
+// FD is one descriptor in the QEMU process, so 3 or above.
+type FD struct {
+	Num int
+	// Opaque is what QEMU reports for the descriptor in query-fdsets, and the
+	// only way back from /dev/fdset/<ID> to what the caller opened: the path, in
+	// practice, so that a caller asking which image a device has open gets an
+	// answer it can compare.
+	Opaque string
+}
+
 // NIC is one virtio-net device, backed by a TAP file descriptor the caller has
 // already opened and passed to the QEMU process.
 type NIC struct {
 	// TapFD is the descriptor number in the QEMU process, so 3 or above.
 	TapFD int
-	MAC   string
+	// VhostFD is /dev/vhost-net already open, handed to QEMU as a descriptor, for a
+	// QEMU that may open no device node. Zero has QEMU open the node itself.
+	VhostFD int
+	MAC     string
 
 	// MTU is the largest frame the guest may send, announced through
 	// VIRTIO_NET_F_MTU. Zero leaves the guest at its own default, 1500.
@@ -324,10 +418,35 @@ type Spec struct {
 	// has no serial port for a caller to drive and no network it is required to
 	// have.
 	VsockCID int
+	// VsockFD is /dev/vhost-vsock already open, handed to QEMU as a descriptor, for a
+	// QEMU that may open no device node: one whose root has no /dev. Zero has QEMU open
+	// the node itself.
+	VsockFD int
+
+	// KVMFDSet is /dev/kvm as a descriptor set, for the same QEMU. Only under KVM, which
+	// is the accelerator that opens a device. The shape still says accel=kvm: where the
+	// accelerator's descriptor came from is not the machine a template is loaded into.
+	KVMFDSet int
 
 	// QMPSocket is a Unix socket path QEMU listens on for QMP. Required to do
 	// anything to a running machine, including shutting it down.
 	QMPSocket string
+	// QMPFD is QMPSocket already listening, handed to QEMU as a descriptor, for a
+	// QEMU that may not create a socket where the caller would connect to it. Set
+	// one or the other; QMPFD2 is the same for QMPSocket2.
+	QMPFD  int
+	QMPFD2 int
+
+	// FDSets are the descriptor sets the command line names: a Disk's Chain,
+	// SerialFDSet.
+	FDSets []FDSet
+
+	// SerialFDSet is the console as a descriptor set QEMU writes to, instead of
+	// Serial, for a QEMU that may open nothing by path. It is opened append-only:
+	// QEMU otherwise truncates what it opens, and the usual console descriptor is
+	// a FIFO, which cannot be truncated — "-serial file:/dev/fdset/<ID>" fails
+	// with EINVAL on one (measured, QEMU 11.1.1).
+	SerialFDSet int
 
 	// QMPSocket2 is a second monitor, on its own socket, for a second thing that drives
 	// this machine.
@@ -505,6 +624,16 @@ func (s Spec) Shape() Shape {
 	}
 }
 
+// machineArg is the shape's machine string as -machine takes it. With KVMFDSet the
+// accelerator moves to its own -accel, which carries the descriptor: QEMU refuses the
+// two together — "The -accel and "-machine accel=" options are incompatible" (11.1.1).
+func (s Spec) machineArg(shape Shape) string {
+	if s.KVMFDSet == 0 {
+		return shape.Machine
+	}
+	return strings.Replace(shape.Machine, ",accel=kvm", "", 1)
+}
+
 // derivedFromHost reports whether a CPU model takes its feature set from the
 // silicon it runs on, which is what makes a template built with it unusable on
 // another machine — and what migratable=on applies to.
@@ -600,13 +729,81 @@ func (s Spec) Validate() error {
 	case s.HotplugPorts < 0 || s.HotplugPorts > MaxHotplugPorts:
 		return fmt.Errorf("%d root ports for devices arriving later, and the slot range holds %d",
 			s.HotplugPorts, MaxHotplugPorts)
+	case s.QMPSocket != "" && s.QMPFD != 0, s.QMPSocket2 != "" && s.QMPFD2 != 0:
+		return fmt.Errorf("a monitor is given both a socket path and a descriptor")
+	case s.QMPFD != 0 && s.QMPFD < 3, s.QMPFD2 != 0 && s.QMPFD2 < 3:
+		return fmt.Errorf("a monitor descriptor below 3 is stdin, stdout or stderr")
+	}
+	sets, err := s.fdSets()
+	if err != nil {
+		return err
+	}
+	switch {
+	case s.SerialFDSet != 0 && s.Serial != "":
+		return fmt.Errorf("a console given both as a chardev and as a descriptor set")
+	case s.SerialFDSet != 0 && !sets[s.SerialFDSet]:
+		return fmt.Errorf("the console is descriptor set %d, which Spec.FDSets does not have", s.SerialFDSet)
+	case s.KVMFDSet != 0 && s.Accel != "" && s.Accel != "kvm":
+		return fmt.Errorf("a /dev/kvm descriptor for accel=%s, which opens no device", s.Accel)
+	case s.KVMFDSet != 0 && !sets[s.KVMFDSet]:
+		return fmt.Errorf("/dev/kvm is descriptor set %d, which Spec.FDSets does not have", s.KVMFDSet)
+	case s.VsockFD != 0 && s.VsockCID == 0:
+		return fmt.Errorf("a /dev/vhost-vsock descriptor for a machine with no vsock")
+	case s.VsockFD != 0 && s.VsockFD < 3:
+		return fmt.Errorf("a /dev/vhost-vsock descriptor below 3 is stdin, stdout or stderr")
+	}
+	for i, n := range s.NICs {
+		if n.VhostFD != 0 && n.VhostFD < 3 {
+			return fmt.Errorf("NIC %d: a /dev/vhost-net descriptor below 3 is stdin, stdout or stderr", i)
+		}
 	}
 	for i, d := range s.Disks {
-		if d.Path == "" {
-			return fmt.Errorf("disk %d has no path", i)
+		if err := d.validate(sets); err != nil {
+			return fmt.Errorf("disk %d: %w", i, err)
+		}
+	}
+	return nil
+}
+
+// fdSets indexes Spec.FDSets by id, refusing what QEMU would refuse later and less clearly.
+func (s Spec) fdSets() (map[int]bool, error) {
+	sets := make(map[int]bool, len(s.FDSets))
+	for _, set := range s.FDSets {
+		switch {
+		case sets[set.ID]:
+			return nil, fmt.Errorf("descriptor set %d is given twice", set.ID)
+		case len(set.FDs) == 0:
+			return nil, fmt.Errorf("descriptor set %d holds no descriptor", set.ID)
 		}
-		if d.Format == "" {
-			return fmt.Errorf("disk %d (%s) has no format: it is not guessed", i, d.Path)
+		for _, fd := range set.FDs {
+			if fd.Num < 3 {
+				return nil, fmt.Errorf("descriptor set %d names descriptor %d, which is stdin, stdout or stderr", set.ID, fd.Num)
+			}
+		}
+		sets[set.ID] = true
+	}
+	return sets, nil
+}
+
+func (d Disk) validate(sets map[int]bool) error {
+	switch {
+	case d.Path == "" && len(d.Chain) == 0:
+		return fmt.Errorf("no path and no chain")
+	case d.Path != "" && len(d.Chain) != 0:
+		return fmt.Errorf("both a path and a chain: which one the guest reads is not a guess")
+	case d.Path != "" && d.Format == "":
+		return fmt.Errorf("%s has no format: it is not guessed", d.Path)
+	case len(d.Chain) != 0 && (d.Cache != "" || d.Format != ""):
+		return fmt.Errorf("a chain carries a format per image and takes QEMU's caching")
+	}
+	for j, img := range d.Chain {
+		switch {
+		case !sets[img.FDSet]:
+			return fmt.Errorf("image %d is in descriptor set %d, which Spec.FDSets does not have", j, img.FDSet)
+		case img.Format == "":
+			return fmt.Errorf("image %d has no format: it is not guessed", j)
+		case img.Format == "raw" && j != len(d.Chain)-1:
+			return fmt.Errorf("image %d is raw, which has no backing, and images follow it", j)
 		}
 	}
 	return nil
@@ -636,12 +833,16 @@ func (s Spec) Args() ([]string, error) {
 		// done from outside rather than from inside QEMU.
 		"-sandbox", "on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny",
 
-		"-machine", shape.Machine,
+		"-machine", s.machineArg(shape),
 		"-cpu", shape.CPU,
 		"-smp", shape.SMP,
 		"-m", shape.Memory,
 	}
 
+	if s.KVMFDSet != 0 {
+		args = append(args, "-accel", fmt.Sprintf("kvm,device=/dev/fdset/%d", s.KVMFDSet))
+	}
+
 	if s.Memory.File != "" {
 		// A restore opens the file read-only. QEMU otherwise opens it read-write even to
 		// map it private, so every VM restored from a template could write the template
@@ -722,9 +923,12 @@ func (s Spec) Args() ([]string, error) {
 			virtioModern, SlotBalloon))
 
 	if s.VsockCID != 0 {
-		args = append(args, "-device",
-			fmt.Sprintf("vhost-vsock-pci,guest-cid=%d,%s,addr=0x%x",
-				s.VsockCID, virtioModern, SlotVsock))
+		dev := fmt.Sprintf("vhost-vsock-pci,guest-cid=%d,%s,addr=0x%x",
+			s.VsockCID, virtioModern, SlotVsock)
+		if s.VsockFD != 0 {
+			dev += fmt.Sprintf(",vhostfd=%d", s.VsockFD)
+		}
+		args = append(args, "-device", dev)
 	}
 
 	// virtio-mem, when this VM is allowed to grow: the region between its boot
@@ -788,7 +992,16 @@ func (s Spec) Args() ([]string, error) {
 		if d.Serial != "" {
 			dev += ",serial=" + d.Serial
 		}
-		args = append(args, "-drive", drive, "-device", dev)
+		if len(d.Chain) != 0 {
+			chain, err := d.chainArgs(i)
+			if err != nil {
+				return nil, err
+			}
+			args = append(args, chain...)
+		} else {
+			args = append(args, "-drive", drive)
+		}
+		args = append(args, "-device", dev)
 	}
 
 	for i, n := range s.NICs {
@@ -807,14 +1020,21 @@ func (s Spec) Args() ([]string, error) {
 		if n.MTU > 0 {
 			dev += fmt.Sprintf(",host_mtu=%d", n.MTU)
 		}
-		args = append(args,
-			"-netdev", fmt.Sprintf("tap,id=net%d,fd=%d,vhost=on", i, n.TapFD),
-			"-device", dev)
+		netdev := fmt.Sprintf("tap,id=net%d,fd=%d,vhost=on", i, n.TapFD)
+		if n.VhostFD != 0 {
+			netdev += fmt.Sprintf(",vhostfd=%d", n.VhostFD)
+		}
+		args = append(args, "-netdev", netdev, "-device", dev)
 	}
 
-	if s.Serial != "" {
+	switch {
+	case s.SerialFDSet != 0:
+		args = append(args,
+			"-chardev", fmt.Sprintf("file,id=serial0,path=/dev/fdset/%d,append=on", s.SerialFDSet),
+			"-serial", "chardev:serial0")
+	case s.Serial != "":
 		args = append(args, "-serial", s.Serial)
-	} else {
+	default:
 		args = append(args, "-serial", "none")
 	}
 
@@ -825,6 +1045,28 @@ func (s Spec) Args() ([]string, error) {
 		args = append(args, "-qmp",
 			fmt.Sprintf("unix:%s,server=on,wait=off", sock))
 	}
+	// A monitor on a socket the caller made and listens on: QEMU accepts on the descriptor
+	// and never needs a place in the filesystem to put a socket.
+	for i, fd := range []int{s.QMPFD, s.QMPFD2} {
+		if fd == 0 {
+			continue
+		}
+		// -object monitor-qmp and not -mon, which QEMU 11.1 warns is deprecated.
+		id := fmt.Sprintf("qmpfd%d", i)
+		args = append(args,
+			"-chardev", fmt.Sprintf("socket,id=%s,fd=%d,server=on,wait=off", id, fd),
+			"-object", fmt.Sprintf("monitor-qmp,id=mon-%s,chardev=%s", id, id))
+	}
+	// Commas are doubled: QEMU splits an option on a single one, and an opaque is a path.
+	for _, set := range s.FDSets {
+		for _, fd := range set.FDs {
+			spec := fmt.Sprintf("fd=%d,set=%d", fd.Num, set.ID)
+			if fd.Opaque != "" {
+				spec += ",opaque=" + strings.ReplaceAll(fd.Opaque, ",", ",,")
+			}
+			args = append(args, "-add-fd", spec)
+		}
+	}
 
 	// -incoming, in whichever of its two forms this machine was given. Validate has
 	// already refused a spec carrying both.
@@ -993,7 +1235,7 @@ func (s Spec) topology() string {
 	if s.Memory.MaxMB > s.Memory.SizeMB {
 		fmt.Fprintf(&b, ";virtio-mem-pci@%#x", SlotMem)
 	}
-	if s.Serial != "" {
+	if s.Serial != "" || s.SerialFDSet != 0 {
 		b.WriteString(";isa-serial")
 	}
 	// The empty root ports, which are devices present when the state is loaded even