Attach VM root volumes as disk devices

doc/.custom_wordlist.txt

@@ -303,6 +303,7 @@ uptime
 URI
 URIs
 userspace
+UUIDs
 vCPU
 vCPUs
 VDPA

doc/api-extensions.md

@@ -2542,3 +2542,9 @@ This introduces the configuration keys {config:option}`storage-ceph-pool-conf:ce
 ## `network_get_target`
 
 Adds optional `target` parameter to `GET /1.0/network`. When target is set, forward the request to the specified cluster member and return the non-managed interfaces from that member.
+
+## `instance_root_volume_attachment`
+
+Adds support for instance root volumes to be attached to other instances as disk
+devices. Introduces the `<type>/<volume>` syntax for the `source` property of
+disk devices.

doc/explanation/storage.md

@@ -120,10 +120,10 @@ Storage volumes can be of the following types:
 
 `container`/`virtual-machine`
 : LXD automatically creates one of these storage volumes when you launch an instance.
-  It is used as the root disk for the instance, and it is destroyed when the instance is deleted.
+  It is used as the root disk for the instance and is destroyed when the instance is deleted.
 
-  This storage volume is created in the storage pool that is specified in the profile used when launching the instance (or the default profile, if no profile is specified).
-  The storage pool can be explicitly specified by providing the `--storage` flag to the launch command.
+  The storage pool can be explicitly specified by providing the `--storage` flag to the {ref}`launch command <lxc_launch.md>`.
+  If no pool or profile is specified, LXD uses the storage pool of the default profile's root disk device.
 
 `image`
 : LXD automatically creates one of these storage volumes when it unpacks an image to launch one or more instances from it.
@@ -157,7 +157,7 @@ Each storage volume uses one of the following content types:
 
   Custom storage volumes of content type `block` can only be attached to virtual machines.
   By default, they can only be attached to one instance at a time, because simultaneous access can lead to data corruption.
-  Sharing a custom storage volumes of content type `block` is made possible through the usage of the `security.shared` configuration key.
+  Sharing custom storage volumes of content type `block` is made possible through the usage of the `security.shared` configuration key.
 
 `iso`
 : This content type is used for custom ISO volumes.

doc/howto/storage_volumes.md

@@ -179,6 +179,34 @@ For example, to set the default volume size for `my-pool`, use the following com
 
     lxc storage set my-pool volume.size=15GiB
 
+## Attach instance root volumes to other instances
+Virtual-machine root volumes can be attached as disk devices to other virtual machines.
+In order to prevent concurrent access, `security.protection.start` must be set on
+an instance before its root volume can be attached to another virtual-machine.
+
+```{caution}
+Because instances created from the same image share the same partition and file system
+UUIDs and labels, booting an instance with two root file systems mounted may result
+in the wrong root file system being used. This may result in unexpected behavior
+or data loss. **It is strongly recommended to only attach virtual-machine root
+volumes to other virtual machines when the target virtual-machine is running.**
+```
+
+Assuming `vm1` is stopped and `vm2` is running, attach the `virtual-machine/vm1` storage
+volume to `vm2`:
+
+    lxc config set vm1 security.protection.start=true
+    lxc storage volume attach my-pool virtual-machine/vm1 vm2
+
+`virtual-machine/vm1` must be detached from `vm2` before `security.protection.start`
+can be unset from `vm1`:
+
+    lxc storage volume detach my-pool virtual-machine/vm1 vm2
+    lxc config unset vm1 security.protection.start
+
+`security.shared` can also be used on `virtual-machine` volumes to enable concurrent
+access. Note that concurrent access to block volumes may result in data loss.
+
 ## Resize a storage volume
 
 If you need more storage in a volume, you can increase the size of your storage volume.

doc/metadata.txt

@@ -2193,7 +2193,7 @@ See {ref}`container-security` for more information.
 
 ```{config:option} security.protection.delete instance-security
 :defaultdesc: "`false`"
-:liveupdate: "yes"
+:liveupdate: "container"
 :shortdesc: "Whether to prevent the instance from being deleted"
 :type: "bool"
 
@@ -2210,7 +2210,7 @@ Set this option to `true` to prevent the instance's file system from being UID/G
 
 ```{config:option} security.protection.start instance-security
 :defaultdesc: "`false`"
-:liveupdate: "yes"
+:liveupdate: "container"
 :shortdesc: "Whether to prevent the instance from being started"
 :type: "bool"
 
@@ -4881,7 +4881,7 @@ prior to creating the storage pool.
 <!-- config group storage-btrfs-pool-conf end -->
 <!-- config group storage-btrfs-volume-conf start -->
 ```{config:option} security.shared storage-btrfs-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"
@@ -5073,7 +5073,7 @@ If not set, `ext4` is assumed.
 ```
 
 ```{config:option} security.shared storage-ceph-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"
@@ -5404,7 +5404,7 @@ to be placed on the socket I/O.
 <!-- config group storage-dir-pool-conf end -->
 <!-- config group storage-dir-volume-conf start -->
 ```{config:option} security.shared storage-dir-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"
@@ -5622,7 +5622,7 @@ The size must be at least 4096 bytes, and a multiple of 512 bytes.
 ```
 
 ```{config:option} security.shared storage-lvm-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"
@@ -5832,7 +5832,7 @@ If not set, `ext4` is assumed.
 ```
 
 ```{config:option} security.shared storage-powerflex-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"
@@ -6003,7 +6003,7 @@ If not set, `ext4` is assumed.
 ```
 
 ```{config:option} security.shared storage-zfs-volume-conf
-:condition: "custom block volume"
+:condition: "virtual-machine or custom block volume"
 :defaultdesc: "same as `volume.security.shared` or `false`"
 :scope: "global"
 :shortdesc: "Enable volume sharing"

lxc/storage_volume.go

@@ -164,10 +164,12 @@ type cmdStorageVolumeAttach struct {
 
 func (c *cmdStorageVolumeAttach) command() *cobra.Command {
 	cmd := &cobra.Command{}
-	cmd.Use = usage("attach", i18n.G("[<remote>:]<pool> <volume> <instance> [<device name>] [<path>]"))
+	cmd.Use = usage("attach", i18n.G("[<remote>:]<pool> [<type>/]<volume> <instance> [<device name>] [<path>]"))
 	cmd.Short = i18n.G("Attach new storage volumes to instances")
 	cmd.Long = cli.FormatSection(i18n.G("Description"), i18n.G(
-		`Attach new storage volumes to instances`))
+		`Attach new storage volumes to instances
+
+<type> must be one of "custom" or "virtual-machine"`))
 
 	cmd.RunE = c.run
 
@@ -210,8 +212,8 @@ func (c *cmdStorageVolumeAttach) run(cmd *cobra.Command, args []string) error {
 	}
 
 	volName, volType := parseVolume("custom", args[1])
-	if volType != "custom" {
-		return errors.New(i18n.G("Only \"custom\" volumes can be attached to instances"))
+	if volType != "custom" && volType != "virtual-machine" {
+		return errors.New(i18n.G(`Only "custom" and "virtual-machine" volumes can be attached to instances`))
 	}
 
 	// Attach the volume
@@ -257,7 +259,7 @@ func (c *cmdStorageVolumeAttach) run(cmd *cobra.Command, args []string) error {
 	device := map[string]string{
 		"type":   "disk",
 		"pool":   resource.name,
-		"source": volName,
+		"source": args[1],
 		"path":   devPath,
 	}
 
@@ -279,10 +281,12 @@ type cmdStorageVolumeAttachProfile struct {
 
 func (c *cmdStorageVolumeAttachProfile) command() *cobra.Command {
 	cmd := &cobra.Command{}
-	cmd.Use = usage("attach-profile", i18n.G("[<remote:>]<pool> <volume> <profile> [<device name>] [<path>]"))
+	cmd.Use = usage("attach-profile", i18n.G("[<remote:>]<pool> [<type>/]<volume> <profile> [<device name>] [<path>]"))
 	cmd.Short = i18n.G("Attach new storage volumes to profiles")
 	cmd.Long = cli.FormatSection(i18n.G("Description"), i18n.G(
-		`Attach new storage volumes to profiles`))
+		`Attach new storage volumes to profiles
+
+<type> must be one of "custom" or "virtual-machine"`))
 
 	cmd.RunE = c.run
 
@@ -340,8 +344,8 @@ func (c *cmdStorageVolumeAttachProfile) run(cmd *cobra.Command, args []string) e
 	}
 
 	volName, volType := parseVolume("custom", args[1])
-	if volType != "custom" {
-		return errors.New(i18n.G("Only \"custom\" volumes can be attached to instances"))
+	if volType != "custom" && volType != "virtual-machine" {
+		return errors.New(i18n.G(`Only "custom" and "virtual-machine" volumes can be attached to profiles`))
 	}
 
 	// Check if the requested storage volume actually exists
@@ -354,7 +358,7 @@ func (c *cmdStorageVolumeAttachProfile) run(cmd *cobra.Command, args []string) e
 	device := map[string]string{
 		"type":   "disk",
 		"pool":   resource.name,
-		"source": vol.Name,
+		"source": args[1],
 	}
 
 	// Ignore path for block volumes
@@ -803,7 +807,7 @@ type cmdStorageVolumeDetach struct {
 
 func (c *cmdStorageVolumeDetach) command() *cobra.Command {
 	cmd := &cobra.Command{}
-	cmd.Use = usage("detach", i18n.G("[<remote>:]<pool> <volume> <instance> [<device name>]"))
+	cmd.Use = usage("detach", i18n.G("[<remote>:]<pool> [<type>/]<volume> <instance> [<device name>]"))
 	cmd.Short = i18n.G("Detach storage volumes from instances")
 	cmd.Long = cli.FormatSection(i18n.G("Description"), i18n.G(
 		`Detach storage volumes from instances`))
@@ -861,14 +865,13 @@ func (c *cmdStorageVolumeDetach) run(cmd *cobra.Command, args []string) error {
 	}
 
 	volName, volType := parseVolume("custom", args[1])
-	if volType != "custom" {
-		return errors.New(i18n.G(`Only "custom" volumes can be attached to instances`))
-	}
 
 	// Find the device
 	if devName == "" {
 		for n, d := range inst.Devices {
-			if d["type"] == "disk" && d["pool"] == resource.name && d["source"] == volName {
+			sourceName, sourceType := parseVolume("custom", d["source"])
+
+			if d["type"] == "disk" && d["pool"] == resource.name && volType == sourceType && volName == sourceName {
 				if devName != "" {
 					return errors.New(i18n.G("More than one device matches, specify the device name"))
 				}
@@ -906,7 +909,7 @@ type cmdStorageVolumeDetachProfile struct {
 
 func (c *cmdStorageVolumeDetachProfile) command() *cobra.Command {
 	cmd := &cobra.Command{}
-	cmd.Use = usage("detach-profile", i18n.G("[<remote:>]<pool> <volume> <profile> [<device name>]"))
+	cmd.Use = usage("detach-profile", i18n.G("[<remote:>]<pool> [<type>/]<volume> <profile> [<device name>]"))
 	cmd.Short = i18n.G("Detach storage volumes from profiles")
 	cmd.Long = cli.FormatSection(i18n.G("Description"), i18n.G(
 		`Detach storage volumes from profiles`))
@@ -963,14 +966,13 @@ func (c *cmdStorageVolumeDetachProfile) run(cmd *cobra.Command, args []string) e
 	}
 
 	volName, volType := parseVolume("custom", args[1])
-	if volType != "custom" {
-		return errors.New(i18n.G(`Only "custom" volumes can be attached to instances`))
-	}
 
 	// Find the device
 	if devName == "" {
 		for n, d := range profile.Devices {
-			if d["type"] == "disk" && d["pool"] == resource.name && d["source"] == volName {
+			sourceName, sourceType := parseVolume("custom", d["source"])
+
+			if d["type"] == "disk" && d["pool"] == resource.name && volType == sourceType && volName == sourceName {
 				if devName != "" {
 					return errors.New(i18n.G("More than one device matches, specify the device name"))
 				}

lxd/device/disk.go

@@ -147,15 +147,16 @@ func (d *disk) sourceIsLocalPath(source string) bool {
 	return true
 }
 
-// Check that unshared custom storage block volumes are not added to profiles or multiple instances.
+// Check that unshared custom storage block volumes are not added to profiles or
+// multiple instances unless they will not be accessed concurrently.
 func (d *disk) checkBlockVolSharing(instanceType instancetype.Type, projectName string, volume *api.StorageVolume) error {
 	// Skip the checks if the volume is set to be shared or is not a block volume.
 	if volume.ContentType != cluster.StoragePoolVolumeContentTypeNameBlock || shared.IsTrue(volume.Config["security.shared"]) {
 		return nil
 	}
 
 	if instanceType == instancetype.Any {
-		return fmt.Errorf("Cannot add custom storage block volume to profiles if security.shared is false or unset")
+		return fmt.Errorf("Cannot add block volume to profiles if security.shared is false or unset")
 	}
 
 	err := storagePools.VolumeUsedByInstanceDevices(d.state, d.pool.Name(), projectName, volume, true, func(inst db.InstanceArgs, project api.Project, usedByDevices []string) error {
@@ -164,13 +165,24 @@ func (d *disk) checkBlockVolSharing(instanceType instancetype.Type, projectName
 			return nil
 		}
 
-		return db.ErrListStop
-	})
-	if err != nil {
-		if err == db.ErrListStop {
-			return fmt.Errorf("Cannot add custom storage block volume to more than one instance if security.shared is false or unset")
+		// Don't count a VM volume's instance if security.protection.start is preventing that instance from starting.
+		// It's safe to share block volumes with an instance that cannot start.
+		if volume.Type == cluster.StoragePoolVolumeTypeNameVM && volume.Project == inst.Project && volume.Name == inst.Name {
+			apiInst, err := inst.ToAPI()
+			if err != nil {
+				return err
+			}
+
+			apiInst.ExpandedConfig = instancetype.ExpandInstanceConfig(d.state.GlobalConfig.Dump(), apiInst.Config, inst.Profiles)
+
+			if shared.IsTrue(apiInst.ExpandedConfig["security.protection.start"]) {
+				return nil
+			}
 		}
 
+		return fmt.Errorf("Cannot add block volume to more than one instance if security.shared is false or unset")
+	})
+	if err != nil {
 		return err
 	}
 
@@ -468,6 +480,17 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error {
 					return err
 				}
 
+				if d.inst != nil {
+					instVolType, err := storagePools.InstanceTypeToVolumeType(d.inst.Type())
+					if err != nil {
+						return err
+					}
+
+					if instVolType == volumeType && d.inst.Name() == volumeName {
+						return errors.New("Instance root device cannot be attached to itself")
+					}
+				}
+
 				// Derive the effective storage project name from the instance config's project.
 				storageProjectName, err = project.StorageVolumeProject(d.state.DB.Cluster, instConf.Project().Name, dbVolumeType)
 				if err != nil {
@@ -1579,9 +1602,6 @@ func (d *disk) mountPoolVolume() (func(), string, *storagePools.MountInfo, error
 		return nil, "", nil, err
 	}
 
-	volStorageName := project.StorageVolume(storageProjectName, volumeName)
-	srcPath := storageDrivers.GetVolumeMountPath(d.config["pool"], volumeType, volStorageName)
-
 	mountInfo, err = d.pool.MountVolume(storageProjectName, volumeName, volumeType, nil)
 	if err != nil {
 		return nil, "", nil, fmt.Errorf(`Failed mounting storage volume "%s/%s" from storage pool %q: %w`, volumeTypeName, volumeName, d.pool.Name(), err)
@@ -1600,6 +1620,15 @@ func (d *disk) mountPoolVolume() (func(), string, *storagePools.MountInfo, error
 		return nil, "", nil, fmt.Errorf("Failed to fetch local storage volume record: %w", err)
 	}
 
+	var volStorageName string
+	if dbVolume.Type == cluster.StoragePoolVolumeTypeNameCustom {
+		volStorageName = project.StorageVolume(storageProjectName, volumeName)
+	} else {
+		volStorageName = project.Instance(storageProjectName, volumeName)
+	}
+
+	srcPath := storageDrivers.GetVolumeMountPath(d.config["pool"], volumeType, volStorageName)
+
 	if d.inst.Type() == instancetype.Container {
 		if dbVolume.ContentType != cluster.StoragePoolVolumeContentTypeNameFS {
 			return nil, "", nil, fmt.Errorf("Only filesystem volumes are supported for containers")
@@ -1612,8 +1641,6 @@ func (d *disk) mountPoolVolume() (func(), string, *storagePools.MountInfo, error
 	}
 
 	if dbVolume.ContentType == cluster.StoragePoolVolumeContentTypeNameBlock || dbVolume.ContentType == cluster.StoragePoolVolumeContentTypeNameISO {
-		volStorageName := project.StorageVolume(storageProjectName, volumeName)
-
 		volume := d.pool.GetVolume(volumeType, storageDrivers.ContentType(dbVolume.ContentType), volStorageName, dbVolume.Config)
 
 		srcPath, err = d.pool.Driver().GetVolumeDiskPath(volume)