Per-pool project limits (from Incus)

doc/api-extensions.md

@@ -2464,3 +2464,9 @@ Adds the following internal metrics:
 
 * Total completed requests
 * Number of ongoing requests
+
+## `projects_limits_disk_pool`
+
+This introduces per-pool project disk limits, introducing a `limits.disk.pool.NAME`
+configuration option to the project limits. When `limits.disk.pool.POOLNAME: 0`
+for a project, the pool is excluded from `lxc storage list` in that project.

doc/metadata.txt

@@ -1,5 +1,18 @@
 // Code generated by lxd-metadata; DO NOT EDIT.
 
+<!-- config group -limits, start -->
+```{config:option} limits.disk.pool.POOL_NAME -limits,
+:shortdesc: "Maximum disk space used by the project on this pool"
+:type: "string"
+This value is the maximum value of the aggregate disk
+space used by all instance volumes, custom volumes, and images of the
+project on this specific storage pool.
+
+When set to 0, the pool is excluded from storage pool list for
+the project.
+```
+
+<!-- config group -limits, end -->
 <!-- config group cluster-cluster start -->
 ```{config:option} scheduler.instance cluster-cluster
 :defaultdesc: "`all`"

lxc/project.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"sort"
 	"strings"
 
@@ -959,23 +960,31 @@ func (c *cmdProjectInfo) run(cmd *cobra.Command, args []string) error {
 	byteLimits := []string{"disk", "memory"}
 	data := [][]string{}
 	for k, v := range projectState.Resources {
+		shortKey := strings.SplitN(k, ".", 2)[0]
+
 		limit := i18n.G("UNLIMITED")
 		if v.Limit >= 0 {
-			if shared.ValueInSlice(k, byteLimits) {
+			if slices.Contains(byteLimits, shortKey) {
 				limit = units.GetByteSizeStringIEC(v.Limit, 2)
 			} else {
 				limit = fmt.Sprintf("%d", v.Limit)
 			}
 		}
 
 		usage := ""
-		if shared.ValueInSlice(k, byteLimits) {
+		if slices.Contains(byteLimits, shortKey) {
 			usage = units.GetByteSizeStringIEC(v.Usage, 2)
 		} else {
 			usage = fmt.Sprintf("%d", v.Usage)
 		}
 
-		data = append(data, []string{strings.ToUpper(k), limit, usage})
+		columnName := strings.ToUpper(k)
+		fields := strings.SplitN(columnName, ".", 2)
+		if len(fields) == 2 {
+			columnName = fmt.Sprintf("%s (%s)", fields[0], fields[1])
+		}
+
+		data = append(data, []string{columnName, limit, usage})
 	}
 
 	sort.Sort(cli.SortColumnsNaturally(data))

lxd/api_project.go

@@ -1380,6 +1380,37 @@ func projectValidateConfig(s *state.State, config map[string]string) error {
 		"restricted.snapshots": isEitherAllowOrBlock,
 	}
 
+	// Add the storage pool keys.
+	err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
+		var err error
+
+		// Load all the pools.
+		pools, err := tx.GetStoragePoolNames(ctx)
+		if err != nil {
+			return err
+		}
+
+		// Add the storage-pool specific config keys.
+		for _, poolName := range pools {
+			// lxdmeta:generate(entity=project, group=limits, key=limits.disk.pool.POOL_NAME)
+			// This value is the maximum value of the aggregate disk
+			// space used by all instance volumes, custom volumes, and images of the
+			// project on this specific storage pool.
+			//
+			// When set to 0, the pool is excluded from storage pool list for
+			// the project.
+			// ---
+			//  type: string
+			//  shortdesc: Maximum disk space used by the project on this pool
+			projectConfigKeys[fmt.Sprintf("limits.disk.pool.%s", poolName)] = validate.Optional(validate.IsSize)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("Failed loading storage pool names: %w", err)
+	}
+
 	for k, v := range config {
 		key := k
 

lxd/metadata/configuration.json

@@ -1,5 +1,18 @@
 {
 	"configs": {
+		"": {
+			"limits,": {
+				"keys": [
+					{
+						"limits.disk.pool.POOL_NAME": {
+							"longdesc": "This value is the maximum value of the aggregate disk\nspace used by all instance volumes, custom volumes, and images of the\nproject on this specific storage pool.\n\nWhen set to 0, the pool is excluded from storage pool list for\nthe project.",
+							"shortdesc": "Maximum disk space used by the project on this pool",
+							"type": "string"
+						}
+					}
+				]
+			}
+		},
 		"cluster": {
 			"cluster": {
 				"keys": [

lxd/project/limits/permissions.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -23,6 +24,36 @@ import (
 	"github.com/canonical/lxd/shared/validate"
 )
 
+// projectLimitDiskPool is the prefix used for pool-specific disk limits.
+var projectLimitDiskPool = "limits.disk.pool."
+
+// HiddenStoragePools returns a list of storage pools that should be hidden from users of the project.
+func HiddenStoragePools(ctx context.Context, tx *db.ClusterTx, projectName string) ([]string, error) {
+	dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName)
+	if err != nil {
+		return nil, fmt.Errorf("Failed getting project: %w", err)
+	}
+
+	project, err := dbProject.ToAPI(ctx, tx.Tx())
+	if err != nil {
+		return nil, err
+	}
+
+	hiddenPools := []string{}
+	for k, v := range project.Config {
+		if !strings.HasPrefix(k, projectLimitDiskPool) || v != "0" {
+			continue
+		}
+
+		fields := strings.SplitN(k, projectLimitDiskPool, 2)
+		if len(fields) == 2 {
+			hiddenPools = append(hiddenPools, fields[1])
+		}
+	}
+
+	return hiddenPools, nil
+}
+
 // AllowInstanceCreation returns an error if any project-specific limit or
 // restriction is violated when creating a new instance.
 func AllowInstanceCreation(globalConfig *clusterConfig.Config, tx *db.ClusterTx, projectName string, req api.InstancesPost) error {
@@ -234,7 +265,7 @@ func checkRestrictionsOnVolatileConfig(project api.Project, instanceType instanc
 
 // AllowVolumeCreation returns an error if any project-specific limit or
 // restriction is violated when creating a new custom volume in a project.
-func AllowVolumeCreation(globalConfig *clusterConfig.Config, tx *db.ClusterTx, projectName string, req api.StorageVolumesPost) error {
+func AllowVolumeCreation(globalConfig *clusterConfig.Config, tx *db.ClusterTx, projectName string, poolName string, req api.StorageVolumesPost) error {
 	var globalConfigDump map[string]any
 	if globalConfig != nil {
 		globalConfigDump = globalConfig.Dump()
@@ -256,8 +287,9 @@ func AllowVolumeCreation(globalConfig *clusterConfig.Config, tx *db.ClusterTx, p
 
 	// Add the volume being created.
 	info.Volumes = append(info.Volumes, db.StorageVolumeArgs{
-		Name:   req.Name,
-		Config: req.Config,
+		Name:     req.Name,
+		Config:   req.Config,
+		PoolName: poolName,
 	})
 
 	err = checkRestrictionsAndAggregateLimits(globalConfig, tx, info)
@@ -329,8 +361,9 @@ func checkRestrictionsAndAggregateLimits(globalConfig *clusterConfig.Config, tx
 	// across all project instances.
 	aggregateKeys := []string{}
 	isRestricted := false
+
 	for key, value := range info.Project.Config {
-		if shared.ValueInSlice(key, allAggregateLimits) {
+		if slices.Contains(allAggregateLimits, key) || strings.HasPrefix(key, projectLimitDiskPool) {
 			aggregateKeys = append(aggregateKeys, key)
 			continue
 		}
@@ -388,7 +421,14 @@ func getAggregateLimits(info *projectInfo, aggregateKeys []string) (map[string]a
 		max := int64(-1)
 		limit := info.Project.Config[key]
 		if limit != "" {
-			parser := aggregateLimitConfigValueParsers[key]
+			keyName := key
+
+			// Handle pool-specific limits.
+			if strings.HasPrefix(key, projectLimitDiskPool) {
+				keyName = "limits.disk"
+			}
+
+			parser := aggregateLimitConfigValueParsers[keyName]
 			max, err = parser(info.Project.Config[key])
 			if err != nil {
 				return nil, err
@@ -417,7 +457,14 @@ func checkAggregateLimits(info *projectInfo, aggregateKeys []string) error {
 	}
 
 	for _, key := range aggregateKeys {
-		parser := aggregateLimitConfigValueParsers[key]
+		keyName := key
+
+		// Handle pool-specific limits.
+		if strings.HasPrefix(key, projectLimitDiskPool) {
+			keyName = "limits.disk"
+		}
+
+		parser := aggregateLimitConfigValueParsers[keyName]
 		max, err := parser(info.Project.Config[key])
 		if err != nil {
 			return err
@@ -427,6 +474,7 @@ func checkAggregateLimits(info *projectInfo, aggregateKeys []string) error {
 			return fmt.Errorf("Reached maximum aggregate value %q for %q in project %q", info.Project.Config[key], key, info.Project.Name)
 		}
 	}
+
 	return nil
 }
 
@@ -1125,15 +1173,29 @@ func validateAggregateLimit(totals map[string]int64, key, value string) error {
 		return nil
 	}
 
-	parser := aggregateLimitConfigValueParsers[key]
+	keyName := key
+
+	// Handle pool-specific limits.
+	if strings.HasPrefix(key, projectLimitDiskPool) {
+		keyName = "limits.disk"
+	}
+
+	parser := aggregateLimitConfigValueParsers[keyName]
 	limit, err := parser(value)
 	if err != nil {
 		return fmt.Errorf("Invalid value %q for limit %q: %w", value, key, err)
 	}
 
 	total := totals[key]
 	if limit < total {
-		printer := aggregateLimitConfigValuePrinters[key]
+		keyName := key
+
+		// Handle pool-specific limits.
+		if strings.HasPrefix(key, projectLimitDiskPool) {
+			keyName = "limits.disk"
+		}
+
+		printer := aggregateLimitConfigValuePrinters[keyName]
 		return fmt.Errorf("%q is too low: current total is %q", key, printer(total))
 	}
 
@@ -1287,8 +1349,18 @@ func getTotalsAcrossProjectEntities(info *projectInfo, keys []string, skipUnset
 
 	for _, key := range keys {
 		totals[key] = 0
-		if key == "limits.disk" {
+		if key == "limits.disk" || strings.HasPrefix(key, projectLimitDiskPool) {
+			poolName := ""
+			fields := strings.SplitN(key, projectLimitDiskPool, 2)
+			if len(fields) == 2 {
+				poolName = fields[1]
+			}
+
 			for _, volume := range info.Volumes {
+				if poolName != "" && volume.PoolName != poolName {
+					continue
+				}
+
 				value, ok := volume.Config["size"]
 				if !ok {
 					if skipUnset {
@@ -1329,14 +1401,31 @@ func getInstanceLimits(instance api.Instance, keys []string, skipUnset bool, sto
 
 	for _, key := range keys {
 		var limit int64
-		parser := aggregateLimitConfigValueParsers[key]
+		keyName := key
+
+		// Handle pool-specific limits.
+		if strings.HasPrefix(key, projectLimitDiskPool) {
+			keyName = "limits.disk"
+		}
+
+		parser := aggregateLimitConfigValueParsers[keyName]
+
+		if key == "limits.disk" || strings.HasPrefix(key, projectLimitDiskPool) {
+			poolName := ""
+			fields := strings.SplitN(key, projectLimitDiskPool, 2)
+			if len(fields) == 2 {
+				poolName = fields[1]
+			}
 
-		if key == "limits.disk" {
 			_, device, err := instancetype.GetRootDiskDevice(instance.Devices)
 			if err != nil {
 				return nil, fmt.Errorf("Failed getting root disk device for instance %q in project %q: %w", instance.Name, instance.Project, err)
 			}
 
+			if poolName != "" && device["pool"] != poolName {
+				continue
+			}
+
 			value, ok := device["size"]
 			if !ok || value == "" {
 				if skipUnset {

lxd/project/limits/state.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 
 	"github.com/canonical/lxd/lxd/db"
 	"github.com/canonical/lxd/lxd/instance/instancetype"
@@ -29,6 +30,16 @@ func GetCurrentAllocations(globalConfig map[string]any, ctx context.Context, tx
 		return nil, err
 	}
 
+	// Get per-pool limits.
+	poolLimits := []string{}
+	for k := range info.Project.Config {
+		if strings.HasPrefix(k, projectLimitDiskPool) {
+			poolLimits = append(poolLimits, k)
+		}
+	}
+
+	allAggregateLimits := append(allAggregateLimits, poolLimits...)
+
 	// Get the instance aggregated values.
 	raw, err := getAggregateLimits(info, allAggregateLimits)
 	if err != nil {
@@ -41,6 +52,13 @@ func GetCurrentAllocations(globalConfig map[string]any, ctx context.Context, tx
 	result["networks"] = raw["limits.networks"]
 	result["processes"] = raw["limits.processes"]
 
+	// Add the pool-specific disk limits.
+	for k, v := range raw {
+		if strings.HasPrefix(k, projectLimitDiskPool) && v.Limit > 0 {
+			result[fmt.Sprintf("disk.%s", strings.SplitN(k, ".", 4)[3])] = v
+		}
+	}
+
 	// Get the instance count values.
 	count, limit, err := getTotalInstanceCountLimit(info)
 	if err != nil {

lxd/storage/backend_lxd.go

@@ -7572,7 +7572,7 @@ func (b *lxdBackend) CreateCustomVolumeFromISO(projectName string, volName strin
 	}
 
 	err := b.state.DB.Cluster.Transaction(b.state.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
-		return limits.AllowVolumeCreation(b.state.GlobalConfig, tx, projectName, req)
+		return limits.AllowVolumeCreation(b.state.GlobalConfig, tx, projectName, b.name, req)
 	})
 	if err != nil {
 		return fmt.Errorf("Failed checking volume creation allowed: %w", err)
@@ -7677,7 +7677,7 @@ func (b *lxdBackend) CreateCustomVolumeFromBackup(srcBackup backup.Info, srcData
 	}
 
 	err = b.state.DB.Cluster.Transaction(b.state.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
-		return limits.AllowVolumeCreation(b.state.GlobalConfig, tx, srcBackup.Project, req)
+		return limits.AllowVolumeCreation(b.state.GlobalConfig, tx, srcBackup.Project, b.name, req)
 	})
 	if err != nil {
 		return fmt.Errorf("Failed checking volume creation allowed: %w", err)