Test qcow2 images with different allocation

Add a helper for creating an image from list of extents. With this we can create a raw or qcow2 image with allocation described by the extents. Add 4 tests, testing different allocation patterns: - TestExtentsSome: some allocated clusters and some holes - TestExtentsPartial: writing partial cluster allocates entire cluster - TestExtentsMerge: consecutive extents of same type are merged - TestExtentsZero: different extents types that read as zeros For each test we verify qcow2 and qcow2 compressed (zlib) images. Signed-off-by: Nir Soffer <[email protected]>
lima-vm · Nov 3, 2024 · a5becd5 · a5becd5
1 parent acc7520
commit a5becd5
Showing 1 changed file with 358 additions and 0 deletions.
diff --git a/qcow2reader_test.go b/qcow2reader_test.go
@@ -7,15 +7,18 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/lima-vm/go-qcow2reader"
 	"github.com/lima-vm/go-qcow2reader/convert"
 	"github.com/lima-vm/go-qcow2reader/image"
 	"github.com/lima-vm/go-qcow2reader/test/qemuimg"
+	"github.com/lima-vm/go-qcow2reader/test/qemuio"
 )
 
 const (
+	KiB = int64(1) << 10
 	MiB = int64(1) << 20
 	GiB = int64(1) << 30
 )
@@ -159,6 +162,361 @@ func BenchmarkExtentsUnallocated(b *testing.B) {
 	}
 }
 
+func TestExtentsSome(t *testing.T) {
+	clusterSize := 64 * KiB
+	extents := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  1 * clusterSize,
+			Length: 1 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     2 * clusterSize,
+			Length:    2 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  4 * clusterSize,
+			Length: 96 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     100 * clusterSize,
+			Length:    8 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  108 * clusterSize,
+			Length: 892 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     1000 * clusterSize,
+			Length:    16 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  1016 * clusterSize,
+			Length: 8984 * clusterSize,
+			Zero:   true,
+		},
+	}
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(extents, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(extents), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsPartial(t *testing.T) {
+	clusterSize := 64 * KiB
+
+	// Writing part of of a cluster allocates entire cluster in the qcow2 image.
+	extents := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    1,
+			Allocated: true,
+		},
+		{
+			Start:  1 * clusterSize,
+			Length: 98 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     100*clusterSize - 1,
+			Length:    1,
+			Allocated: true,
+		},
+	}
+
+	// Listing extents works in cluster granularity.
+	full := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  1 * clusterSize,
+			Length: 98 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     99 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(full, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(full), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsMerge(t *testing.T) {
+	clusterSize := 64 * KiB
+
+	// Create image with consecutive extents of same type.
+	extents := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:     1 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  2 * clusterSize,
+			Length: 98 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     100 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:     101 * clusterSize,
+			Length:    1 * clusterSize,
+			Allocated: true,
+		},
+	}
+
+	// Extents with same type are merged.
+	merged := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    2 * clusterSize,
+			Allocated: true,
+		},
+		{
+			Start:  2 * clusterSize,
+			Length: 98 * clusterSize,
+			Zero:   true,
+		},
+		{
+			Start:     100 * clusterSize,
+			Length:    2 * clusterSize,
+			Allocated: true,
+		},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(merged, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(compressed(merged), actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func TestExtentsZero(t *testing.T) {
+	clusterSize := 64 * KiB
+
+	// Create image with different clusters that read as zeros.
+	extents := []image.Extent{
+		{
+			Start:     0 * clusterSize,
+			Length:    1000 * clusterSize,
+			Allocated: true,
+			Zero:      true,
+		},
+		{
+			Start:  1000 * clusterSize,
+			Length: 1000 * clusterSize,
+			Zero:   true,
+		},
+	}
+
+	qcow2 := filepath.Join(t.TempDir(), "image")
+	if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil {
+		t.Fatal(err)
+	}
+	t.Run("qcow2", func(t *testing.T) {
+		actual, err := listExtents(qcow2)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Equal(extents, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+	t.Run("qcow2 zlib", func(t *testing.T) {
+		qcow2Zlib := qcow2 + ".zlib"
+		if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil {
+			t.Fatal(err)
+		}
+		actual, err := listExtents(qcow2Zlib)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// When converting to qcow2 images all clusters that read as zeros are
+		// converted to unallocated clusters.
+		converted := []image.Extent{
+			{
+				Start:  0 * clusterSize,
+				Length: 2000 * clusterSize,
+				Zero:   true,
+			},
+		}
+		if !slices.Equal(converted, actual) {
+			t.Fatalf("expected %v, got %v", extents, actual)
+		}
+	})
+}
+
+func compressed(extents []image.Extent) []image.Extent {
+	var res []image.Extent
+	for _, extent := range extents {
+		if extent.Allocated {
+			extent.Compressed = true
+		}
+		res = append(res, extent)
+	}
+	return res
+}
+
+func listExtents(path string) ([]image.Extent, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	img, err := qcow2reader.Open(f)
+	if err != nil {
+		return nil, err
+	}
+	defer img.Close()
+
+	var extents []image.Extent
+	var start int64
+
+	end := img.Size()
+	for start < end {
+		extent, err := img.Next(start, end-start)
+		if err != nil {
+			return nil, err
+		}
+		if extent.Start != start {
+			return nil, fmt.Errorf("invalid extent start: %+v", extent)
+		}
+		if extent.Length <= 0 {
+			return nil, fmt.Errorf("invalid extent length: %+v", extent)
+		}
+		extents = append(extents, extent)
+		start += extent.Length
+	}
+	return extents, nil
+}
+
+// createTestImageWithExtents creates a n image with the allocation described
+// by extents.
+func createTestImageWithExtents(path string, format qemuimg.Format, extents []image.Extent) error {
+	lastExtent := extents[len(extents)-1]
+	size := lastExtent.Start + lastExtent.Length
+	if err := qemuimg.Create(path, format, size, "", ""); err != nil {
+		return err
+	}
+	for _, extent := range extents {
+		if !extent.Allocated {
+			continue
+		}
+		start := extent.Start
+		length := extent.Length
+		for length > 0 {
+			// qemu-io requires length < 2g.
+			n := length
+			if n >= 2*GiB {
+				n = 2*GiB - 64*KiB
+			}
+			if extent.Zero {
+				if err := qemuio.Zero(path, format, start, n); err != nil {
+					return err
+				}
+			} else {
+				if err := qemuio.Write(path, format, start, n, 0x55); err != nil {
+					return err
+				}
+			}
+			start += n
+			length -= n
+		}
+	}
+	return nil
+}
+
 // Benchmark completely empty sparse image (0% utilization).  This is the best
 // case when we don't have to read any cluster from storage.
 func Benchmark0p(b *testing.B) {