Merge pull request #39 from nirs/block-status-preps

Preparing for adding block status interface

.github/workflows/test.yml

@@ -11,23 +11,23 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Run golangci-lint
         uses: golangci/[email protected]
         with:
-          version: v1.52.2
+          version: v1.60.1
           args: --verbose
-      - name: Unit tests
-        run: go test -v ./...
-      - name: Install go-qcow2reader-example
-        run: cd ./cmd/go-qcow2reader-example && go install
       - name: Install qemu-img as a test dependency
         run: |
           sudo apt-get update
           sudo apt-get install -y qemu-utils
+      - name: Unit tests
+        run: go test -v ./...
+      - name: Install go-qcow2reader-example
+        run: cd ./cmd/go-qcow2reader-example && go install
       - name: Cache test-images-ro
         id: cache-test-images-ro
         uses: actions/cache@v4

cmd/go-qcow2reader-example/go.mod

@@ -1,6 +1,6 @@
 module github.com/lima-vm/go-qcow2reader/cmd/go-qcow2reader-example
 
-go 1.20
+go 1.22
 
 require (
 	github.com/klauspost/compress v1.16.5

convert/convert.go

@@ -11,7 +11,7 @@ import (
 const BufferSize = 1024 * 1024
 
 // Smaller value may increase the overhead of synchornizing multiple works.
-// Larger value may be less efficient for smaller images. The defualt value
+// Larger value may be less efficient for smaller images. The default value
 // gives good results for the lima default Ubuntu image.
 const SegmentSize = 32 * BufferSize
 
@@ -80,7 +80,7 @@ type Converter struct {
 	err    error
 }
 
-// New returns a new converter intialized from options.
+// New returns a new converter initialized from options.
 func New(opts Options) (*Converter, error) {
 	if err := opts.Validate(); err != nil {
 		return nil, err
@@ -113,7 +113,7 @@ func (c *Converter) nextSegment() (int64, int64, bool) {
 	return start, c.offset, false
 }
 
-// setError keeps the first error set. Setting the error signal other workes to
+// setError keeps the first error set. Setting the error signal other workers to
 // abort the operation.
 func (c *Converter) setError(err error) {
 	c.mutex.Lock()
@@ -166,7 +166,7 @@ func (c *Converter) Convert(wa io.WriterAt, ra io.ReaderAt, size int64) error {
 						}
 
 						// EOF for the last read of the last segment is expected, but since we
-						// read exactly size bytes, we shoud never get a zero read.
+						// read exactly size bytes, we should never get a zero read.
 						if nr == 0 {
 							c.setError(errors.New("unexpected EOF"))
 							return

go.mod

@@ -1,3 +1,3 @@
 module github.com/lima-vm/go-qcow2reader
 
-go 1.20
+go 1.22

image/qcow2/qcow2.go

@@ -536,6 +536,7 @@ type Qcow2 struct {
 	HeaderExtensions    []HeaderExtension `json:"header_extensions"`
 	errUnreadable       error
 	clusterSize         int
+	l2Entries           int
 	l1Table             []l1TableEntry
 	l2TableCache        *lru.Cache[l1TableEntry, []l2TableEntry]
 	decompressor        Decompressor
@@ -585,6 +586,13 @@ func Open(ra io.ReaderAt, openWithType image.OpenWithType) (*Qcow2, error) {
 			}
 		}
 
+		// Used to get cluster metadata.
+		if img.extendedL2() {
+			img.l2Entries = img.clusterSize / 16
+		} else {
+			img.l2Entries = img.clusterSize / 8
+		}
+
 		// Load L1 table
 		img.l1Table, err = readL1Table(ra, img.Header.L1TableOffset, img.Header.L1Size)
 		if err != nil {
@@ -699,56 +707,96 @@ func (img *Qcow2) getL2Table(l1Entry l1TableEntry) ([]l2TableEntry, error) {
 	return l2Table, nil
 }
 
-// readAtAligned requires that off and off+len(p)-1 belong to the same cluster.
-func (img *Qcow2) readAtAligned(p []byte, off int64) (int, error) {
-	l2Entries := img.clusterSize / 8
-	if img.extendedL2() {
-		l2Entries = img.clusterSize / 16
-	}
-	l1Index := int((off / int64(img.clusterSize)) / int64(l2Entries))
-	if l1Index >= len(img.l1Table) {
-		return 0, fmt.Errorf("index %d exceeds the L1 table length %d", l1Index, img.l1Table)
+type clusterMeta struct {
+	// L1 info.
+	L1Index int
+	L1Entry l1TableEntry
+
+	// L2 info.
+	L2Index int
+	L2Entry l2TableEntry
+
+	// Extended L2 info.
+	ExtL2Entry extendedL2TableEntry
+
+	// Cluster is not allocated in this image, but it may be allocated in the
+	// backing file.
+	Allocated bool
+
+	// Cluster is present in this image and is compressed.
+	Compressed bool
+
+	// Cluster is present in this image and is all zeros.
+	Zero bool
+}
+
+func (img *Qcow2) getClusterMeta(off int64, cm *clusterMeta) error {
+	clusterNo := off / int64(img.clusterSize)
+	cm.L1Index = int(clusterNo / int64(img.l2Entries))
+	if cm.L1Index >= len(img.l1Table) {
+		return fmt.Errorf("index %d exceeds the L1 table length %d", cm.L1Index, len(img.l1Table))
 	}
-	l1Entry := img.l1Table[l1Index]
-	l2TableOffset := l1Entry.l2Offset()
+
+	cm.L1Entry = img.l1Table[cm.L1Index]
+	l2TableOffset := cm.L1Entry.l2Offset()
 	if l2TableOffset == 0 {
-		return img.readAtAlignedUnallocated(p, off)
+		return nil
 	}
-	l2Index := int((off / int64(img.clusterSize)) % int64(l2Entries))
-	var (
-		extL2Entry *extendedL2TableEntry
-		l2Entry    l2TableEntry
-	)
+
+	cm.L2Index = int(clusterNo % int64(img.l2Entries))
+
 	if img.extendedL2() {
 		// TODO
 		extL2Table, err := readExtendedL2Table(img.ra, l2TableOffset, img.clusterSize)
 		if err != nil {
-			return 0, fmt.Errorf("failed to read extended L2 table for L1 entry %v (index %d): %w", l1Entry, l1Index, err)
+			return fmt.Errorf("failed to read extended L2 table for L1 entry %v (index %d): %w", cm.L1Entry, cm.L1Index, err)
 		}
-		if l2Index >= len(extL2Table) {
-			return 0, fmt.Errorf("index %d exceeds the extended L2 table length %d", l2Index, extL2Table)
+		if cm.L2Index >= len(extL2Table) {
+			return fmt.Errorf("index %d exceeds the extended L2 table length %d", cm.L2Index, len(extL2Table))
 		}
-		extL2Entry = &extL2Table[l2Index]
-		l2Entry = extL2Entry.L2TableEntry
+		cm.ExtL2Entry = extL2Table[cm.L2Index]
+		cm.L2Entry = cm.ExtL2Entry.L2TableEntry
 	} else {
-		l2Table, err := img.getL2Table(l1Entry)
+		l2Table, err := img.getL2Table(cm.L1Entry)
 		if err != nil {
-			return 0, fmt.Errorf("failed to read L2 table for L1 entry %v (index %d): %w", l1Entry, l1Index, err)
+			return fmt.Errorf("failed to read L2 table for L1 entry %v (index %d): %w", cm.L1Entry, cm.L1Index, err)
 		}
-		if l2Index >= len(l2Table) {
-			return 0, fmt.Errorf("index %d exceeds the L2 table length %d", l2Index, l2Table)
+		if cm.L2Index >= len(l2Table) {
+			return fmt.Errorf("index %d exceeds the L2 table length %d", cm.L2Index, len(l2Table))
 		}
-		l2Entry = l2Table[l2Index]
+		cm.L2Entry = l2Table[cm.L2Index]
 	}
-	desc := l2Entry.clusterDescriptor()
+
+	desc := cm.L2Entry.clusterDescriptor()
 	if desc == 0 && !img.extendedL2() {
+		return nil
+	}
+
+	cm.Allocated = true
+	if cm.L2Entry.compressed() {
+		cm.Compressed = true
+	} else {
+		cm.Zero = standardClusterDescriptor(desc).allZero()
+	}
+
+	return nil
+}
+
+// readAtAligned requires that off and off+len(p)-1 belong to the same cluster.
+func (img *Qcow2) readAtAligned(p []byte, off int64) (int, error) {
+	var cm clusterMeta
+	if err := img.getClusterMeta(off, &cm); err != nil {
+		return 0, err
+	}
+	if !cm.Allocated {
 		return img.readAtAlignedUnallocated(p, off)
 	}
 	var (
 		n   int
 		err error
 	)
-	if l2Entry.compressed() {
+	desc := cm.L2Entry.clusterDescriptor()
+	if cm.Compressed {
 		compressedDesc := compressedClusterDescriptor(desc)
 		n, err = img.readAtAlignedCompressed(p, off, compressedDesc)
 		if err != nil {
@@ -757,7 +805,7 @@ func (img *Qcow2) readAtAligned(p []byte, off int64) (int, error) {
 	} else {
 		standardDesc := standardClusterDescriptor(desc)
 		if img.extendedL2() {
-			n, err = img.readAtAlignedStandardExtendedL2(p, off, standardDesc, *extL2Entry)
+			n, err = img.readAtAlignedStandardExtendedL2(p, off, standardDesc, cm.ExtL2Entry)
 			if err != nil {
 				err = fmt.Errorf("failed to read standard cluster with Extended L2 (len=%d, off=%d, desc=0x%X): %w", len(p), off, desc, err)
 			}

test/qemuimg/qemuimg.go

@@ -2,8 +2,9 @@ package qemuimg
 
 import (
 	"bytes"
-	"errors"
+	"fmt"
 	"os/exec"
+	"strconv"
 )
 
 type CompressionType string
@@ -26,18 +27,30 @@ func Convert(src, dst string, dstFormat Format, compressionType CompressionType)
 		args = append(args, "-c", "-o", "compression_type="+string(compressionType))
 	}
 	args = append(args, src, dst)
-	cmd := exec.Command("qemu-img", args...)
+	_, err := qemuImg(args)
+	return err
+}
 
+func Create(path string, format Format, size int64, backingFile string, backingFormat Format) error {
+	args := []string{"create", "-f", string(format)}
+	if backingFile != "" {
+		args = append(args, "-b", backingFile, "-F", string(backingFormat))
+	}
+	args = append(args, path, strconv.FormatInt(size, 10))
+	_, err := qemuImg(args)
+	return err
+}
+
+func qemuImg(args []string) ([]byte, error) {
+	cmd := exec.Command("qemu-img", args...)
 	var stderr bytes.Buffer
 	cmd.Stderr = &stderr
-
-	if err := cmd.Run(); err != nil {
-		// Return qemu-img stderr instead of the unhelpful default error (exited
-		// with status 1).
+	out, err := cmd.Output()
+	if err != nil {
 		if _, ok := err.(*exec.ExitError); ok {
-			return errors.New(stderr.String())
+			return out, fmt.Errorf("%w: stderr=%q", err, stderr.String())
 		}
-		return err
+		return out, err
 	}
-	return nil
+	return out, nil
 }

test/qemuio/qemuio.go

@@ -0,0 +1,52 @@
+package qemuio
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+
+	"github.com/lima-vm/go-qcow2reader/test/qemuimg"
+)
+
+// Write writes a number of bytes at a specified offset, allocating all clusters
+// in specified range.
+func Write(path string, format qemuimg.Format, off, len int64, pattern byte) error {
+	// qemu-io -f qcow2 -c 'write -P pattern off len' disk.qcow2
+	command := fmt.Sprintf("write -P %d %d %d", pattern, off, len)
+	_, err := qemuIo([]string{"-f", string(format), "-c", command, path})
+	return err
+}
+
+// Zero writes zeros at a specified offset. The behavior depens on qcow2
+// version; In qcow2 v3, allocate zero clusters, marming entire cluster as zero.
+// In qcow2 v2, if the cluster are unallocated and there is no backing file, do
+// nothing. Otherwise allocate clusters and write actual zeros.
+func Zero(path string, format qemuimg.Format, off, len int64) error {
+	// qemu-io -f qcow2 -c 'write -z off len' disk.qcow2
+	command := fmt.Sprintf("write -z %d %d", off, len)
+	_, err := qemuIo([]string{"-f", string(format), "-c", command, path})
+	return err
+}
+
+// Discard unmap number of bytes at specified offset. Allocated cluster are
+// deaallocated and replaced with zero clusters.
+func Discard(path string, format qemuimg.Format, off, len int64, unmap bool) error {
+	// qemu-io -f qcow2 -c 'write -zu off len' disk.qcow2
+	command := fmt.Sprintf("write -zu %d %d", off, len)
+	_, err := qemuIo([]string{"-f", string(format), "-c", command, path})
+	return err
+}
+
+func qemuIo(args []string) ([]byte, error) {
+	cmd := exec.Command("qemu-io", args...)
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	out, err := cmd.Output()
+	if err != nil {
+		if _, ok := err.(*exec.ExitError); ok {
+			return out, fmt.Errorf("%w: stderr=%q", err, stderr.String())
+		}
+		return out, err
+	}
+	return out, nil
+}