WIP: Replace copySparse with go-qcow2reader/convert

With this converting the default ubuntu 24.10 qcow2 compressed image to
raw is 5.4 times faster:

Before:

    % limactl create --tty=false
    3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 317.58 MiB/s

After:

    % limactl create --tty=false
    3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 1.67 GiB/s

Depends on lima-vm/go-qcow2reader#36

Signed-off-by: Nir Soffer <[email protected]>

go.mod

@@ -142,3 +142,6 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+// Temporary raplcement for testing https://github.com/lima-vm/go-qcow2reader/pull/36
+replace github.com/lima-vm/go-qcow2reader v0.2.1 => github.com/nirs/go-qcow2reader v0.0.0-20241025131821-46183d3fe6b2

go.sum

@@ -175,8 +175,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/lima-vm/go-qcow2reader v0.2.1 h1:aeusQHn4m+uNhf3Z4oBUaE+xpMsNHjNCMkKUbhEgbXU=
-github.com/lima-vm/go-qcow2reader v0.2.1/go.mod h1:e3p29BzLT8hNh4jbLezdFAHU4eBijf0bm5GilStCRKE=
 github.com/lima-vm/sshocker v0.3.4 h1:5rn6vMkfqwZSZiBW+Udo505OIRhPB4xbLUDdEnFgWwI=
 github.com/lima-vm/sshocker v0.3.4/go.mod h1:QT4c7XNmeQTv79h5/8EgiS7U51B9BLenlXV7idCY0tE=
 github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y=
@@ -218,6 +216,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nirs/go-qcow2reader v0.0.0-20241025131821-46183d3fe6b2 h1:7is5ySe7HvLqcYOCd+v6m9COvzAqKX+y636oSXl5+XQ=
+github.com/nirs/go-qcow2reader v0.0.0-20241025131821-46183d3fe6b2/go.mod h1:e3p29BzLT8hNh4jbLezdFAHU4eBijf0bm5GilStCRKE=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=

pkg/nativeimgutil/nativeimgutil.go

@@ -2,8 +2,6 @@
 package nativeimgutil
 
 import (
-	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -12,6 +10,7 @@ import (
 	"github.com/containerd/continuity/fs"
 	"github.com/docker/go-units"
 	"github.com/lima-vm/go-qcow2reader"
+	"github.com/lima-vm/go-qcow2reader/convert"
 	"github.com/lima-vm/go-qcow2reader/image/qcow2"
 	"github.com/lima-vm/go-qcow2reader/image/raw"
 	"github.com/lima-vm/lima/pkg/progressbar"
@@ -75,17 +74,20 @@ func ConvertToRaw(source, dest string, size *int64, allowSourceWithBackingFile b
 	}
 
 	// Copy
-	srcImgR := io.NewSectionReader(srcImg, 0, srcImg.Size())
 	bar, err := progressbar.New(srcImg.Size())
 	if err != nil {
 		return err
 	}
-	const bufSize = 1024 * 1024
+	conv, err := convert.New(convert.Options{})
+	if err != nil {
+		return err
+	}
 	bar.Start()
-	copied, err := copySparse(destTmpF, bar.NewProxyReader(srcImgR), bufSize)
+	pra := progressbar.ProxyReaderAt{ReaderAt: srcImg, Bar: bar}
+	err = conv.Convert(destTmpF, &pra, srcImg.Size())
 	bar.Finish()
 	if err != nil {
-		return fmt.Errorf("failed to call copySparse(), bufSize=%d, copied=%d: %w", bufSize, copied, err)
+		return fmt.Errorf("failed to convert image: %w", err)
 	}
 
 	// Resize
@@ -128,42 +130,6 @@ func convertRawToRaw(source, dest string, size *int64) error {
 	return nil
 }
 
-func copySparse(w *os.File, r io.Reader, bufSize int64) (int64, error) {
-	var (
-		n   int64
-		eof bool
-	)
-
-	zeroBuf := make([]byte, bufSize)
-	buf := make([]byte, bufSize)
-	for !eof {
-		rN, rErr := r.Read(buf)
-		if rErr != nil {
-			eof = errors.Is(rErr, io.EOF)
-			if !eof {
-				return n, fmt.Errorf("failed to read: %w", rErr)
-			}
-		}
-		// TODO: qcow2reader should have a method to notify whether buf is zero
-		if bytes.Equal(buf[:rN], zeroBuf[:rN]) {
-			n += int64(rN)
-		} else {
-			wN, wErr := w.WriteAt(buf[:rN], n)
-			if wN > 0 {
-				n += int64(wN)
-			}
-			if wErr != nil {
-				return n, fmt.Errorf("failed to write: %w", wErr)
-			}
-			if wN != rN {
-				return n, fmt.Errorf("read %d, but wrote %d bytes", rN, wN)
-			}
-		}
-	}
-
-	return n, nil
-}
-
 func MakeSparse(f *os.File, n int64) error {
 	if _, err := f.Seek(n, io.SeekStart); err != nil {
 		return err

pkg/progressbar/progressbar.go

@@ -1,6 +1,7 @@
 package progressbar
 
 import (
+	"io"
 	"os"
 	"time"
 
@@ -9,6 +10,17 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+type ProxyReaderAt struct {
+	io.ReaderAt
+	Bar *pb.ProgressBar
+}
+
+func (r *ProxyReaderAt) ReadAt(p []byte, off int64) (int, error) {
+	n, err := r.ReaderAt.ReadAt(p, off)
+	r.Bar.Add(n)
+	return n, err
+}
+
 func New(size int64) (*pb.ProgressBar, error) {
 	bar := pb.New64(size)