From 6fb3d49bc8ed6c41a22c2382dadf0f94858b3e17 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Fri, 25 Oct 2024 22:46:46 +0300 Subject: [PATCH 1/2] Update go-qcow2reader to v0.3.0 To consume the new convert package. Signed-off-by: Nir Soffer --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ce4c81ff44ef..3026a923f02d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/yamlfmt v0.13.0 github.com/invopop/jsonschema v0.12.0 - github.com/lima-vm/go-qcow2reader v0.2.1 + github.com/lima-vm/go-qcow2reader v0.3.0 github.com/lima-vm/sshocker v0.3.4 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-shellwords v1.0.12 diff --git a/go.sum b/go.sum index 650bf29295de..236b061b148d 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lima-vm/go-qcow2reader v0.2.1 h1:aeusQHn4m+uNhf3Z4oBUaE+xpMsNHjNCMkKUbhEgbXU= -github.com/lima-vm/go-qcow2reader v0.2.1/go.mod h1:e3p29BzLT8hNh4jbLezdFAHU4eBijf0bm5GilStCRKE= +github.com/lima-vm/go-qcow2reader v0.3.0 h1:7nDU29oXPNymm/UMMfqN8bYXejlhxEYIHO76RRUzbbc= +github.com/lima-vm/go-qcow2reader v0.3.0/go.mod h1:e3p29BzLT8hNh4jbLezdFAHU4eBijf0bm5GilStCRKE= github.com/lima-vm/sshocker v0.3.4 h1:5rn6vMkfqwZSZiBW+Udo505OIRhPB4xbLUDdEnFgWwI= github.com/lima-vm/sshocker v0.3.4/go.mod h1:QT4c7XNmeQTv79h5/8EgiS7U51B9BLenlXV7idCY0tE= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= From 0221c115332de50bb9f7d252a61498ddc5d72368 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Mon, 21 Oct 2024 02:27:16 +0300 Subject: [PATCH 2/2] Replace copySparse with go-qcow2reader/convert With this converting the default ubuntu 24.10 qcow2 compressed image to raw is 5.4 times faster: Before: % limactl create --tty=false 3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 317.58 MiB/s After: % limactl create --tty=false 3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 1.67 GiB/s Signed-off-by: Nir Soffer --- pkg/nativeimgutil/nativeimgutil.go | 50 +++++------------------------- pkg/progressbar/progressbar.go | 12 +++++++ 2 files changed, 20 insertions(+), 42 deletions(-) diff --git a/pkg/nativeimgutil/nativeimgutil.go b/pkg/nativeimgutil/nativeimgutil.go index 89d4963d0c4b..070786ae2a81 100644 --- a/pkg/nativeimgutil/nativeimgutil.go +++ b/pkg/nativeimgutil/nativeimgutil.go @@ -2,8 +2,6 @@ package nativeimgutil import ( - "bytes" - "errors" "fmt" "io" "os" @@ -12,6 +10,7 @@ import ( "github.com/containerd/continuity/fs" "github.com/docker/go-units" "github.com/lima-vm/go-qcow2reader" + "github.com/lima-vm/go-qcow2reader/convert" "github.com/lima-vm/go-qcow2reader/image/qcow2" "github.com/lima-vm/go-qcow2reader/image/raw" "github.com/lima-vm/lima/pkg/progressbar" @@ -75,17 +74,20 @@ func ConvertToRaw(source, dest string, size *int64, allowSourceWithBackingFile b } // Copy - srcImgR := io.NewSectionReader(srcImg, 0, srcImg.Size()) bar, err := progressbar.New(srcImg.Size()) if err != nil { return err } - const bufSize = 1024 * 1024 + conv, err := convert.New(convert.Options{}) + if err != nil { + return err + } bar.Start() - copied, err := copySparse(destTmpF, bar.NewProxyReader(srcImgR), bufSize) + pra := progressbar.ProxyReaderAt{ReaderAt: srcImg, Bar: bar} + err = conv.Convert(destTmpF, &pra, srcImg.Size()) bar.Finish() if err != nil { - return fmt.Errorf("failed to call copySparse(), bufSize=%d, copied=%d: %w", bufSize, copied, err) + return fmt.Errorf("failed to convert image: %w", err) } // Resize @@ -128,42 +130,6 @@ func convertRawToRaw(source, dest string, size *int64) error { return nil } -func copySparse(w *os.File, r io.Reader, bufSize int64) (int64, error) { - var ( - n int64 - eof bool - ) - - zeroBuf := make([]byte, bufSize) - buf := make([]byte, bufSize) - for !eof { - rN, rErr := r.Read(buf) - if rErr != nil { - eof = errors.Is(rErr, io.EOF) - if !eof { - return n, fmt.Errorf("failed to read: %w", rErr) - } - } - // TODO: qcow2reader should have a method to notify whether buf is zero - if bytes.Equal(buf[:rN], zeroBuf[:rN]) { - n += int64(rN) - } else { - wN, wErr := w.WriteAt(buf[:rN], n) - if wN > 0 { - n += int64(wN) - } - if wErr != nil { - return n, fmt.Errorf("failed to write: %w", wErr) - } - if wN != rN { - return n, fmt.Errorf("read %d, but wrote %d bytes", rN, wN) - } - } - } - - return n, nil -} - func MakeSparse(f *os.File, n int64) error { if _, err := f.Seek(n, io.SeekStart); err != nil { return err diff --git a/pkg/progressbar/progressbar.go b/pkg/progressbar/progressbar.go index 79b22ae76d09..ba33a248a01c 100644 --- a/pkg/progressbar/progressbar.go +++ b/pkg/progressbar/progressbar.go @@ -1,6 +1,7 @@ package progressbar import ( + "io" "os" "time" @@ -9,6 +10,17 @@ import ( "github.com/sirupsen/logrus" ) +type ProxyReaderAt struct { + io.ReaderAt + Bar *pb.ProgressBar +} + +func (r *ProxyReaderAt) ReadAt(p []byte, off int64) (int, error) { + n, err := r.ReaderAt.ReadAt(p, off) + r.Bar.Add(n) + return n, err +} + func New(size int64) (*pb.ProgressBar, error) { bar := pb.New64(size)