Skip to content

Commit

Permalink
Replace copySparse with go-qcow2reader/convert
Browse files Browse the repository at this point in the history
With this converting the default ubuntu 24.10 qcow2 compressed image to
raw is 5.4 times faster:

Before:

    % limactl create --tty=false
    3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 317.58 MiB/s

After:

    % limactl create --tty=false
    3.50 GiB / 3.50 GiB [-------------------------------------] 100.00% 1.67 GiB/s

Signed-off-by: Nir Soffer <[email protected]>
  • Loading branch information
nirs committed Oct 25, 2024
1 parent 6fb3d49 commit 0221c11
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 42 deletions.
50 changes: 8 additions & 42 deletions pkg/nativeimgutil/nativeimgutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
package nativeimgutil

import (
"bytes"
"errors"
"fmt"
"io"
"os"
Expand All @@ -12,6 +10,7 @@ import (
"github.com/containerd/continuity/fs"
"github.com/docker/go-units"
"github.com/lima-vm/go-qcow2reader"
"github.com/lima-vm/go-qcow2reader/convert"
"github.com/lima-vm/go-qcow2reader/image/qcow2"
"github.com/lima-vm/go-qcow2reader/image/raw"
"github.com/lima-vm/lima/pkg/progressbar"
Expand Down Expand Up @@ -75,17 +74,20 @@ func ConvertToRaw(source, dest string, size *int64, allowSourceWithBackingFile b
}

// Copy
srcImgR := io.NewSectionReader(srcImg, 0, srcImg.Size())
bar, err := progressbar.New(srcImg.Size())
if err != nil {
return err
}
const bufSize = 1024 * 1024
conv, err := convert.New(convert.Options{})
if err != nil {
return err
}
bar.Start()
copied, err := copySparse(destTmpF, bar.NewProxyReader(srcImgR), bufSize)
pra := progressbar.ProxyReaderAt{ReaderAt: srcImg, Bar: bar}
err = conv.Convert(destTmpF, &pra, srcImg.Size())
bar.Finish()
if err != nil {
return fmt.Errorf("failed to call copySparse(), bufSize=%d, copied=%d: %w", bufSize, copied, err)
return fmt.Errorf("failed to convert image: %w", err)
}

// Resize
Expand Down Expand Up @@ -128,42 +130,6 @@ func convertRawToRaw(source, dest string, size *int64) error {
return nil
}

func copySparse(w *os.File, r io.Reader, bufSize int64) (int64, error) {
var (
n int64
eof bool
)

zeroBuf := make([]byte, bufSize)
buf := make([]byte, bufSize)
for !eof {
rN, rErr := r.Read(buf)
if rErr != nil {
eof = errors.Is(rErr, io.EOF)
if !eof {
return n, fmt.Errorf("failed to read: %w", rErr)
}
}
// TODO: qcow2reader should have a method to notify whether buf is zero
if bytes.Equal(buf[:rN], zeroBuf[:rN]) {
n += int64(rN)
} else {
wN, wErr := w.WriteAt(buf[:rN], n)
if wN > 0 {
n += int64(wN)
}
if wErr != nil {
return n, fmt.Errorf("failed to write: %w", wErr)
}
if wN != rN {
return n, fmt.Errorf("read %d, but wrote %d bytes", rN, wN)
}
}
}

return n, nil
}

func MakeSparse(f *os.File, n int64) error {
if _, err := f.Seek(n, io.SeekStart); err != nil {
return err
Expand Down
12 changes: 12 additions & 0 deletions pkg/progressbar/progressbar.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package progressbar

import (
"io"
"os"
"time"

Expand All @@ -9,6 +10,17 @@ import (
"github.com/sirupsen/logrus"
)

type ProxyReaderAt struct {
io.ReaderAt
Bar *pb.ProgressBar
}

func (r *ProxyReaderAt) ReadAt(p []byte, off int64) (int, error) {
n, err := r.ReaderAt.ReadAt(p, off)
r.Bar.Add(n)
return n, err
}

func New(size int64) (*pb.ProgressBar, error) {
bar := pb.New64(size)

Expand Down

0 comments on commit 0221c11

Please sign in to comment.