Skip to content

Commit

Permalink
Add convert sub command to example program
Browse files Browse the repository at this point in the history 
Add convert subcommand using the new convert package and include the new
command in the functional tests. The new example is also good way to
benchmark the library with real images as we can see bellow.

Testing show significant speedup for compressed images and for
unallocated or zero clusters, and smaller speedup for uncompressed
images.

| image        | size      | compression | throughput   | speedup |
|--------------|-----------|-------------|--------------|---------|
| Ubuntu 24.04 |   3.5 GiB | -           |   6.04 GiB/s |    1.51 |
| Ubuntu 24.04 |   3.5 GiB | zlib        |   1.62 GiB/s |    5.42 |
| Empty image  | 100.0 GiB | -           | 240.15 GiB/s |    7.32 |

Ubuntu 24.04 image in qcow2 format:

    % hyperfine -w3 "./go-qcow2reader-example read /tmp/images/test.qcow2 >/tmp/tmp.img" \
                    "./go-qcow2reader-example convert /tmp/images/test.qcow2 /tmp/tmp.img"
    Benchmark 1: ./go-qcow2reader-example read /tmp/images/test.qcow2 >/tmp/tmp.img
      Time (mean ± σ):     874.8 ms ±  41.3 ms    [User: 64.3 ms, System: 717.5 ms]
      Range (min … max):   851.9 ms … 985.3 ms    10 runs

    Benchmark 2: ./go-qcow2reader-example convert /tmp/images/test.qcow2 /tmp/tmp.img
      Time (mean ± σ):     579.4 ms ±  22.8 ms    [User: 90.5 ms, System: 681.2 ms]
      Range (min … max):   556.0 ms … 631.3 ms    10 runs

    Summary
      './go-qcow2reader-example convert /tmp/images/test.qcow2 /tmp/tmp.img' ran
        1.51 ± 0.09 times faster than './go-qcow2reader-example read /tmp/images/test.qcow2 >/tmp/tmp.img'

Ubuntu 24.04 image in qcow2 compressed format:

    % hyperfine -w3 -r3 "./go-qcow2reader-example read /tmp/images/test.zlib.qcow2 >/tmp/tmp.img" \
                        "./go-qcow2reader-example convert /tmp/images/test.zlib.qcow2 /tmp/tmp.img"
    Benchmark 1: ./go-qcow2reader-example read /tmp/images/test.zlib.qcow2 >/tmp/tmp.img
      Time (mean ± σ):     11.702 s ±  0.200 s    [User: 10.423 s, System: 1.121 s]
      Range (min … max):   11.533 s … 11.923 s    3 runs

    Benchmark 2: ./go-qcow2reader-example convert /tmp/images/test.zlib.qcow2 /tmp/tmp.img
      Time (mean ± σ):      2.161 s ±  0.027 s    [User: 10.980 s, System: 1.032 s]
      Range (min … max):    2.139 s …  2.191 s    3 runs

    Summary
      './go-qcow2reader-example convert /tmp/images/test.zlib.qcow2 /tmp/tmp.img' ran
        5.42 ± 0.11 times faster than './go-qcow2reader-example read /tmp/images/test.zlib.qcow2 >/tmp/tmp.img'

100 GiB empty sparse image in qcow2 format. Comparing to the read
command is not useful since it writes 100 GiB of zeros, so I'm comparing
1 and 8 workers.

    % hyperfine -w3 "./go-qcow2reader-example convert -workers 1 /tmp/images/test.0p.qcow2 /tmp/tmp.img" \
                    "./go-qcow2reader-example convert -workers 8 /tmp/images/test.0p.qcow2 /tmp/tmp.img"
    Benchmark 1: ./go-qcow2reader-example convert -workers 1 /tmp/images/test.0p.qcow2 /tmp/tmp.img
      Time (mean ± σ):      3.050 s ±  0.023 s    [User: 2.991 s, System: 0.054 s]
      Range (min … max):    3.036 s …  3.107 s    10 runs

    Benchmark 2: ./go-qcow2reader-example convert -workers 8 /tmp/images/test.0p.qcow2 /tmp/tmp.img
      Time (mean ± σ):     416.4 ms ±   3.4 ms    [User: 3252.7 ms, System: 16.3 ms]
      Range (min … max):   412.0 ms … 421.9 ms    10 runs

    Summary
      './go-qcow2reader-example convert -workers 8 /tmp/images/test.0p.qcow2 /tmp/tmp.img' ran
        7.32 ± 0.08 times faster than './go-qcow2reader-example convert -workers 1 /tmp/images/test.0p.qcow2 /tmp/tmp.img'

Signed-off-by: Nir Soffer <[email protected]>
  • Loading branch information
@nirs
nirs committed Oct 25, 2024
1 parent d597d74 commit 232960d
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 1 deletion.
88 changes: 88 additions & 0 deletions cmd/go-qcow2reader-example/convert.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package main

import (
"errors"
"flag"
"fmt"
"os"

"github.com/lima-vm/go-qcow2reader"
"github.com/lima-vm/go-qcow2reader/convert"
"github.com/lima-vm/go-qcow2reader/log"
)

func cmdConvert(args []string) error {
var (
// Required
source, target string

// Options
debug bool
options convert.Options
)

fs := flag.NewFlagSet("convert", flag.ExitOnError)
fs.Usage = func() {
fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s convert [OPTIONS...] SOURCE TARGET\n", os.Args[0])
flag.PrintDefaults()
}
fs.BoolVar(&debug, "debug", false, "enable printing debug messages")
fs.Int64Var(&options.SegmentSize, "segment-size", convert.SegmentSize, "worker segment size in bytes")
fs.IntVar(&options.BufferSize, "buffer-size", convert.BufferSize, "buffer size in bytes")
fs.IntVar(&options.Workers, "workers", convert.Workers, "number of workers")
if err := fs.Parse(args); err != nil {
return err
}

if debug {
log.SetDebugFunc(logDebug)
}

switch len(fs.Args()) {
case 0:
return errors.New("no file was specified")
case 1:
return errors.New("target file is required")
case 2:
source = fs.Arg(0)
target = fs.Arg(1)
default:
return errors.New("too many files were specified")
}

f, err := os.Open(source)
if err != nil {
return err
}
defer f.Close()

img, err := qcow2reader.Open(f)
if err != nil {
return err
}
defer img.Close()

t, err := os.Create(target)
if err != nil {
return err
}
defer t.Close()

if err := t.Truncate(img.Size()); err != nil {
return err
}

c, err := convert.New(options)
if err != nil {
return err
}
if err := c.Convert(t, img, img.Size()); err != nil {
return err
}

if err := t.Sync(); err != nil {
return err
}

return t.Close()
}
2 changes: 2 additions & 0 deletions cmd/go-qcow2reader-example/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ func main() {
err = cmdInfo(args)
case "read":
err = cmdRead(args)
case "convert":
err = cmdConvert(args)
default:
usage()
}
Expand Down
22 changes: 21 additions & 1 deletion hack/compare-with-qemu-img.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ fi
name_qcow2="$1"
name_raw_a="${name_qcow2}.raw_a"
name_raw_b="${name_qcow2}.raw_b"
name_raw_c="${name_qcow2}.raw_c"

echo "Input file: ${name_qcow2}"
set -x
Expand Down Expand Up @@ -35,7 +36,15 @@ go-qcow2reader-example read "${name_qcow2}" >"${name_raw_b}"
sha256sum "${name_raw_b}" | tee "${name_raw_b}.sha256"
set +x

rm -f "${name_raw_c}" "${name_raw_c}.sha256"
echo "Converting ${name_qcow2} to ${name_raw_c} with go-qcow2reader convert"
set -x
go-qcow2reader-example convert "${name_qcow2}" "${name_raw_c}"
sha256sum "${name_raw_c}" | tee "${name_raw_c}.sha256"
set +x

expected="$(cut -d " " -f 1 <"${name_raw_a}.sha256")"

got="$(cut -d " " -f 1 <"${name_raw_b}.sha256")"
echo "Comparing: ${expected} vs ${got}"
if [ "${expected}" = "${got}" ]; then
Expand All @@ -47,6 +56,17 @@ else
exit 1
fi

got="$(cut -d " " -f 1 <"${name_raw_c}.sha256")"
echo "Comparing: ${expected} vs ${got}"
if [ "${expected}" = "${got}" ]; then
echo "OK"
else
echo "FAIL"
set -x
qemu-img compare "${name_raw_a}" "${name_raw_c}"
exit 1
fi

echo "===== Phase 2: random read ====="
for offset in 1 22 333 4444 55555 666666 7777777 88888888; do
for length in 1 22 333 4444 55555 666666 7777777 88888888; do
Expand All @@ -68,5 +88,5 @@ done

echo "===== Cleaning up... ====="
set -x
rm -f "${name_raw_b}" "${name_raw_b}.sha256"
rm -f "${name_raw_b}" "${name_raw_b}.sha256" "${name_raw_c}" "${name_raw_c}.sha256"
set +x

0 comments on commit 232960d

Please sign in to comment.