diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b1e8c5d..1b815ae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] name: build runs-on: ${{ matrix.os }} steps: @@ -20,7 +20,7 @@ jobs: sudo apt-get update sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \ libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \ - squashfuse squashfs-tools + squashfuse squashfs-tools erofs-utils - name: setup lxc run: | chmod ugo+x $HOME diff --git a/erofs/erofs.go b/erofs/erofs.go new file mode 100644 index 0000000..af79833 --- /dev/null +++ b/erofs/erofs.go @@ -0,0 +1,743 @@ +// This package is a small go "library" (read: exec wrapper) around the +// mkfs.erofs binary that provides some useful primitives. +package erofs + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/log" + "machinerun.io/atomfs/mount" +) + +var checkZstdSupported sync.Once +var zstdIsSuspported bool + +var exPolInfo struct { + once sync.Once + err error + policy *ExtractPolicy +} + +// ExcludePaths represents a list of paths to exclude in a erofs listing. +// Users should do something like filepath.Walk() over the whole filesystem, +// calling AddExclude() or AddInclude() based on whether they want to include +// or exclude a particular file. Note that if e.g. /usr is excluded, then +// everyting underneath is also implicitly excluded. The +// AddExclude()/AddInclude() methods do the math to figure out what is the +// correct set of things to exclude or include based on what paths have been +// previously included or excluded. +type ExcludePaths struct { + exclude map[string]bool + include []string +} + +type erofsFuseInfoStruct struct { + Path string + Version string + SupportsNotfiy bool +} + +var once sync.Once +var erofsFuseInfo = erofsFuseInfoStruct{"", "", false} + +func NewExcludePaths() *ExcludePaths { + return &ExcludePaths{ + exclude: map[string]bool{}, + include: []string{}, + } +} + +func (eps *ExcludePaths) AddExclude(p string) { + for _, inc := range eps.include { + // If /usr/bin/ls has changed but /usr hasn't, we don't want to list + // /usr in the include paths any more, so let's be sure to only + // add things which aren't prefixes. + if strings.HasPrefix(inc, p) { + return + } + } + eps.exclude[p] = true +} + +func (eps *ExcludePaths) AddInclude(orig string, isDir bool) { + // First, remove this thing and all its parents from exclude. + p := orig + + // normalize to the first dir + if !isDir { + p = path.Dir(p) + } + for { + // our paths are all absolute, so this is a base case + if p == "/" { + break + } + + delete(eps.exclude, p) + p = filepath.Dir(p) + } + + // now add it to the list of includes, so we don't accidentally re-add + // anything above. + eps.include = append(eps.include, orig) +} + +func (eps *ExcludePaths) String() (string, error) { + var buf bytes.Buffer + for p := range eps.exclude { + _, err := buf.WriteString(p) + if err != nil { + return "", err + } + _, err = buf.WriteString("\n") + if err != nil { + return "", err + } + } + + _, err := buf.WriteString("\n") + if err != nil { + return "", err + } + + return buf.String(), nil +} + +func MakeErofs(tempdir string, rootfs string, eps *ExcludePaths, verity VerityMetadata) (io.ReadCloser, string, string, error) { + var excludesFile string + var err error + var toExclude string + var rootHash string + + if eps != nil { + toExclude, err = eps.String() + if err != nil { + return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list") + } + } + + if len(toExclude) != 0 { + excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-") + if err != nil { + return nil, "", rootHash, err + } + defer os.Remove(excludes.Name()) + + excludesFile = excludes.Name() + _, err = excludes.WriteString(toExclude) + excludes.Close() + if err != nil { + return nil, "", rootHash, err + } + } + + tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-") + if err != nil { + return nil, "", rootHash, err + } + tmpErofs.Close() + os.Remove(tmpErofs.Name()) + defer os.Remove(tmpErofs.Name()) + args := []string{tmpErofs.Name(), rootfs} + compression := GzipCompression + if mkerofsSupportsZstd() { + args = append(args, "-z", "zstd") + compression = ZstdCompression + } + if len(toExclude) != 0 { + args = append(args, "--exclude-path", excludesFile) + } + cmd := exec.Command("mkfs.erofs", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err = cmd.Run(); err != nil { + return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs") + } + + if verity { + rootHash, err = appendVerityData(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, err + } + } + + blob, err := os.Open(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, errors.WithStack(err) + } + + return blob, GenerateErofsMediaType(compression, verity), rootHash, nil +} +func isMountedAtDir(src, dest string) (bool, error) { + dstat, err := os.Stat(dest) + if os.IsNotExist(err) { + return false, nil + } + if !dstat.IsDir() { + return false, nil + } + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return false, err + } + + fdest, err := filepath.Abs(dest) + if err != nil { + return false, err + } + for _, m := range mounts { + if m.Target == fdest { + return true, nil + } + } + + return false, nil +} + +func findErofsFuseInfo() { + var erofsPath string + if p := which("erofsfuse"); p != "" { + erofsPath = p + } else { + erofsPath = which("erofsfuse") + } + if erofsPath == "" { + return + } + version, supportsNotify := erofsfuseSupportsMountNotification(erofsPath) + log.Infof("Found erofsfuse at %s (version=%s notify=%t)", erofsPath, version, supportsNotify) + erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version, supportsNotify} +} + +// erofsfuseSupportsMountNotification - returns true if erofsfuse supports mount +// notification, false otherwise +// erofsfuse is the path to the erofsfuse binary +func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { + cmd := exec.Command(erofsfuse) + + // `erofsfuse` always returns an error... so we ignore it. + out, _ := cmd.CombinedOutput() + + firstLine := strings.Split(string(out[:]), "\n")[0] + version := strings.Split(firstLine, " ")[1] + v, err := semver.NewVersion(version) + if err != nil { + return version, false + } + // erofsfuse notify mechanism was merged in 0.5.0 + constraint, err := semver.NewConstraint(">= 0.5.0") + if err != nil { + return version, false + } + if constraint.Check(v) { + return version, true + } + return version, false +} + +var squashNotFound = errors.Errorf("erofsfuse program not found") + +// erofsFuse - mount squashFile to extractDir +// return a pointer to the erofsfuse cmd. +// The caller of the this is responsible for the process created. +func erofsFuse(squashFile, extractDir string) (*exec.Cmd, error) { + var cmd *exec.Cmd + + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return cmd, squashNotFound + } + + notifyOpts := "" + notifyPath := "" + if erofsFuseInfo.SupportsNotfiy { + sockdir, err := os.MkdirTemp("", "sock") + if err != nil { + return cmd, err + } + defer os.RemoveAll(sockdir) + notifyPath = filepath.Join(sockdir, "notifypipe") + if err := syscall.Mkfifo(notifyPath, 0640); err != nil { + return cmd, err + } + notifyOpts = "notify_pipe=" + notifyPath + } + + // given extractDir of path/to/some/dir[/], log to path/to/some/.dir-squashfs.log + extractDir = strings.TrimSuffix(extractDir, "/") + + var cmdOut io.Writer + var err error + + logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-erofsfuse.log") + if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { + log.Infof("Failed to open %s for write: %v", logf, err) + return cmd, err + } + + fiPre, err := os.Lstat(extractDir) + if err != nil { + return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) + } + if fiPre.Mode()&os.ModeSymlink != 0 { + return cmd, errors.Errorf("Refusing to mount onto a symbolic linkd") + } + + // It would be nice to only enable debug (or maybe to only log to file at all) + // if 'stacker --debug', but we do not have access to that info here. + // to debug erofsfuse, use "allow_other,debug" + optionArgs := "allow_other,debug" + if notifyOpts != "" { + optionArgs += "," + notifyOpts + } + cmd = exec.Command(erofsFuseInfo.Path, "-f", "-o", optionArgs, squashFile, extractDir) + cmd.Stdin = nil + cmd.Stdout = cmdOut + cmd.Stderr = cmdOut + cmdOut.Write([]byte(fmt.Sprintf("# %s\n", strings.Join(cmd.Args, " ")))) + log.Debugf("Extracting %s -> %s with %s [%s]", squashFile, extractDir, erofsFuseInfo.Path, logf) + err = cmd.Start() + if err != nil { + return cmd, err + } + + // now poll/wait for one of 3 things to happen + // a. child process exits - if it did, then some error has occurred. + // b. the directory Entry is different than it was before the call + // to erofsfuse. We have to do this because we do not have another + // way to know when the mount has been populated. + // https://github.com/vasi/erofsfuse/issues/49 + // c. a timeout (timeLimit) was hit + startTime := time.Now() + timeLimit := 30 * time.Second + alarmCh := make(chan struct{}) + go func() { + cmd.Wait() + close(alarmCh) + }() + if erofsFuseInfo.SupportsNotfiy { + notifyCh := make(chan byte) + log.Infof("%s supports notify pipe, watching %q", erofsFuseInfo.Path, notifyPath) + go func() { + f, err := os.Open(notifyPath) + if err != nil { + return + } + defer f.Close() + b1 := make([]byte, 1) + for { + n1, err := f.Read(b1) + if err != nil { + return + } + if err == nil && n1 >= 1 { + break + } + } + notifyCh <- b1[0] + }() + if err != nil { + return cmd, errors.Wrapf(err, "Failed reading %q", notifyPath) + } + + select { + case <-alarmCh: + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", squashFile, erofsFuseInfo.Path, timeLimit) + case ret := <-notifyCh: + if ret == 's' { + return cmd, nil + } else { + return cmd, errors.Errorf("erofsfuse returned an error, check %s", logf) + } + } + } + for count := 0; !fileChanged(fiPre, extractDir); count++ { + if cmd.ProcessState != nil { + // process exited, the Wait() call in the goroutine above + // caused ProcessState to be populated. + return cmd, errors.Errorf("erofsfuse mount of %s with %s exited unexpectedly with %d", squashFile, erofsFuseInfo.Path, cmd.ProcessState.ExitCode()) + } + if time.Since(startTime) > timeLimit { + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", squashFile, erofsFuseInfo.Path, timeLimit) + } + if count%10 == 1 { + log.Debugf("%s is not yet mounted...(%s)", extractDir, time.Since(startTime)) + } + time.Sleep(time.Duration(50 * time.Millisecond)) + } + + return cmd, nil +} + +type ExtractPolicy struct { + Extractors []SquashExtractor + Extractor SquashExtractor + Excuses map[string]error + initialized bool + mutex sync.Mutex +} + +type SquashExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} + +func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { + p := &ExtractPolicy{ + Extractors: []SquashExtractor{}, + Excuses: map[string]error{}, + } + + allEx := []SquashExtractor{ + &KernelExtractor{}, + &ErofsFuseExtractor{}, + &UnsquashfsExtractor{}, + } + byName := map[string]SquashExtractor{} + for _, i := range allEx { + byName[i.Name()] = i + } + + for _, i := range args { + extractor, ok := byName[i] + if !ok { + return nil, errors.Errorf("Unknown extractor: '%s'", i) + } + excuse := extractor.IsAvailable() + if excuse != nil { + p.Excuses[i] = excuse + continue + } + p.Extractors = append(p.Extractors, extractor) + } + return p, nil +} + +type UnsquashfsExtractor struct { + mutex sync.Mutex +} + +func (k *UnsquashfsExtractor) Name() string { + return "unsquashfs" +} + +func (k *UnsquashfsExtractor) IsAvailable() error { + if which("unsquashfs") == "" { + return errors.Errorf("no 'unsquashfs' in PATH") + } + return nil +} + +func (k *UnsquashfsExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + // check if already extracted + empty, err := isEmptyDir(extractDir) + if err != nil { + return errors.Wrapf(err, "Error checking for empty dir") + } + if !empty { + return nil + } + + log.Debugf("unsquashfs %s -> %s", squashFile, extractDir) + cmd := exec.Command("unsquashfs", "-f", "-d", extractDir, squashFile) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = nil + err = cmd.Run() + + // on failure, remove the directory + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, squashFile, rmErr) + } + return err + } + + // assert that extraction must create files. This way we can assume non-empty dir above + // was populated by unsquashfs. + empty, err = isEmptyDir(extractDir) + if err != nil { + return errors.Errorf("Failed to read %s after successful extraction of %s: %v", + extractDir, squashFile, err) + } + if empty { + return errors.Errorf("%s was an empty fs image", squashFile) + } + + return nil +} + +type KernelExtractor struct { + mutex sync.Mutex +} + +func (k *KernelExtractor) Name() string { + return "kmount" +} + +func (k *KernelExtractor) IsAvailable() error { + if !amHostRoot() { + return errors.Errorf("not host root") + } + return nil +} + +func (k *KernelExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(squashFile, extractDir); err != nil { + return err + } else if mounted { + return nil + } + + ecmd := []string{"mount", "-tsquashfs", "-oloop,ro", squashFile, extractDir} + var output bytes.Buffer + cmd := exec.Command(ecmd[0], ecmd[1:]...) + cmd.Stdin = nil + cmd.Stdout = &output + cmd.Stderr = cmd.Stdout + err := cmd.Run() + if err == nil { + return nil + } + + var retErr error + + exitError, ok := err.(*exec.ExitError) + if !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", + squashFile, ecmd, err) + } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", + squashFile, ecmd, err) + } else { + retErr = errors.Errorf("kmount(%s) exited %d: %v", squashFile, status.ExitStatus(), output.String()) + } + + return retErr +} + +type ErofsFuseExtractor struct { + mutex sync.Mutex +} + +func (k *ErofsFuseExtractor) Name() string { + return "erofsfuse" +} + +func (k *ErofsFuseExtractor) IsAvailable() error { + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return errors.Errorf("no 'erofsfuse' in PATH") + } + return nil +} + +func (k *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(erofsFile, extractDir); mounted && err == nil { + log.Debugf("[%s] %s already mounted -> %s", k.Name(), erofsFile, extractDir) + return nil + } else if err != nil { + return err + } + + cmd, err := erofsFuse(erofsFile, extractDir) + if err != nil { + return err + } + + log.Debugf("erofsfuse mounted (%d) %s -> %s", cmd.Process.Pid, erofsFile, extractDir) + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process %s: %v", cmd, err) + } + return nil +} + +// ExtractSingleSquashPolicy - extract squashfile to extractDir +func ExtractSingleSquashPolicy(squashFile, extractDir string, policy *ExtractPolicy) error { + const initName = "init" + if policy == nil { + return errors.Errorf("policy cannot be nil") + } + + // avoid taking a lock if already initialized (possibly premature optimization) + if !policy.initialized { + policy.mutex.Lock() + // We may have been waiting on the initializer. If so, then the policy will now be initialized. + // if not, then we are the initializer. + if !policy.initialized { + defer policy.mutex.Unlock() + defer func() { + policy.initialized = true + }() + } else { + policy.mutex.Unlock() + } + } + + err := os.MkdirAll(extractDir, 0755) + if err != nil { + return err + } + + fdest, err := filepath.Abs(extractDir) + if err != nil { + return err + } + + if policy.initialized { + if err, ok := policy.Excuses[initName]; ok { + return err + } + return policy.Extractor.Mount(squashFile, fdest) + } + + // At this point we are the initialzer + if policy.Excuses == nil { + policy.Excuses = map[string]error{} + } + + if len(policy.Extractors) == 0 { + policy.Excuses[initName] = errors.Errorf("policy had no extractors") + return policy.Excuses[initName] + } + + var extractor SquashExtractor + allExcuses := []string{} + for _, extractor = range policy.Extractors { + err = extractor.Mount(squashFile, fdest) + if err == nil { + policy.Extractor = extractor + log.Debugf("Selected squashfs extractor %s", extractor.Name()) + return nil + } + policy.Excuses[extractor.Name()] = err + } + + for n, exc := range policy.Excuses { + allExcuses = append(allExcuses, fmt.Sprintf("%s: %v", n, exc)) + } + + // nothing worked. populate Excuses[initName] + policy.Excuses[initName] = errors.Errorf("No suitable extractor found:\n %s", strings.Join(allExcuses, "\n ")) + return policy.Excuses[initName] +} + +// ExtractSingleSquash - extract the squashFile to extractDir +// Initialize a extractPolicy struct and then call ExtractSingleSquashPolicy +// wik()th that. +func ExtractSingleSquash(squashFile string, extractDir string) error { + exPolInfo.once.Do(func() { + const envName = "STACKER_SQUASHFS_EXTRACT_POLICY" + const defPolicy = "kmount erofsfuse unsquashfs" + val := os.Getenv(envName) + if val == "" { + val = defPolicy + } + exPolInfo.policy, exPolInfo.err = NewExtractPolicy(strings.Fields(val)...) + if exPolInfo.err == nil { + for k, v := range exPolInfo.policy.Excuses { + log.Debugf(" squashfs extractor %s is not available: %v", k, v) + } + } + }) + + if exPolInfo.err != nil { + return exPolInfo.err + } + + return ExtractSingleSquashPolicy(squashFile, extractDir, exPolInfo.policy) +} + +func mkerofsSupportsZstd() bool { + checkZstdSupported.Do(func() { + var stdoutBuffer strings.Builder + var stderrBuffer strings.Builder + + cmd := exec.Command("mkfs.erofs", "--help") + cmd.Stdout = &stdoutBuffer + cmd.Stderr = &stderrBuffer + + // Ignore errs here as `mkerofs --help` exit status code is 1 + _ = cmd.Run() + + if strings.Contains(stdoutBuffer.String(), "zstd") || + strings.Contains(stderrBuffer.String(), "zstd") { + zstdIsSuspported = true + } + }) + + return zstdIsSuspported +} + +func isEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// which - like the unix utility, return empty string for not-found. +// this might fit well in lib/, but currently lib's test imports +// squashfs creating a import loop. +func which(name string) string { + return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) +} + +func whichSearch(name string, paths []string) string { + var search []string + + if strings.ContainsRune(name, os.PathSeparator) { + if filepath.IsAbs(name) { + search = []string{name} + } else { + search = []string{"./" + name} + } + } else { + search = []string{} + for _, p := range paths { + search = append(search, filepath.Join(p, name)) + } + } + + for _, fPath := range search { + if err := unix.Access(fPath, unix.X_OK); err == nil { + return fPath + } + } + + return "" +} diff --git a/erofs/mediatype.go b/erofs/mediatype.go new file mode 100644 index 0000000..6726ea1 --- /dev/null +++ b/erofs/mediatype.go @@ -0,0 +1,37 @@ +package erofs + +import ( + "fmt" + "strings" +) + +type ErofsCompression string +type VerityMetadata bool + +const ( + BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs" + + GzipCompression ErofsCompression = "gzip" + ZstdCompression ErofsCompression = "zstd" + + veritySuffix = "verity" + + VerityMetadataPresent VerityMetadata = true + VerityMetadataMissing VerityMetadata = false +) + +func IsErofsMediaType(mediaType string) bool { + return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs) +} + +func GenerateErofsMediaType(comp ErofsCompression, verity VerityMetadata) string { + verityString := "" + if verity { + verityString = fmt.Sprintf("+%s", veritySuffix) + } + return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString) +} + +func HasVerityMetadata(mediaType string) VerityMetadata { + return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix)) +} diff --git a/erofs/superblock.go b/erofs/superblock.go new file mode 100644 index 0000000..094f68c --- /dev/null +++ b/erofs/superblock.go @@ -0,0 +1,247 @@ +package erofs + +import ( + "encoding/binary" + "io" + "os" + + "github.com/pkg/errors" +) + +/* + +https://docs.kernel.org/filesystems/erofs.html + +On-disk details + + |-> aligned with the block size + ____________________________________________________________ +| |SB| | ... | Metadata | ... | Data | Metadata | ... | Data | +|_|__|_|_____|__________|_____|______|__________|_____|______| +0 +1K + +*/ + +const ( + // Definitions for superblock. + superblockMagicV1 = 0xe0f5e1e2 + superblockMagic = superblockMagicV1 + superblockOffset = 1024 + + // Inode slot size in bit shift. + InodeSlotBits = 5 + + // Max file name length. + MaxNameLen = 255 +) + +// Bit definitions for Inode*::Format. +const ( + InodeLayoutBit = 0 + InodeLayoutBits = 1 + + InodeDataLayoutBit = 1 + InodeDataLayoutBits = 3 +) + +// Inode layouts. +const ( + InodeLayoutCompact = 0 + InodeLayoutExtended = 1 +) + +// Inode data layouts. +const ( + InodeDataLayoutFlatPlain = iota + InodeDataLayoutFlatCompressionLegacy + InodeDataLayoutFlatInline + InodeDataLayoutFlatCompression + InodeDataLayoutChunkBased + InodeDataLayoutMax +) + +// Features w/ backward compatibility. +// This is not exhaustive, unused features are not listed. +const ( + FeatureCompatSuperBlockChecksum = 0x00000001 +) + +// Features w/o backward compatibility. +// +// Any features that aren't in FeatureIncompatSupported are incompatible +// with this implementation. +// +// This is not exhaustive, unused features are not listed. +const ( + FeatureIncompatSupported = 0x0 +) + +// Sizes of on-disk structures in bytes. +const ( + superblockSize = 128 + InodeCompactSize = 32 + InodeExtendedSize = 64 + DirentSize = 12 +) + +type superblock struct { + Magic uint32 + Checksum uint32 + FeatureCompat uint32 + BlockSizeBits uint8 + ExtSlots uint8 + RootNid uint16 + Inodes uint64 + BuildTime uint64 + BuildTimeNsec uint32 + Blocks uint32 + MetaBlockAddr uint32 + XattrBlockAddr uint32 + UUID [16]uint8 + VolumeName [16]uint8 + FeatureIncompat uint32 + Union1 uint16 + ExtraDevices uint16 + DevTableSlotOff uint16 + Reserved [38]uint8 +} + +/* +// checkRange checks whether the range [off, off+n) is valid. +func (i *Image) checkRange(off, n uint64) bool { + size := uint64(len(i.bytes)) + end := off + n + return off < size && off <= end && end <= size +} + +// BytesAt returns the bytes at [off, off+n) of the image. +func (i *Image) BytesAt(off, n uint64) ([]byte, error) { + if ok := i.checkRange(off, n); !ok { + //log.Warningf("Invalid byte range (off: 0x%x, n: 0x%x) for image (size: 0x%x)", off, n, len(i.bytes)) + return nil, linuxerr.EFAULT + } + return i.bytes[off : off+n], nil +} + +// unmarshalAt deserializes data from the bytes at [off, off+n) of the image. +func (i *Image) unmarshalAt(data marshal.Marshallable, off uint64) error { + bytes, err := i.BytesAt(off, uint64(data.SizeBytes())) + if err != nil { + //log.Warningf("Failed to deserialize %T from 0x%x.", data, off) + return err + } + data.UnmarshalUnsafe(bytes) + return nil +} + +// initSuperBlock initializes the superblock of this image. +func (i *Image) initSuperBlock() error { + // i.sb is used in the hot path. Let's save a copy of the superblock. + if err := i.unmarshalAt(&i.sb, SuperBlockOffset); err != nil { + return fmt.Errorf("image size is too small") + } + + if i.sb.Magic != SuperBlockMagicV1 { + return fmt.Errorf("unknown magic: 0x%x", i.sb.Magic) + } + + if err := i.verifyChecksum(); err != nil { + return err + } + + if featureIncompat := i.sb.FeatureIncompat & ^uint32(FeatureIncompatSupported); featureIncompat != 0 { + return fmt.Errorf("unsupported incompatible features detected: 0x%x", featureIncompat) + } + + if i.BlockSize()%hostarch.PageSize != 0 { + return fmt.Errorf("unsupported block size: 0x%x", i.BlockSize()) + } + + return nil +} + +// verifyChecksum verifies the checksum of the superblock. +func (i *Image) verifyChecksum() error { + if i.sb.FeatureCompat&FeatureCompatSuperBlockChecksum == 0 { + return nil + } + + sb := i.sb + sb.Checksum = 0 + table := crc32.MakeTable(crc32.Castagnoli) + checksum := crc32.Checksum(marshal.Marshal(&sb), table) +// unmarshalAt deserializes data from the bytes at [off, off+n) of the image. +func (i *Image) unmarshalAt(data marshal.Marshallable, off uint64) error { + bytes, err := i.BytesAt(off, uint64(data.SizeBytes())) + if err != nil { + log.Warningf("Failed to deserialize %T from 0x%x.", data, off) + return err + } + data.UnmarshalUnsafe(bytes) + return nil +} + off := SuperBlockOffset + uint64(i.sb.SizeBytes()) + if bytes, err := i.BytesAt(off, uint64(i.BlockSize())-off); err != nil { + return fmt.Errorf("image size is too small") + } else { + checksum = ^crc32.Update(checksum, table, bytes) + } + if checksum != i.sb.Checksum { + return fmt.Errorf("invalid checksum: 0x%x, expected: 0x%x", checksum, i.sb.Checksum) + } + + return nil +} +*/ + +func parseSuperblock(b []byte) (*superblock, error) { + if len(b) != superblockSize { + return nil, errors.Errorf("superblock had %d bytes instead of expected %d", len(b), superblockSize) + } + + magic := binary.LittleEndian.Uint32(b[0:4]) + if magic != superblockMagic { + return nil, errors.Errorf("superblock had magic of %d instead of expected %d", magic, superblockMagic) + } + + // FIXME: also verify checksum + + s := &superblock{ + Magic: magic, // b[0:4] + Checksum: binary.LittleEndian.Uint32(b[4:8]), + FeatureCompat: binary.LittleEndian.Uint32(b[8:12]), + BlockSizeBits: b[12], // b[12:13] + ExtSlots: b[13], // b[13:14] + RootNid: binary.LittleEndian.Uint16(b[14:16]), + Inodes: binary.LittleEndian.Uint64(b[16:24]), + BuildTime: binary.LittleEndian.Uint64(b[24:32]), + BuildTimeNsec: binary.LittleEndian.Uint32(b[32:36]), + Blocks: binary.LittleEndian.Uint32(b[36:40]), + MetaBlockAddr: binary.LittleEndian.Uint32(b[40:44]), + XattrBlockAddr: binary.LittleEndian.Uint32(b[44:48]), + UUID: [16]byte(b[48:64]), + VolumeName: [16]byte(b[64:80]), + FeatureIncompat: binary.LittleEndian.Uint32(b[80:84]), + Union1: binary.LittleEndian.Uint16(b[84:86]), + ExtraDevices: binary.LittleEndian.Uint16(b[86:88]), + DevTableSlotOff: binary.LittleEndian.Uint16(b[88:90]), + Reserved: [38]byte(b[90:128]), + } + + return s, nil +} + +func readSuperblock(path string) (*superblock, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + defer reader.Close() + + buf := make([]byte, superblockOffset+superblockSize) + if _, err := io.ReadFull(reader, buf); err != nil { + return nil, err + } + + return parseSuperblock(buf[superblockOffset:]) +} diff --git a/erofs/verity.go b/erofs/verity.go new file mode 100644 index 0000000..98f3c13 --- /dev/null +++ b/erofs/verity.go @@ -0,0 +1,523 @@ +package erofs + +// #cgo pkg-config: libcryptsetup devmapper --static +// #include +// #include +// #include +// #include +/* +int get_verity_params(char *device, char **params) +{ + struct dm_task *dmt; + struct dm_info dmi; + int r; + uint64_t start, length; + char *type, *tmpParams; + + dmt = dm_task_create(DM_DEVICE_TABLE); + if (!dmt) + return 1; + + r = 2; + if (!dm_task_secure_data(dmt)) + goto out; + + r = 3; + if (!dm_task_set_name(dmt, device)) + goto out; + + r = 4; + if (!dm_task_run(dmt)) + goto out; + + r = 5; + if (!dm_task_get_info(dmt, &dmi)) + goto out; + + r = 6; + if (!dmi.exists) + goto out; + + r = 7; + if (dmi.target_count <= 0) + goto out; + + r = 8; + dm_get_next_target(dmt, NULL, &start, &length, &type, &tmpParams); + if (!type) + goto out; + + r = 9; + if (strcasecmp(type, CRYPT_VERITY)) { + fprintf(stderr, "type: %s (%s) %d\n", type, CRYPT_VERITY, strcmp(type, CRYPT_VERITY)); + goto out; + } + *params = strdup(tmpParams); + + r = 0; +out: + dm_task_destroy(dmt); + return r; +} +*/ +import "C" + +import ( + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + "unsafe" + + "github.com/freddierice/go-losetup" + "github.com/martinjungblut/go-cryptsetup" + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/mount" +) + +const VerityRootHashAnnotation = "io.stackeroci.stacker.erofs_verity_root_hash" + +type verityDeviceType struct { + Flags uint + DataDevice string + HashOffset uint64 +} + +func (verity verityDeviceType) Name() string { + return C.CRYPT_VERITY +} + +func (verity verityDeviceType) Unmanaged() (unsafe.Pointer, func()) { + var cParams C.struct_crypt_params_verity + + cParams.hash_name = C.CString("sha256") + cParams.data_device = C.CString(verity.DataDevice) + cParams.fec_device = nil + cParams.fec_roots = 0 + + cParams.salt_size = 32 // DEFAULT_VERITY_SALT_SIZE for x86 + cParams.salt = nil + + // these can't be larger than a page size, but we want them to be as + // big as possible so the hash data is small, so let's set them to a + // page size. + cParams.data_block_size = C.uint(os.Getpagesize()) + cParams.hash_block_size = C.uint(os.Getpagesize()) + + cParams.data_size = C.ulong(verity.HashOffset / uint64(os.Getpagesize())) + cParams.hash_area_offset = C.ulong(verity.HashOffset) + cParams.fec_area_offset = 0 + cParams.hash_type = 1 // use format version 1 (i.e. "modern", non chrome-os) + cParams.flags = C.uint(verity.Flags) + + deallocate := func() { + C.free(unsafe.Pointer(cParams.hash_name)) + C.free(unsafe.Pointer(cParams.data_device)) + } + + return unsafe.Pointer(&cParams), deallocate +} + +func isCryptsetupEINVAL(err error) bool { + cse, ok := err.(*cryptsetup.Error) + return ok && cse.Code() == -22 +} + +var cryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") + +func appendVerityData(file string) (string, error) { + fi, err := os.Lstat(file) + if err != nil { + return "", errors.WithStack(err) + } + + verityOffset := fi.Size() + + fmt.Printf("VERIFY1 - offset:+%v\n", verityOffset) + + // we expect mkerofs to have padded the file to the nearest 4k + // (dm-verity requires device block size, which is 512 for loopback, + // which is a multiple of 4k), let's check that here + if verityOffset%512 != 0 { + return "", errors.Errorf("bad verity file size %d", verityOffset) + } + + verityDevice, err := cryptsetup.Init(file) + if err != nil { + return "", errors.WithStack(err) + } + + fmt.Printf("VERIFY1 - offset:%+v dev:%+v\n", verityOffset, verityDevice) + + verityType := verityDeviceType{ + Flags: cryptsetup.CRYPT_VERITY_CREATE_HASH, + DataDevice: file, + HashOffset: uint64(verityOffset), + } + err = verityDevice.Format(verityType, cryptsetup.GenericParams{}) + if err != nil { + return "", errors.WithStack(err) + } + + fmt.Printf("VERIFY1 - offset:%+v dev:%+v err:%v\n", verityOffset, verityDevice, err) + + // a bit ugly, but this is the only API for querying the root + // hash (short of invoking the veritysetup binary), and it was + // added in libcryptsetup commit 188cb114af94 ("Add support for + // verity in crypt_volume_key_get and use it in status"), which + // is relatively recent (ubuntu 20.04 does not have this patch, + // for example). + // + // before that, we get a -22. so, let's test for that and + // render a special error message. + rootHash, _, err := verityDevice.VolumeKeyGet(cryptsetup.CRYPT_ANY_SLOT, "") + if isCryptsetupEINVAL(err) { + return "", cryptsetupTooOld + } else if err != nil { + return "", err + } + + fmt.Printf("VERIFY1 - offset:%+v dev:%+v err:%v rootHash:%v\n", verityOffset, verityDevice, err, rootHash) + + return fmt.Sprintf("%x", rootHash), errors.WithStack(err) +} + +func verityDataLocation(sblock *superblock) (uint64, error) { + return uint64((1 << sblock.BlockSizeBits) * sblock.Blocks), nil +} + +func verityName(p string) string { + return fmt.Sprintf("%s-%s", p, veritySuffix) +} + +func fileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} + +// Mount a filesystem as container root, without host root +// privileges. We do this using squashfuse. +func GuestMount(squashFile string, mountpoint string) error { + if isMountpoint(mountpoint) { + return errors.Errorf("%s is already mounted", mountpoint) + } + + abs, err := filepath.Abs(squashFile) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", squashFile, err) + } + squashFile = abs + + abs, err = filepath.Abs(mountpoint) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) + } + mountpoint = abs + + cmd, err := erofsFuse(squashFile, mountpoint) + if err != nil { + return err + } + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process after guestmount %s: %v", squashFile, err) + } + return nil +} + +func isMountpoint(dest string) bool { + mounted, err := mount.IsMountpoint(dest) + return err == nil && mounted +} + +// Takes /proc/self/uid_map contents as one string +// Returns true if this is a uidmap representing the whole host +// uid range. +func uidmapIsHost(oneline string) bool { + oneline = strings.TrimSuffix(oneline, "\n") + if len(oneline) == 0 { + return false + } + lines := strings.Split(oneline, "\n") + if len(lines) != 1 { + return false + } + words := strings.Fields(lines[0]) + if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { + return false + } + + return true +} + +func amHostRoot() bool { + // if not uid 0, not host root + if os.Geteuid() != 0 { + return false + } + // if uid_map doesn't map 0 to 0, not host root + bytes, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + return false + } + return uidmapIsHost(string(bytes)) +} + +func Mount(erofs, mountpoint, rootHash string) error { + if !amHostRoot() { + return GuestMount(erofs, mountpoint) + } + err := HostMount(erofs, mountpoint, rootHash) + if err == nil || rootHash != "" { + return err + } + return GuestMount(erofs, mountpoint) +} + +func HostMount(erofs string, mountpoint string, rootHash string) error { + fi, err := os.Stat(erofs) + if err != nil { + return errors.WithStack(err) + } + + sblock, err := readSuperblock(erofs) + if err != nil { + return err + } + + verityOffset, err := verityDataLocation(sblock) + if err != nil { + return err + } + + if verityOffset == uint64(fi.Size()) && rootHash != "" { + return errors.Errorf("asked for verity but no data present") + } + + if rootHash == "" && verityOffset != uint64(fi.Size()) { + return errors.Errorf("verity data present but no root hash specified") + } + + mountSourcePath := "" + + var verityDevice *cryptsetup.Device + name := verityName(path.Base(erofs)) + + loopDevNeedsClosedOnErr := false + var loopDev losetup.Device + + // set up the verity device if necessary + if rootHash != "" { + verityDevPath := path.Join("/dev/mapper", name) + mountSourcePath = verityDevPath + _, err = os.Stat(verityDevPath) + if err != nil { + if !os.IsNotExist(err) { + return errors.WithStack(err) + } + + loopDev, err = losetup.Attach(erofs, 0, true) + if err != nil { + return errors.WithStack(err) + } + loopDevNeedsClosedOnErr = true + + verityDevice, err = cryptsetup.Init(loopDev.Path()) + if err != nil { + return errors.WithStack(err) + } + + verityType := verityDeviceType{ + Flags: 0, + DataDevice: loopDev.Path(), + HashOffset: verityOffset, + } + + err = verityDevice.Load(verityType) + if err != nil { + loopDev.Detach() + return errors.WithStack(err) + } + + // each string byte hex encodes four bits of info... + volumeKeySizeInBytes := len(rootHash) * 4 / 8 + rootHashBytes, err := hex.DecodeString(rootHash) + if err != nil { + loopDev.Detach() + return errors.WithStack(err) + } + + if len(rootHashBytes) != volumeKeySizeInBytes { + loopDev.Detach() + return errors.Errorf("unexpected key size for %s", rootHash) + } + + err = verityDevice.ActivateByVolumeKey(name, string(rootHashBytes), volumeKeySizeInBytes, cryptsetup.CRYPT_ACTIVATE_READONLY) + if err != nil { + loopDev.Detach() + return errors.WithStack(err) + } + } else { + err = ConfirmExistingVerityDeviceHash(verityDevPath, rootHash, rejectVerityFailure) + if err != nil { + return err + } + } + } else { + loopDev, err = losetup.Attach(erofs, 0, true) + if err != nil { + return errors.WithStack(err) + } + defer loopDev.Detach() + mountSourcePath = loopDev.Path() + + } + + err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, "erofs", unix.MS_RDONLY, "")) + if err != nil { + if verityDevice != nil { + verityDevice.Deactivate(name) + loopDev.Detach() + } + if loopDevNeedsClosedOnErr { + loopDev.Detach() + } + return err + } + return nil +} + +func findLoopBackingVerity(device string) (int64, error) { + fi, err := os.Stat(device) + if err != nil { + return -1, errors.WithStack(err) + } + + var minor uint32 + switch stat := fi.Sys().(type) { + case *unix.Stat_t: + minor = unix.Minor(uint64(stat.Rdev)) + case *syscall.Stat_t: + minor = unix.Minor(uint64(stat.Rdev)) + default: + return -1, errors.Errorf("unknown stat info type %T", stat) + } + + ents, err := os.ReadDir(fmt.Sprintf("/sys/block/dm-%d/slaves", minor)) + if err != nil { + return -1, errors.WithStack(err) + } + + if len(ents) != 1 { + return -1, errors.Errorf("too many slaves for %v", device) + } + loop := ents[0] + + deviceNo, err := strconv.ParseInt(strings.TrimPrefix(filepath.Base(loop.Name()), "loop"), 10, 64) + if err != nil { + return -1, errors.Wrapf(err, "bad loop dev %v", loop.Name()) + } + + return deviceNo, nil +} + +func Umount(mountpoint string) error { + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return err + } + + // first, find the verity device that backs the mount + theMount, found := mounts.FindMount(mountpoint) + if !found { + return errors.Errorf("%s is not a mountpoint", mountpoint) + } + + err = unix.Unmount(mountpoint, 0) + if err != nil { + return errors.Wrapf(err, "failed unmounting %v", mountpoint) + } + + if _, err := os.Stat(theMount.Source); err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WithStack(err) + } + + // was this a verity mount or a regular loopback mount? (if it's a + // regular loopback mount, we detached it above, so need to do anything + // special here; verity doesn't play as nicely) + if strings.HasSuffix(theMount.Source, veritySuffix) { + // find the loop device that backs the verity device + deviceNo, err := findLoopBackingVerity(theMount.Source) + if err != nil { + return err + } + + loopDev := losetup.New(uint64(deviceNo), 0) + // here, we don't have the loopback device any more (we detached it + // above). the cryptsetup API allows us to pass NULL for the crypt + // device, but go-cryptsetup doesn't have a way to initialize a NULL + // crypt device short of making the struct by hand like this. + err = (&cryptsetup.Device{}).Deactivate(theMount.Source) + if err != nil { + return errors.WithStack(err) + } + + // finally, kill the loop dev + err = loopDev.Detach() + if err != nil { + return errors.Wrapf(err, "failed to detach loop dev for %v", theMount.Source) + } + } + + return nil +} + +// If we are using squashfuse, then we will be unable to get verity has from +// the mount device. This is not a safe thing, we we only allow it when the +// device was mounted originally with AllowMissingVerityData. + +const ( + rejectVerityFailure = false + allowVerityFailure = false +) + +func ConfirmExistingVerityDeviceHash(devicePath string, rootHash string, allowVerityFailure bool) error { + device := filepath.Base(devicePath) + cDevice := C.CString(device) + defer C.free(unsafe.Pointer(cDevice)) + + var cParams *C.char + + rc := C.get_verity_params(cDevice, &cParams) + if rc != 0 { + if allowVerityFailure { + return nil + } + return errors.Errorf("problem getting hash from %v: %v", device, rc) + } + defer C.free(unsafe.Pointer(cParams)) + + params := C.GoString(cParams) + + // https://gitlab.com/cryptsetup/cryptsetup/-/wikis/DMVerity + fields := strings.Fields(params) + if len(fields) < 10 { + return errors.Errorf("invalid dm params for %v: %v", device, params) + } + + if rootHash != fields[8] { + return errors.Errorf("invalid root hash for %v: %v (expected: %v)", device, fields[7], rootHash) + } + + return nil +} diff --git a/erofs/verity_static.go b/erofs/verity_static.go new file mode 100644 index 0000000..549c5d1 --- /dev/null +++ b/erofs/verity_static.go @@ -0,0 +1,10 @@ +//go:build static_build +// +build static_build + +package erofs + +// cryptsetup's pkgconfig is broken (it does not set Requires.private or +// Libs.private at all), so we do the LDLIBS for it by hand. + +// #cgo LDFLAGS: -lcryptsetup -lcrypto -lssl -lblkid -luuid -ljson-c -lpthread -ldl +import "C" diff --git a/erofs/verity_test.go b/erofs/verity_test.go new file mode 100644 index 0000000..f9ef7c5 --- /dev/null +++ b/erofs/verity_test.go @@ -0,0 +1,134 @@ +package erofs + +import ( + "fmt" + "io" + "os" + "os/exec" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +type uidmapTestcase struct { + uidmap string + expected bool +} + +var uidmapTests = []uidmapTestcase{ + uidmapTestcase{ + uidmap: ` 0 0 4294967295`, + expected: true, + }, + uidmapTestcase{ + uidmap: ` 0 0 1000 +2000 2000 1`, + expected: false, + }, + uidmapTestcase{ + uidmap: ` 0 0 1000`, + expected: false, + }, + uidmapTestcase{ + uidmap: ` 10 0 4294967295`, + expected: false, + }, + uidmapTestcase{ + uidmap: ` 0 10 4294967295`, + expected: false, + }, + uidmapTestcase{ + uidmap: ` 0 0 1`, + expected: false, + }, +} + +func TestAmHostRoot(t *testing.T) { + assert := assert.New(t) + for _, testcase := range uidmapTests { + v := uidmapIsHost(testcase.uidmap) + assert.Equal(v, testcase.expected) + } +} + +func TestVerityMetadata(t *testing.T) { + assert := assert.New(t) + + rootfs, err := os.MkdirTemp("", "stacker_verity_test_rootfs") + assert.NoError(err) + defer os.RemoveAll(rootfs) + + tempdir, err := os.MkdirTemp("", "stacker_verity_test_tempdir") + assert.NoError(err) + defer os.RemoveAll(tempdir) + + err = os.WriteFile(path.Join(rootfs, "foo"), []byte("bar"), 0644) + assert.NoError(err) + + reader, _, rootHash, err := MakeErofs(tempdir, rootfs, nil, VerityMetadataPresent) + if err == cryptsetupTooOld { + t.Skip("libcryptsetup too old") + } + assert.NoError(err) + + content, err := io.ReadAll(reader) + assert.NoError(err) + erofsFile := path.Join(tempdir, "foo.erofs") + err = os.WriteFile(erofsFile, content, 0600) + assert.NoError(err) + + sblock, err := readSuperblock(erofsFile) + assert.NoError(err) + + verityOffset, err := verityDataLocation(sblock) + assert.NoError(err) + + // now let's try to verify it at least in userspace. exec cryptsetup + // because i'm lazy and it's only in tests + cmd := exec.Command("veritysetup", "verify", erofsFile, erofsFile, rootHash, + "--hash-offset", fmt.Sprintf("%d", verityOffset)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + assert.NoError(err) + + // what if we fiddle with the verity data? note that we have to fiddle + // with the beginning of the verity block, which will be 4k long for + // our small erofs file, because the stuff at the end of the verity + // block is unused. + const bytesToFlip = 2 + const flipAtOffset = -4087 + + f, err := os.OpenFile(erofsFile, os.O_RDWR, 0644) + assert.NoError(err) + defer f.Close() + _, err = f.Seek(flipAtOffset, os.SEEK_END) + assert.NoError(err) + + buf := make([]byte, bytesToFlip) + n, err := f.Read(buf) + assert.Equal(n, bytesToFlip) + assert.NoError(err) + + for i := range buf { + buf[i] = buf[i] ^ 0xff + } + + _, err = f.Seek(flipAtOffset, os.SEEK_END) + assert.NoError(err) + n, err = f.Write(buf) + assert.Equal(n, bytesToFlip) + assert.NoError(err) + assert.NoError(f.Sync()) + assert.NoError(f.Close()) + + cmd = exec.Command("veritysetup", "verify", erofsFile, erofsFile, rootHash, + "--hash-offset", fmt.Sprintf("%d", verityOffset)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + assert.Error(err) +}