From c01a838d5c39239c5c10c776772387b3580c2edf Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 6 Sep 2024 19:33:17 +0000 Subject: [PATCH 1/2] feat: initial commit for erofs support Signed-off-by: Ramkumar Chinchani --- .github/workflows/build.yaml | 4 +- erofs/erofs.go | 744 +++++++++++++++++++++++++++++++++++ erofs/mediatype.go | 37 ++ erofs/superblock.go | 247 ++++++++++++ erofs/verity.go | 519 ++++++++++++++++++++++++ erofs/verity_static.go | 10 + erofs/verity_test.go | 135 +++++++ 7 files changed, 1694 insertions(+), 2 deletions(-) create mode 100644 erofs/erofs.go create mode 100644 erofs/mediatype.go create mode 100644 erofs/superblock.go create mode 100644 erofs/verity.go create mode 100644 erofs/verity_static.go create mode 100644 erofs/verity_test.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b1e8c5d..1b815ae 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-20.04, ubuntu-22.04] + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] name: build runs-on: ${{ matrix.os }} steps: @@ -20,7 +20,7 @@ jobs: sudo apt-get update sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \ libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \ - squashfuse squashfs-tools + squashfuse squashfs-tools erofs-utils - name: setup lxc run: | chmod ugo+x $HOME diff --git a/erofs/erofs.go b/erofs/erofs.go new file mode 100644 index 0000000..e088b71 --- /dev/null +++ b/erofs/erofs.go @@ -0,0 +1,744 @@ +// This package is a small go "library" (read: exec wrapper) around the +// mkfs.erofs binary that provides some useful primitives. +package erofs + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/log" + "machinerun.io/atomfs/mount" +) + +var checkZstdSupported sync.Once +var zstdIsSuspported bool + +var exPolInfo struct { + once sync.Once + err error + policy *ExtractPolicy +} + +// ExcludePaths represents a list of paths to exclude in a erofs listing. +// Users should do something like filepath.Walk() over the whole filesystem, +// calling AddExclude() or AddInclude() based on whether they want to include +// or exclude a particular file. Note that if e.g. /usr is excluded, then +// everyting underneath is also implicitly excluded. The +// AddExclude()/AddInclude() methods do the math to figure out what is the +// correct set of things to exclude or include based on what paths have been +// previously included or excluded. +type ExcludePaths struct { + exclude map[string]bool + include []string +} + +type erofsFuseInfoStruct struct { + Path string + Version string + SupportsNotfiy bool +} + +var once sync.Once +var erofsFuseInfo = erofsFuseInfoStruct{"", "", false} + +func NewExcludePaths() *ExcludePaths { + return &ExcludePaths{ + exclude: map[string]bool{}, + include: []string{}, + } +} + +func (eps *ExcludePaths) AddExclude(p string) { + for _, inc := range eps.include { + // If /usr/bin/ls has changed but /usr hasn't, we don't want to list + // /usr in the include paths any more, so let's be sure to only + // add things which aren't prefixes. + if strings.HasPrefix(inc, p) { + return + } + } + eps.exclude[p] = true +} + +func (eps *ExcludePaths) AddInclude(orig string, isDir bool) { + // First, remove this thing and all its parents from exclude. + p := orig + + // normalize to the first dir + if !isDir { + p = path.Dir(p) + } + for { + // our paths are all absolute, so this is a base case + if p == "/" { + break + } + + delete(eps.exclude, p) + p = filepath.Dir(p) + } + + // now add it to the list of includes, so we don't accidentally re-add + // anything above. + eps.include = append(eps.include, orig) +} + +func (eps *ExcludePaths) String() (string, error) { + var buf bytes.Buffer + for p := range eps.exclude { + _, err := buf.WriteString(p) + if err != nil { + return "", err + } + _, err = buf.WriteString("\n") + if err != nil { + return "", err + } + } + + _, err := buf.WriteString("\n") + if err != nil { + return "", err + } + + return buf.String(), nil +} + +func MakeErofs(tempdir string, rootfs string, eps *ExcludePaths, verity VerityMetadata) (io.ReadCloser, string, string, error) { + var excludesFile string + var err error + var toExclude string + var rootHash string + + if eps != nil { + toExclude, err = eps.String() + if err != nil { + return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list") + } + } + + if len(toExclude) != 0 { + excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-") + if err != nil { + return nil, "", rootHash, err + } + defer os.Remove(excludes.Name()) + + excludesFile = excludes.Name() + _, err = excludes.WriteString(toExclude) + excludes.Close() + if err != nil { + return nil, "", rootHash, err + } + } + + tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-") + if err != nil { + return nil, "", rootHash, err + } + tmpErofs.Close() + os.Remove(tmpErofs.Name()) + defer os.Remove(tmpErofs.Name()) + args := []string{tmpErofs.Name(), rootfs} + compression := GzipCompression + if mkerofsSupportsZstd() { + args = append(args, "-z", "zstd") + compression = ZstdCompression + } + if len(toExclude) != 0 { + args = append(args, "--exclude-path", excludesFile) + } + cmd := exec.Command("mkfs.erofs", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err = cmd.Run(); err != nil { + return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs") + } + + if verity { + rootHash, err = appendVerityData(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, err + } + } + + blob, err := os.Open(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, errors.WithStack(err) + } + + return blob, GenerateErofsMediaType(compression, verity), rootHash, nil +} + +func isMountedAtDir(_, dest string) (bool, error) { + dstat, err := os.Stat(dest) + if os.IsNotExist(err) { + return false, nil + } + if !dstat.IsDir() { + return false, nil + } + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return false, err + } + + fdest, err := filepath.Abs(dest) + if err != nil { + return false, err + } + for _, m := range mounts { + if m.Target == fdest { + return true, nil + } + } + + return false, nil +} + +func findErofsFuseInfo() { + var erofsPath string + if p := which("erofsfuse"); p != "" { + erofsPath = p + } else { + erofsPath = which("erofsfuse") + } + if erofsPath == "" { + return + } + version, supportsNotify := erofsfuseSupportsMountNotification(erofsPath) + log.Infof("Found erofsfuse at %s (version=%s notify=%t)", erofsPath, version, supportsNotify) + erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version, supportsNotify} +} + +// erofsfuseSupportsMountNotification - returns true if erofsfuse supports mount +// notification, false otherwise +// erofsfuse is the path to the erofsfuse binary +func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { + cmd := exec.Command(erofsfuse) + + // `erofsfuse` always returns an error... so we ignore it. + out, _ := cmd.CombinedOutput() + + firstLine := strings.Split(string(out[:]), "\n")[0] + version := strings.Split(firstLine, " ")[1] + v, err := semver.NewVersion(version) + if err != nil { + return version, false + } + // erofsfuse notify mechanism was merged in 0.5.0 + constraint, err := semver.NewConstraint(">= 0.5.0") + if err != nil { + return version, false + } + if constraint.Check(v) { + return version, true + } + return version, false +} + +var squashNotFound = errors.Errorf("erofsfuse program not found") + +// erofsFuse - mount squashFile to extractDir +// return a pointer to the erofsfuse cmd. +// The caller of the this is responsible for the process created. +func erofsFuse(squashFile, extractDir string) (*exec.Cmd, error) { + var cmd *exec.Cmd + + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return cmd, squashNotFound + } + + notifyOpts := "" + notifyPath := "" + if erofsFuseInfo.SupportsNotfiy { + sockdir, err := os.MkdirTemp("", "sock") + if err != nil { + return cmd, err + } + defer os.RemoveAll(sockdir) + notifyPath = filepath.Join(sockdir, "notifypipe") + if err := syscall.Mkfifo(notifyPath, 0640); err != nil { + return cmd, err + } + notifyOpts = "notify_pipe=" + notifyPath + } + + // given extractDir of path/to/some/dir[/], log to path/to/some/.dir-squashfs.log + extractDir = strings.TrimSuffix(extractDir, "/") + + var cmdOut io.Writer + var err error + + logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-erofsfuse.log") + if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { + log.Infof("Failed to open %s for write: %v", logf, err) + return cmd, err + } + + fiPre, err := os.Lstat(extractDir) + if err != nil { + return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) + } + if fiPre.Mode()&os.ModeSymlink != 0 { + return cmd, errors.Errorf("Refusing to mount onto a symbolic linkd") + } + + // It would be nice to only enable debug (or maybe to only log to file at all) + // if 'stacker --debug', but we do not have access to that info here. + // to debug erofsfuse, use "allow_other,debug" + optionArgs := "allow_other,debug" + if notifyOpts != "" { + optionArgs += "," + notifyOpts + } + cmd = exec.Command(erofsFuseInfo.Path, "-f", "-o", optionArgs, squashFile, extractDir) + cmd.Stdin = nil + cmd.Stdout = cmdOut + cmd.Stderr = cmdOut + cmdOut.Write([]byte(fmt.Sprintf("# %s\n", strings.Join(cmd.Args, " ")))) + if err != nil { + return cmd, errors.Wrapf(err, "Failed writing to %s", logf) + } + log.Debugf("Extracting %s -> %s with %s [%s]", squashFile, extractDir, erofsFuseInfo.Path, logf) + err = cmd.Start() + if err != nil { + return cmd, err + } + + // now poll/wait for one of 3 things to happen + // a. child process exits - if it did, then some error has occurred. + // b. the directory Entry is different than it was before the call + // to erofsfuse. We have to do this because we do not have another + // way to know when the mount has been populated. + // https://github.com/vasi/erofsfuse/issues/49 + // c. a timeout (timeLimit) was hit + startTime := time.Now() + timeLimit := 30 * time.Second + alarmCh := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(alarmCh) + }() + if erofsFuseInfo.SupportsNotfiy { + notifyCh := make(chan byte) + log.Infof("%s supports notify pipe, watching %q", erofsFuseInfo.Path, notifyPath) + go func() { + f, err := os.Open(notifyPath) + if err != nil { + return + } + defer f.Close() + b1 := make([]byte, 1) + for { + n1, err := f.Read(b1) + if err != nil { + return + } + if err == nil && n1 >= 1 { + break + } + } + notifyCh <- b1[0] + }() + + select { + case <-alarmCh: + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", squashFile, erofsFuseInfo.Path, timeLimit) + case ret := <-notifyCh: + if ret == 's' { + return cmd, nil + } else { + return cmd, errors.Errorf("erofsfuse returned an error, check %s", logf) + } + } + } + for count := 0; !fileChanged(fiPre, extractDir); count++ { + if cmd.ProcessState != nil { + // process exited, the Wait() call in the goroutine above + // caused ProcessState to be populated. + return cmd, errors.Errorf("erofsfuse mount of %s with %s exited unexpectedly with %d", squashFile, erofsFuseInfo.Path, cmd.ProcessState.ExitCode()) + } + if time.Since(startTime) > timeLimit { + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", squashFile, erofsFuseInfo.Path, timeLimit) + } + if count%10 == 1 { + log.Debugf("%s is not yet mounted...(%s)", extractDir, time.Since(startTime)) + } + time.Sleep(time.Duration(50 * time.Millisecond)) + } + + return cmd, nil +} + +type ExtractPolicy struct { + Extractors []SquashExtractor + Extractor SquashExtractor + Excuses map[string]error + initialized bool + mutex sync.Mutex +} + +type SquashExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} + +func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { + p := &ExtractPolicy{ + Extractors: []SquashExtractor{}, + Excuses: map[string]error{}, + } + + allEx := []SquashExtractor{ + &KernelExtractor{}, + &ErofsFuseExtractor{}, + &UnsquashfsExtractor{}, + } + byName := map[string]SquashExtractor{} + for _, i := range allEx { + byName[i.Name()] = i + } + + for _, i := range args { + extractor, ok := byName[i] + if !ok { + return nil, errors.Errorf("Unknown extractor: '%s'", i) + } + excuse := extractor.IsAvailable() + if excuse != nil { + p.Excuses[i] = excuse + continue + } + p.Extractors = append(p.Extractors, extractor) + } + return p, nil +} + +type UnsquashfsExtractor struct { + mutex sync.Mutex +} + +func (k *UnsquashfsExtractor) Name() string { + return "unsquashfs" +} + +func (k *UnsquashfsExtractor) IsAvailable() error { + if which("unsquashfs") == "" { + return errors.Errorf("no 'unsquashfs' in PATH") + } + return nil +} + +func (k *UnsquashfsExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + // check if already extracted + empty, err := isEmptyDir(extractDir) + if err != nil { + return errors.Wrapf(err, "Error checking for empty dir") + } + if !empty { + return nil + } + + log.Debugf("unsquashfs %s -> %s", squashFile, extractDir) + cmd := exec.Command("unsquashfs", "-f", "-d", extractDir, squashFile) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = nil + err = cmd.Run() + + // on failure, remove the directory + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, squashFile, rmErr) + } + return err + } + + // assert that extraction must create files. This way we can assume non-empty dir above + // was populated by unsquashfs. + empty, err = isEmptyDir(extractDir) + if err != nil { + return errors.Errorf("Failed to read %s after successful extraction of %s: %v", + extractDir, squashFile, err) + } + if empty { + return errors.Errorf("%s was an empty fs image", squashFile) + } + + return nil +} + +type KernelExtractor struct { + mutex sync.Mutex +} + +func (k *KernelExtractor) Name() string { + return "kmount" +} + +func (k *KernelExtractor) IsAvailable() error { + if !amHostRoot() { + return errors.Errorf("not host root") + } + return nil +} + +func (k *KernelExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(squashFile, extractDir); err != nil { + return err + } else if mounted { + return nil + } + + ecmd := []string{"mount", "-tsquashfs", "-oloop,ro", squashFile, extractDir} + var output bytes.Buffer + cmd := exec.Command(ecmd[0], ecmd[1:]...) + cmd.Stdin = nil + cmd.Stdout = &output + cmd.Stderr = cmd.Stdout + err := cmd.Run() + if err == nil { + return nil + } + + var retErr error + + exitError, ok := err.(*exec.ExitError) + if !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", + squashFile, ecmd, err) + } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", + squashFile, ecmd, err) + } else { + retErr = errors.Errorf("kmount(%s) exited %d: %v", squashFile, status.ExitStatus(), output.String()) + } + + return retErr +} + +type ErofsFuseExtractor struct { + mutex sync.Mutex +} + +func (k *ErofsFuseExtractor) Name() string { + return "erofsfuse" +} + +func (k *ErofsFuseExtractor) IsAvailable() error { + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return errors.Errorf("no 'erofsfuse' in PATH") + } + return nil +} + +func (k *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(erofsFile, extractDir); mounted && err == nil { + log.Debugf("[%s] %s already mounted -> %s", k.Name(), erofsFile, extractDir) + return nil + } else if err != nil { + return err + } + + cmd, err := erofsFuse(erofsFile, extractDir) + if err != nil { + return err + } + + log.Debugf("erofsfuse mounted (%d) %s -> %s", cmd.Process.Pid, erofsFile, extractDir) + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process %s: %v", cmd, err) + } + return nil +} + +// ExtractSingleSquashPolicy - extract squashfile to extractDir +func ExtractSingleSquashPolicy(squashFile, extractDir string, policy *ExtractPolicy) error { + const initName = "init" + if policy == nil { + return errors.Errorf("policy cannot be nil") + } + + // avoid taking a lock if already initialized (possibly premature optimization) + if !policy.initialized { + policy.mutex.Lock() + // We may have been waiting on the initializer. If so, then the policy will now be initialized. + // if not, then we are the initializer. + if !policy.initialized { + defer policy.mutex.Unlock() + defer func() { + policy.initialized = true + }() + } else { + policy.mutex.Unlock() + } + } + + err := os.MkdirAll(extractDir, 0755) + if err != nil { + return err + } + + fdest, err := filepath.Abs(extractDir) + if err != nil { + return err + } + + if policy.initialized { + if err, ok := policy.Excuses[initName]; ok { + return err + } + return policy.Extractor.Mount(squashFile, fdest) + } + + // At this point we are the initialzer + if policy.Excuses == nil { + policy.Excuses = map[string]error{} + } + + if len(policy.Extractors) == 0 { + policy.Excuses[initName] = errors.Errorf("policy had no extractors") + return policy.Excuses[initName] + } + + var extractor SquashExtractor + allExcuses := []string{} + for _, extractor = range policy.Extractors { + err = extractor.Mount(squashFile, fdest) + if err == nil { + policy.Extractor = extractor + log.Debugf("Selected squashfs extractor %s", extractor.Name()) + return nil + } + policy.Excuses[extractor.Name()] = err + } + + for n, exc := range policy.Excuses { + allExcuses = append(allExcuses, fmt.Sprintf("%s: %v", n, exc)) + } + + // nothing worked. populate Excuses[initName] + policy.Excuses[initName] = errors.Errorf("No suitable extractor found:\n %s", strings.Join(allExcuses, "\n ")) + return policy.Excuses[initName] +} + +// ExtractSingleSquash - extract the squashFile to extractDir +// Initialize a extractPolicy struct and then call ExtractSingleSquashPolicy +// wik()th that. +func ExtractSingleSquash(squashFile string, extractDir string) error { + exPolInfo.once.Do(func() { + const envName = "STACKER_SQUASHFS_EXTRACT_POLICY" + const defPolicy = "kmount erofsfuse unsquashfs" + val := os.Getenv(envName) + if val == "" { + val = defPolicy + } + exPolInfo.policy, exPolInfo.err = NewExtractPolicy(strings.Fields(val)...) + if exPolInfo.err == nil { + for k, v := range exPolInfo.policy.Excuses { + log.Debugf(" squashfs extractor %s is not available: %v", k, v) + } + } + }) + + if exPolInfo.err != nil { + return exPolInfo.err + } + + return ExtractSingleSquashPolicy(squashFile, extractDir, exPolInfo.policy) +} + +func mkerofsSupportsZstd() bool { + checkZstdSupported.Do(func() { + var stdoutBuffer strings.Builder + var stderrBuffer strings.Builder + + cmd := exec.Command("mkfs.erofs", "--help") + cmd.Stdout = &stdoutBuffer + cmd.Stderr = &stderrBuffer + + // Ignore errs here as `mkerofs --help` exit status code is 1 + _ = cmd.Run() + + if strings.Contains(stdoutBuffer.String(), "zstd") || + strings.Contains(stderrBuffer.String(), "zstd") { + zstdIsSuspported = true + } + }) + + return zstdIsSuspported +} + +func isEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// which - like the unix utility, return empty string for not-found. +// this might fit well in lib/, but currently lib's test imports +// squashfs creating a import loop. +func which(name string) string { + return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) +} + +func whichSearch(name string, paths []string) string { + var search []string + + if strings.ContainsRune(name, os.PathSeparator) { + if filepath.IsAbs(name) { + search = []string{name} + } else { + search = []string{"./" + name} + } + } else { + search = []string{} + for _, p := range paths { + search = append(search, filepath.Join(p, name)) + } + } + + for _, fPath := range search { + if err := unix.Access(fPath, unix.X_OK); err == nil { + return fPath + } + } + + return "" +} diff --git a/erofs/mediatype.go b/erofs/mediatype.go new file mode 100644 index 0000000..6726ea1 --- /dev/null +++ b/erofs/mediatype.go @@ -0,0 +1,37 @@ +package erofs + +import ( + "fmt" + "strings" +) + +type ErofsCompression string +type VerityMetadata bool + +const ( + BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs" + + GzipCompression ErofsCompression = "gzip" + ZstdCompression ErofsCompression = "zstd" + + veritySuffix = "verity" + + VerityMetadataPresent VerityMetadata = true + VerityMetadataMissing VerityMetadata = false +) + +func IsErofsMediaType(mediaType string) bool { + return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs) +} + +func GenerateErofsMediaType(comp ErofsCompression, verity VerityMetadata) string { + verityString := "" + if verity { + verityString = fmt.Sprintf("+%s", veritySuffix) + } + return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString) +} + +func HasVerityMetadata(mediaType string) VerityMetadata { + return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix)) +} diff --git a/erofs/superblock.go b/erofs/superblock.go new file mode 100644 index 0000000..094f68c --- /dev/null +++ b/erofs/superblock.go @@ -0,0 +1,247 @@ +package erofs + +import ( + "encoding/binary" + "io" + "os" + + "github.com/pkg/errors" +) + +/* + +https://docs.kernel.org/filesystems/erofs.html + +On-disk details + + |-> aligned with the block size + ____________________________________________________________ +| |SB| | ... | Metadata | ... | Data | Metadata | ... | Data | +|_|__|_|_____|__________|_____|______|__________|_____|______| +0 +1K + +*/ + +const ( + // Definitions for superblock. + superblockMagicV1 = 0xe0f5e1e2 + superblockMagic = superblockMagicV1 + superblockOffset = 1024 + + // Inode slot size in bit shift. + InodeSlotBits = 5 + + // Max file name length. + MaxNameLen = 255 +) + +// Bit definitions for Inode*::Format. +const ( + InodeLayoutBit = 0 + InodeLayoutBits = 1 + + InodeDataLayoutBit = 1 + InodeDataLayoutBits = 3 +) + +// Inode layouts. +const ( + InodeLayoutCompact = 0 + InodeLayoutExtended = 1 +) + +// Inode data layouts. +const ( + InodeDataLayoutFlatPlain = iota + InodeDataLayoutFlatCompressionLegacy + InodeDataLayoutFlatInline + InodeDataLayoutFlatCompression + InodeDataLayoutChunkBased + InodeDataLayoutMax +) + +// Features w/ backward compatibility. +// This is not exhaustive, unused features are not listed. +const ( + FeatureCompatSuperBlockChecksum = 0x00000001 +) + +// Features w/o backward compatibility. +// +// Any features that aren't in FeatureIncompatSupported are incompatible +// with this implementation. +// +// This is not exhaustive, unused features are not listed. +const ( + FeatureIncompatSupported = 0x0 +) + +// Sizes of on-disk structures in bytes. +const ( + superblockSize = 128 + InodeCompactSize = 32 + InodeExtendedSize = 64 + DirentSize = 12 +) + +type superblock struct { + Magic uint32 + Checksum uint32 + FeatureCompat uint32 + BlockSizeBits uint8 + ExtSlots uint8 + RootNid uint16 + Inodes uint64 + BuildTime uint64 + BuildTimeNsec uint32 + Blocks uint32 + MetaBlockAddr uint32 + XattrBlockAddr uint32 + UUID [16]uint8 + VolumeName [16]uint8 + FeatureIncompat uint32 + Union1 uint16 + ExtraDevices uint16 + DevTableSlotOff uint16 + Reserved [38]uint8 +} + +/* +// checkRange checks whether the range [off, off+n) is valid. +func (i *Image) checkRange(off, n uint64) bool { + size := uint64(len(i.bytes)) + end := off + n + return off < size && off <= end && end <= size +} + +// BytesAt returns the bytes at [off, off+n) of the image. +func (i *Image) BytesAt(off, n uint64) ([]byte, error) { + if ok := i.checkRange(off, n); !ok { + //log.Warningf("Invalid byte range (off: 0x%x, n: 0x%x) for image (size: 0x%x)", off, n, len(i.bytes)) + return nil, linuxerr.EFAULT + } + return i.bytes[off : off+n], nil +} + +// unmarshalAt deserializes data from the bytes at [off, off+n) of the image. +func (i *Image) unmarshalAt(data marshal.Marshallable, off uint64) error { + bytes, err := i.BytesAt(off, uint64(data.SizeBytes())) + if err != nil { + //log.Warningf("Failed to deserialize %T from 0x%x.", data, off) + return err + } + data.UnmarshalUnsafe(bytes) + return nil +} + +// initSuperBlock initializes the superblock of this image. +func (i *Image) initSuperBlock() error { + // i.sb is used in the hot path. Let's save a copy of the superblock. + if err := i.unmarshalAt(&i.sb, SuperBlockOffset); err != nil { + return fmt.Errorf("image size is too small") + } + + if i.sb.Magic != SuperBlockMagicV1 { + return fmt.Errorf("unknown magic: 0x%x", i.sb.Magic) + } + + if err := i.verifyChecksum(); err != nil { + return err + } + + if featureIncompat := i.sb.FeatureIncompat & ^uint32(FeatureIncompatSupported); featureIncompat != 0 { + return fmt.Errorf("unsupported incompatible features detected: 0x%x", featureIncompat) + } + + if i.BlockSize()%hostarch.PageSize != 0 { + return fmt.Errorf("unsupported block size: 0x%x", i.BlockSize()) + } + + return nil +} + +// verifyChecksum verifies the checksum of the superblock. +func (i *Image) verifyChecksum() error { + if i.sb.FeatureCompat&FeatureCompatSuperBlockChecksum == 0 { + return nil + } + + sb := i.sb + sb.Checksum = 0 + table := crc32.MakeTable(crc32.Castagnoli) + checksum := crc32.Checksum(marshal.Marshal(&sb), table) +// unmarshalAt deserializes data from the bytes at [off, off+n) of the image. +func (i *Image) unmarshalAt(data marshal.Marshallable, off uint64) error { + bytes, err := i.BytesAt(off, uint64(data.SizeBytes())) + if err != nil { + log.Warningf("Failed to deserialize %T from 0x%x.", data, off) + return err + } + data.UnmarshalUnsafe(bytes) + return nil +} + off := SuperBlockOffset + uint64(i.sb.SizeBytes()) + if bytes, err := i.BytesAt(off, uint64(i.BlockSize())-off); err != nil { + return fmt.Errorf("image size is too small") + } else { + checksum = ^crc32.Update(checksum, table, bytes) + } + if checksum != i.sb.Checksum { + return fmt.Errorf("invalid checksum: 0x%x, expected: 0x%x", checksum, i.sb.Checksum) + } + + return nil +} +*/ + +func parseSuperblock(b []byte) (*superblock, error) { + if len(b) != superblockSize { + return nil, errors.Errorf("superblock had %d bytes instead of expected %d", len(b), superblockSize) + } + + magic := binary.LittleEndian.Uint32(b[0:4]) + if magic != superblockMagic { + return nil, errors.Errorf("superblock had magic of %d instead of expected %d", magic, superblockMagic) + } + + // FIXME: also verify checksum + + s := &superblock{ + Magic: magic, // b[0:4] + Checksum: binary.LittleEndian.Uint32(b[4:8]), + FeatureCompat: binary.LittleEndian.Uint32(b[8:12]), + BlockSizeBits: b[12], // b[12:13] + ExtSlots: b[13], // b[13:14] + RootNid: binary.LittleEndian.Uint16(b[14:16]), + Inodes: binary.LittleEndian.Uint64(b[16:24]), + BuildTime: binary.LittleEndian.Uint64(b[24:32]), + BuildTimeNsec: binary.LittleEndian.Uint32(b[32:36]), + Blocks: binary.LittleEndian.Uint32(b[36:40]), + MetaBlockAddr: binary.LittleEndian.Uint32(b[40:44]), + XattrBlockAddr: binary.LittleEndian.Uint32(b[44:48]), + UUID: [16]byte(b[48:64]), + VolumeName: [16]byte(b[64:80]), + FeatureIncompat: binary.LittleEndian.Uint32(b[80:84]), + Union1: binary.LittleEndian.Uint16(b[84:86]), + ExtraDevices: binary.LittleEndian.Uint16(b[86:88]), + DevTableSlotOff: binary.LittleEndian.Uint16(b[88:90]), + Reserved: [38]byte(b[90:128]), + } + + return s, nil +} + +func readSuperblock(path string) (*superblock, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + defer reader.Close() + + buf := make([]byte, superblockOffset+superblockSize) + if _, err := io.ReadFull(reader, buf); err != nil { + return nil, err + } + + return parseSuperblock(buf[superblockOffset:]) +} diff --git a/erofs/verity.go b/erofs/verity.go new file mode 100644 index 0000000..d0ca4d4 --- /dev/null +++ b/erofs/verity.go @@ -0,0 +1,519 @@ +package erofs + +// #cgo pkg-config: libcryptsetup devmapper --static +// #include +// #include +// #include +// #include +/* +int get_verity_params(char *device, char **params) +{ + struct dm_task *dmt; + struct dm_info dmi; + int r; + uint64_t start, length; + char *type, *tmpParams; + + dmt = dm_task_create(DM_DEVICE_TABLE); + if (!dmt) + return 1; + + r = 2; + if (!dm_task_secure_data(dmt)) + goto out; + + r = 3; + if (!dm_task_set_name(dmt, device)) + goto out; + + r = 4; + if (!dm_task_run(dmt)) + goto out; + + r = 5; + if (!dm_task_get_info(dmt, &dmi)) + goto out; + + r = 6; + if (!dmi.exists) + goto out; + + r = 7; + if (dmi.target_count <= 0) + goto out; + + r = 8; + dm_get_next_target(dmt, NULL, &start, &length, &type, &tmpParams); + if (!type) + goto out; + + r = 9; + if (strcasecmp(type, CRYPT_VERITY)) { + fprintf(stderr, "type: %s (%s) %d\n", type, CRYPT_VERITY, strcmp(type, CRYPT_VERITY)); + goto out; + } + *params = strdup(tmpParams); + + r = 0; +out: + dm_task_destroy(dmt); + return r; +} +*/ +import "C" + +import ( + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + "unsafe" + + "github.com/freddierice/go-losetup" + "github.com/martinjungblut/go-cryptsetup" + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/mount" +) + +const VerityRootHashAnnotation = "io.stackeroci.stacker.erofs_verity_root_hash" + +type verityDeviceType struct { + Flags uint + DataDevice string + HashOffset uint64 +} + +func (verity verityDeviceType) Name() string { + return C.CRYPT_VERITY +} + +func (verity verityDeviceType) Unmanaged() (unsafe.Pointer, func()) { + var cParams C.struct_crypt_params_verity + + cParams.hash_name = C.CString("sha256") + cParams.data_device = C.CString(verity.DataDevice) + cParams.fec_device = nil + cParams.fec_roots = 0 + + cParams.salt_size = 32 // DEFAULT_VERITY_SALT_SIZE for x86 + cParams.salt = nil + + // these can't be larger than a page size, but we want them to be as + // big as possible so the hash data is small, so let's set them to a + // page size. + cParams.data_block_size = C.uint(os.Getpagesize()) + cParams.hash_block_size = C.uint(os.Getpagesize()) + + cParams.data_size = C.ulong(verity.HashOffset / uint64(os.Getpagesize())) + cParams.hash_area_offset = C.ulong(verity.HashOffset) + cParams.fec_area_offset = 0 + cParams.hash_type = 1 // use format version 1 (i.e. "modern", non chrome-os) + cParams.flags = C.uint(verity.Flags) + + deallocate := func() { + C.free(unsafe.Pointer(cParams.hash_name)) + C.free(unsafe.Pointer(cParams.data_device)) + } + + return unsafe.Pointer(&cParams), deallocate +} + +func isCryptsetupEINVAL(err error) bool { + cse, ok := err.(*cryptsetup.Error) + return ok && cse.Code() == -22 +} + +var cryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") + +func appendVerityData(file string) (string, error) { + fi, err := os.Lstat(file) + if err != nil { + return "", errors.WithStack(err) + } + + verityOffset := fi.Size() + + fmt.Printf("VERIFY1 - offset:+%v\n", verityOffset) + + // we expect mkerofs to have padded the file to the nearest 4k + // (dm-verity requires device block size, which is 512 for loopback, + // which is a multiple of 4k), let's check that here + if verityOffset%512 != 0 { + return "", errors.Errorf("bad verity file size %d", verityOffset) + } + + verityDevice, err := cryptsetup.Init(file) + if err != nil { + return "", errors.WithStack(err) + } + + fmt.Printf("VERIFY1 - offset:%+v dev:%+v\n", verityOffset, verityDevice) + + verityType := verityDeviceType{ + Flags: cryptsetup.CRYPT_VERITY_CREATE_HASH, + DataDevice: file, + HashOffset: uint64(verityOffset), + } + err = verityDevice.Format(verityType, cryptsetup.GenericParams{}) + if err != nil { + return "", errors.WithStack(err) + } + + // a bit ugly, but this is the only API for querying the root + // hash (short of invoking the veritysetup binary), and it was + // added in libcryptsetup commit 188cb114af94 ("Add support for + // verity in crypt_volume_key_get and use it in status"), which + // is relatively recent (ubuntu 20.04 does not have this patch, + // for example). + // + // before that, we get a -22. so, let's test for that and + // render a special error message. + rootHash, _, err := verityDevice.VolumeKeyGet(cryptsetup.CRYPT_ANY_SLOT, "") + if isCryptsetupEINVAL(err) { + return "", cryptsetupTooOld + } else if err != nil { + return "", err + } + + return fmt.Sprintf("%x", rootHash), errors.WithStack(err) +} + +func verityDataLocation(sblock *superblock) (uint64, error) { + return uint64((1 << sblock.BlockSizeBits) * sblock.Blocks), nil +} + +func verityName(p string) string { + return fmt.Sprintf("%s-%s", p, veritySuffix) +} + +func fileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} + +// Mount a filesystem as container root, without host root +// privileges. We do this using squashfuse. +func GuestMount(erofsFile string, mountpoint string) error { + if isMountpoint(mountpoint) { + return errors.Errorf("%s is already mounted", mountpoint) + } + + abs, err := filepath.Abs(erofsFile) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", erofsFile, err) + } + erofsFile = abs + + abs, err = filepath.Abs(mountpoint) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) + } + mountpoint = abs + + cmd, err := erofsFuse(erofsFile, mountpoint) + if err != nil { + return err + } + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process after guestmount %s: %v", erofsFile, err) + } + return nil +} + +func isMountpoint(dest string) bool { + mounted, err := mount.IsMountpoint(dest) + return err == nil && mounted +} + +// Takes /proc/self/uid_map contents as one string +// Returns true if this is a uidmap representing the whole host +// uid range. +func uidmapIsHost(oneline string) bool { + oneline = strings.TrimSuffix(oneline, "\n") + if len(oneline) == 0 { + return false + } + lines := strings.Split(oneline, "\n") + if len(lines) != 1 { + return false + } + words := strings.Fields(lines[0]) + if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { + return false + } + + return true +} + +func amHostRoot() bool { + // if not uid 0, not host root + if os.Geteuid() != 0 { + return false + } + // if uid_map doesn't map 0 to 0, not host root + bytes, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + return false + } + return uidmapIsHost(string(bytes)) +} + +func Mount(erofs, mountpoint, rootHash string) error { + if !amHostRoot() { + return GuestMount(erofs, mountpoint) + } + err := HostMount(erofs, mountpoint, rootHash) + if err == nil || rootHash != "" { + return err + } + return GuestMount(erofs, mountpoint) +} + +func HostMount(erofs string, mountpoint string, rootHash string) error { + fi, err := os.Stat(erofs) + if err != nil { + return errors.WithStack(err) + } + + sblock, err := readSuperblock(erofs) + if err != nil { + return err + } + + verityOffset, err := verityDataLocation(sblock) + if err != nil { + return err + } + + if verityOffset == uint64(fi.Size()) && rootHash != "" { + return errors.Errorf("asked for verity but no data present") + } + + if rootHash == "" && verityOffset != uint64(fi.Size()) { + return errors.Errorf("verity data present but no root hash specified") + } + + mountSourcePath := "" + + var verityDevice *cryptsetup.Device + name := verityName(path.Base(erofs)) + + loopDevNeedsClosedOnErr := false + var loopDev losetup.Device + + // set up the verity device if necessary + if rootHash != "" { + verityDevPath := path.Join("/dev/mapper", name) + mountSourcePath = verityDevPath + _, err = os.Stat(verityDevPath) + if err != nil { + if !os.IsNotExist(err) { + return errors.WithStack(err) + } + + loopDev, err = losetup.Attach(erofs, 0, true) + if err != nil { + return errors.WithStack(err) + } + loopDevNeedsClosedOnErr = true + + verityDevice, err = cryptsetup.Init(loopDev.Path()) + if err != nil { + return errors.WithStack(err) + } + + verityType := verityDeviceType{ + Flags: 0, + DataDevice: loopDev.Path(), + HashOffset: verityOffset, + } + + err = verityDevice.Load(verityType) + if err != nil { + _ = loopDev.Detach() + return errors.WithStack(err) + } + + // each string byte hex encodes four bits of info... + volumeKeySizeInBytes := len(rootHash) * 4 / 8 + rootHashBytes, err := hex.DecodeString(rootHash) + if err != nil { + _ = loopDev.Detach() + return errors.WithStack(err) + } + + if len(rootHashBytes) != volumeKeySizeInBytes { + _ = loopDev.Detach() + return errors.Errorf("unexpected key size for %s", rootHash) + } + + err = verityDevice.ActivateByVolumeKey(name, string(rootHashBytes), volumeKeySizeInBytes, cryptsetup.CRYPT_ACTIVATE_READONLY) + if err != nil { + _ = loopDev.Detach() + return errors.WithStack(err) + } + } else { + err = ConfirmExistingVerityDeviceHash(verityDevPath, rootHash, rejectVerityFailure) + if err != nil { + return err + } + } + } else { + loopDev, err = losetup.Attach(erofs, 0, true) + if err != nil { + return errors.WithStack(err) + } + defer func() { _ = loopDev.Detach() }() + mountSourcePath = loopDev.Path() + + } + + err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, "erofs", unix.MS_RDONLY, "")) + if err != nil { + if verityDevice != nil { + _ = verityDevice.Deactivate(name) + _ = loopDev.Detach() + } + if loopDevNeedsClosedOnErr { + _ = loopDev.Detach() + } + return err + } + return nil +} + +func findLoopBackingVerity(device string) (int64, error) { + fi, err := os.Stat(device) + if err != nil { + return -1, errors.WithStack(err) + } + + var minor uint32 + switch stat := fi.Sys().(type) { + case *unix.Stat_t: + minor = unix.Minor(uint64(stat.Rdev)) + case *syscall.Stat_t: + minor = unix.Minor(uint64(stat.Rdev)) + default: + return -1, errors.Errorf("unknown stat info type %T", stat) + } + + ents, err := os.ReadDir(fmt.Sprintf("/sys/block/dm-%d/slaves", minor)) + if err != nil { + return -1, errors.WithStack(err) + } + + if len(ents) != 1 { + return -1, errors.Errorf("too many slaves for %v", device) + } + loop := ents[0] + + deviceNo, err := strconv.ParseInt(strings.TrimPrefix(filepath.Base(loop.Name()), "loop"), 10, 64) + if err != nil { + return -1, errors.Wrapf(err, "bad loop dev %v", loop.Name()) + } + + return deviceNo, nil +} + +func Umount(mountpoint string) error { + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return err + } + + // first, find the verity device that backs the mount + theMount, found := mounts.FindMount(mountpoint) + if !found { + return errors.Errorf("%s is not a mountpoint", mountpoint) + } + + err = unix.Unmount(mountpoint, 0) + if err != nil { + return errors.Wrapf(err, "failed unmounting %v", mountpoint) + } + + if _, err := os.Stat(theMount.Source); err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WithStack(err) + } + + // was this a verity mount or a regular loopback mount? (if it's a + // regular loopback mount, we detached it above, so need to do anything + // special here; verity doesn't play as nicely) + if strings.HasSuffix(theMount.Source, veritySuffix) { + // find the loop device that backs the verity device + deviceNo, err := findLoopBackingVerity(theMount.Source) + if err != nil { + return err + } + + loopDev := losetup.New(uint64(deviceNo), 0) + // here, we don't have the loopback device any more (we detached it + // above). the cryptsetup API allows us to pass NULL for the crypt + // device, but go-cryptsetup doesn't have a way to initialize a NULL + // crypt device short of making the struct by hand like this. + err = (&cryptsetup.Device{}).Deactivate(theMount.Source) + if err != nil { + return errors.WithStack(err) + } + + // finally, kill the loop dev + err = loopDev.Detach() + if err != nil { + return errors.Wrapf(err, "failed to detach loop dev for %v", theMount.Source) + } + } + + return nil +} + +// If we are using squashfuse, then we will be unable to get verity has from +// the mount device. This is not a safe thing, we we only allow it when the +// device was mounted originally with AllowMissingVerityData. + +const ( + rejectVerityFailure = false + allowVerityFailure = false +) + +func ConfirmExistingVerityDeviceHash(devicePath string, rootHash string, allowVerityFailure bool) error { + device := filepath.Base(devicePath) + cDevice := C.CString(device) + defer C.free(unsafe.Pointer(cDevice)) + + var cParams *C.char + + rc := C.get_verity_params(cDevice, &cParams) + if rc != 0 { + if allowVerityFailure { + return nil + } + return errors.Errorf("problem getting hash from %v: %v", device, rc) + } + defer C.free(unsafe.Pointer(cParams)) + + params := C.GoString(cParams) + + // https://gitlab.com/cryptsetup/cryptsetup/-/wikis/DMVerity + fields := strings.Fields(params) + if len(fields) < 10 { + return errors.Errorf("invalid dm params for %v: %v", device, params) + } + + if rootHash != fields[8] { + return errors.Errorf("invalid root hash for %v: %v (expected: %v)", device, fields[7], rootHash) + } + + return nil +} diff --git a/erofs/verity_static.go b/erofs/verity_static.go new file mode 100644 index 0000000..549c5d1 --- /dev/null +++ b/erofs/verity_static.go @@ -0,0 +1,10 @@ +//go:build static_build +// +build static_build + +package erofs + +// cryptsetup's pkgconfig is broken (it does not set Requires.private or +// Libs.private at all), so we do the LDLIBS for it by hand. + +// #cgo LDFLAGS: -lcryptsetup -lcrypto -lssl -lblkid -luuid -ljson-c -lpthread -ldl +import "C" diff --git a/erofs/verity_test.go b/erofs/verity_test.go new file mode 100644 index 0000000..ff9f1b5 --- /dev/null +++ b/erofs/verity_test.go @@ -0,0 +1,135 @@ +package erofs + +import ( + "fmt" + "io" + "os" + "os/exec" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +type uidmapTestcase struct { + uidmap string + expected bool +} + +var uidmapTests = []uidmapTestcase{ + { + uidmap: ` 0 0 4294967295`, + expected: true, + }, + { + uidmap: ` 0 0 1000 +2000 2000 1`, + expected: false, + }, + { + uidmap: ` 0 0 1000`, + expected: false, + }, + { + uidmap: ` 10 0 4294967295`, + expected: false, + }, + { + uidmap: ` 0 10 4294967295`, + expected: false, + }, + { + uidmap: ` 0 0 1`, + expected: false, + }, +} + +func TestAmHostRoot(t *testing.T) { + t.Parallel() + assert := assert.New(t) + for _, testcase := range uidmapTests { + v := uidmapIsHost(testcase.uidmap) + assert.Equal(v, testcase.expected) + } +} + +func TestVerityMetadata(t *testing.T) { + assert := assert.New(t) + + rootfs, err := os.MkdirTemp("", "stacker_verity_test_rootfs") + assert.NoError(err) + defer os.RemoveAll(rootfs) + + tempdir, err := os.MkdirTemp("", "stacker_verity_test_tempdir") + assert.NoError(err) + defer os.RemoveAll(tempdir) + + err = os.WriteFile(path.Join(rootfs, "foo"), []byte("bar"), 0644) + assert.NoError(err) + + reader, _, rootHash, err := MakeErofs(tempdir, rootfs, nil, VerityMetadataPresent) + if err == cryptsetupTooOld { + t.Skip("libcryptsetup too old") + } + assert.NoError(err) + + content, err := io.ReadAll(reader) + assert.NoError(err) + erofsFile := path.Join(tempdir, "foo.erofs") + err = os.WriteFile(erofsFile, content, 0600) + assert.NoError(err) + + sblock, err := readSuperblock(erofsFile) + assert.NoError(err) + + verityOffset, err := verityDataLocation(sblock) + assert.NoError(err) + + // now let's try to verify it at least in userspace. exec cryptsetup + // because i'm lazy and it's only in tests + cmd := exec.Command("veritysetup", "verify", erofsFile, erofsFile, rootHash, + "--hash-offset", fmt.Sprintf("%d", verityOffset)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + assert.NoError(err) + + // what if we fiddle with the verity data? note that we have to fiddle + // with the beginning of the verity block, which will be 4k long for + // our small erofs file, because the stuff at the end of the verity + // block is unused. + const bytesToFlip = 2 + const flipAtOffset = -4087 + + f, err := os.OpenFile(erofsFile, os.O_RDWR, 0644) + assert.NoError(err) + defer f.Close() + _, err = f.Seek(flipAtOffset, os.SEEK_END) + assert.NoError(err) + + buf := make([]byte, bytesToFlip) + n, err := f.Read(buf) + assert.Equal(n, bytesToFlip) + assert.NoError(err) + + for i := range buf { + buf[i] = buf[i] ^ 0xff + } + + _, err = f.Seek(flipAtOffset, os.SEEK_END) + assert.NoError(err) + n, err = f.Write(buf) + assert.Equal(n, bytesToFlip) + assert.NoError(err) + assert.NoError(f.Sync()) + assert.NoError(f.Close()) + + cmd = exec.Command("veritysetup", "verify", erofsFile, erofsFile, rootHash, + "--hash-offset", fmt.Sprintf("%d", verityOffset)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + assert.Error(err) +} From 01d76b1709eb76de8bbe5ab5e732fab22472880a Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 3 Oct 2024 20:46:15 +0000 Subject: [PATCH 2/2] refactor: decouple verity from filesystem interfaces We are going to support multiple underlying filesystems (squash, erofs and maybe more). As long as they are filesystem image blobs, verity data can be appended. Signed-off-by: Ramkumar Chinchani --- cmd/atomfs/mount.go | 2 +- cmd/atomfs/umount.go | 2 +- molecule.go | 4 +- pkg/common/common.go | 11 ++++++ {erofs => pkg/erofs}/erofs.go | 4 +- pkg/erofs/fs.go | 38 +++++++++++++++++++ {erofs => pkg/erofs}/mediatype.go | 0 {erofs => pkg/erofs}/superblock.go | 0 {erofs => pkg/erofs}/verity.go | 2 +- {erofs => pkg/erofs}/verity_static.go | 0 {erofs => pkg/erofs}/verity_test.go | 0 pkg/fs/fs.go | 30 +++++++++++++++ {log => pkg/log}/log.go | 0 {mount => pkg/mount}/mountinfo.go | 0 pkg/squashfs/fs.go | 41 +++++++++++++++++++++ {squashfs => pkg/squashfs}/mediatype.go | 12 ++---- {squashfs => pkg/squashfs}/squashfs.go | 21 +++++++++-- {squashfs => pkg/squashfs}/superblock.go | 0 {squashfs => pkg/squashfs}/verity_static.go | 0 {squashfs => pkg/squashfs}/verity_test.go | 0 pkg/types/types.go | 32 ++++++++++++++++ verity/mediatype.go | 10 +++++ {squashfs => verity}/verity.go | 29 +++------------ 23 files changed, 195 insertions(+), 43 deletions(-) create mode 100644 pkg/common/common.go rename {erofs => pkg/erofs}/erofs.go (99%) create mode 100644 pkg/erofs/fs.go rename {erofs => pkg/erofs}/mediatype.go (100%) rename {erofs => pkg/erofs}/superblock.go (100%) rename {erofs => pkg/erofs}/verity.go (99%) rename {erofs => pkg/erofs}/verity_static.go (100%) rename {erofs => pkg/erofs}/verity_test.go (100%) create mode 100644 pkg/fs/fs.go rename {log => pkg/log}/log.go (100%) rename {mount => pkg/mount}/mountinfo.go (100%) create mode 100644 pkg/squashfs/fs.go rename {squashfs => pkg/squashfs}/mediatype.go (71%) rename {squashfs => pkg/squashfs}/squashfs.go (97%) rename {squashfs => pkg/squashfs}/superblock.go (100%) rename {squashfs => pkg/squashfs}/verity_static.go (100%) rename {squashfs => pkg/squashfs}/verity_test.go (100%) create mode 100644 pkg/types/types.go create mode 100644 verity/mediatype.go rename {squashfs => verity}/verity.go (95%) diff --git a/cmd/atomfs/mount.go b/cmd/atomfs/mount.go index d0ea6b6..cf80a13 100644 --- a/cmd/atomfs/mount.go +++ b/cmd/atomfs/mount.go @@ -12,7 +12,7 @@ import ( "github.com/urfave/cli" "golang.org/x/sys/unix" "machinerun.io/atomfs" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/squashfs" ) var mountCmd = cli.Command{ diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go index a60a4c1..d650962 100644 --- a/cmd/atomfs/umount.go +++ b/cmd/atomfs/umount.go @@ -7,7 +7,7 @@ import ( "syscall" "github.com/urfave/cli" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/mount" ) var umountCmd = cli.Command{ diff --git a/molecule.go b/molecule.go index 5ba3497..6823faf 100644 --- a/molecule.go +++ b/molecule.go @@ -9,8 +9,8 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sys/unix" - "machinerun.io/atomfs/mount" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/mount" + "machinerun.io/atomfs/pkg/squashfs" ) type Molecule struct { diff --git a/pkg/common/common.go b/pkg/common/common.go new file mode 100644 index 0000000..c83afc6 --- /dev/null +++ b/pkg/common/common.go @@ -0,0 +1,11 @@ +package common + +import "os" + +func FileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} diff --git a/erofs/erofs.go b/pkg/erofs/erofs.go similarity index 99% rename from erofs/erofs.go rename to pkg/erofs/erofs.go index e088b71..86d37bc 100644 --- a/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -18,8 +18,8 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" "golang.org/x/sys/unix" - "machinerun.io/atomfs/log" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/log" + "machinerun.io/atomfs/pkg/mount" ) var checkZstdSupported sync.Once diff --git a/pkg/erofs/fs.go b/pkg/erofs/fs.go new file mode 100644 index 0000000..13ff3d5 --- /dev/null +++ b/pkg/erofs/fs.go @@ -0,0 +1,38 @@ +package erofs + +import ( + "io" + + "machinerun.io/atomfs/verity" +) + +type erofs struct { +} + +func New() *erofs { + return &erofs{} +} + +func (fs *erofs) Make(tempdir string, rootfs string, eps *ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) { +} + +// Mount a filesystem as container root, without host root privileges. +func (fs *erofs) GuestMount(fsFile string, mountpoint string) error { + return nil +} + +func (fs *erofs) Mount(fsFile, mountpoint, rootHash string) error { + return nil +} + +func (fs *erofs) HostMount(fsFile string, mountpoint string, rootHash string) error { + return nil +} + +func (fs *erofs) Umount(mountpoint string) error { + return nil +} + +func (fs *erofs) VerityDataLocation() uint64 { + return 0 +} diff --git a/erofs/mediatype.go b/pkg/erofs/mediatype.go similarity index 100% rename from erofs/mediatype.go rename to pkg/erofs/mediatype.go diff --git a/erofs/superblock.go b/pkg/erofs/superblock.go similarity index 100% rename from erofs/superblock.go rename to pkg/erofs/superblock.go diff --git a/erofs/verity.go b/pkg/erofs/verity.go similarity index 99% rename from erofs/verity.go rename to pkg/erofs/verity.go index d0ca4d4..d678a90 100644 --- a/erofs/verity.go +++ b/pkg/erofs/verity.go @@ -77,7 +77,7 @@ import ( "github.com/martinjungblut/go-cryptsetup" "github.com/pkg/errors" "golang.org/x/sys/unix" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/mount" ) const VerityRootHashAnnotation = "io.stackeroci.stacker.erofs_verity_root_hash" diff --git a/erofs/verity_static.go b/pkg/erofs/verity_static.go similarity index 100% rename from erofs/verity_static.go rename to pkg/erofs/verity_static.go diff --git a/erofs/verity_test.go b/pkg/erofs/verity_test.go similarity index 100% rename from erofs/verity_test.go rename to pkg/erofs/verity_test.go diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go new file mode 100644 index 0000000..36941c0 --- /dev/null +++ b/pkg/fs/fs.go @@ -0,0 +1,30 @@ +package fs + +import ( + "machinerun.io/atomfs/pkg/erofs" + "machinerun.io/atomfs/pkg/squashfs" + "machinerun.io/atomfs/pkg/types" +) + +// New creates a filesystem instance. +func New(fsType types.FilesystemType) types.Filesystem { + switch fsType { + case types.Squashfs: + return squashfs.New() + case types.Erofs: + return erofs.New() + default: + return nil + } +} + +// NewFromMediaType creates a filesystem instance based on media-type. +func NewFromMediaType(mediaType string) types.Filesystem { + if squashfs.IsSquashfsMediaType(mediaType) { + return New(types.Squashfs) + } else if erofs.IsErofsMediaType(mediaType) { + return New(types.Erofs) + } + + return nil +} diff --git a/log/log.go b/pkg/log/log.go similarity index 100% rename from log/log.go rename to pkg/log/log.go diff --git a/mount/mountinfo.go b/pkg/mount/mountinfo.go similarity index 100% rename from mount/mountinfo.go rename to pkg/mount/mountinfo.go diff --git a/pkg/squashfs/fs.go b/pkg/squashfs/fs.go new file mode 100644 index 0000000..49c8451 --- /dev/null +++ b/pkg/squashfs/fs.go @@ -0,0 +1,41 @@ +package squashfs + +import ( + "io" + + "machinerun.io/atomfs/verity" +) + +type squashfs struct { +} + +func New() *squashfs { + return &squashfs{} +} + +func (fs *squashfs) Make(tempdir string, rootfs string, eps *ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) { +} + +// Mount a filesystem as container root, without host root privileges. +func (fs *squashfs) GuestMount(fsFile string, mountpoint string) error { + return nil +} + +func (fs *squashfs) Mount(fsFile, mountpoint, rootHash string) error { + return nil +} + +func (fs *squashfs) HostMount(fsFile string, mountpoint string, rootHash string) error { + return nil +} + +func (fs *squashfs) Umount(mountpoint string) error { + return nil +} + +func (fs *squashfs) VerityDataLocation() uint64 { + return 0 +} + +func (fs *squashfs) ExtractSingle(fsFile string, extractDir string) error { +} diff --git a/squashfs/mediatype.go b/pkg/squashfs/mediatype.go similarity index 71% rename from squashfs/mediatype.go rename to pkg/squashfs/mediatype.go index 051fe9b..5c33c95 100644 --- a/squashfs/mediatype.go +++ b/pkg/squashfs/mediatype.go @@ -3,21 +3,17 @@ package squashfs import ( "fmt" "strings" + + "machinerun.io/atomfs/verity" ) type SquashfsCompression string -type VerityMetadata bool const ( BaseMediaTypeLayerSquashfs = "application/vnd.stacker.image.layer.squashfs" GzipCompression SquashfsCompression = "gzip" ZstdCompression SquashfsCompression = "zstd" - - veritySuffix = "verity" - - VerityMetadataPresent VerityMetadata = true - VerityMetadataMissing VerityMetadata = false ) func IsSquashfsMediaType(mediaType string) bool { @@ -27,11 +23,11 @@ func IsSquashfsMediaType(mediaType string) bool { func GenerateSquashfsMediaType(comp SquashfsCompression, verity VerityMetadata) string { verityString := "" if verity { - verityString = fmt.Sprintf("+%s", veritySuffix) + verityString = fmt.Sprintf("+%s", verity.VeritySuffix) } return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerSquashfs, comp, verityString) } func HasVerityMetadata(mediaType string) VerityMetadata { - return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix)) + return VerityMetadata(strings.HasSuffix(mediaType, verity.VeritySuffix)) } diff --git a/squashfs/squashfs.go b/pkg/squashfs/squashfs.go similarity index 97% rename from squashfs/squashfs.go rename to pkg/squashfs/squashfs.go index 328b061..f7cdebd 100644 --- a/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -18,8 +18,10 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" "golang.org/x/sys/unix" - "machinerun.io/atomfs/log" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/log" + "machinerun.io/atomfs/pkg/mount" + _ "machinerun.io/atomfs/verity" ) var checkZstdSupported sync.Once @@ -168,7 +170,7 @@ func MakeSquashfs(tempdir string, rootfs string, eps *ExcludePaths, verity Verit } if verity { - rootHash, err = appendVerityData(tmpSquashfs.Name()) + rootHash, err = verity.AppendVerityData(tmpSquashfs.Name()) if err != nil { return nil, "", rootHash, err } @@ -366,7 +368,7 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { } } } - for count := 0; !fileChanged(fiPre, extractDir); count++ { + for count := 0; !common.FileChanged(fiPre, extractDir); count++ { if cmd.ProcessState != nil { // process exited, the Wait() call in the goroutine above // caused ProcessState to be populated. @@ -742,3 +744,14 @@ func whichSearch(name string, paths []string) string { return "" } + +func verityDataLocation(sblock *superblock) (uint64, error) { + squashLen := sblock.size + + // squashfs is padded out to the nearest 4k + if squashLen%4096 != 0 { + squashLen = squashLen + (4096 - squashLen%4096) + } + + return squashLen, nil +} diff --git a/squashfs/superblock.go b/pkg/squashfs/superblock.go similarity index 100% rename from squashfs/superblock.go rename to pkg/squashfs/superblock.go diff --git a/squashfs/verity_static.go b/pkg/squashfs/verity_static.go similarity index 100% rename from squashfs/verity_static.go rename to pkg/squashfs/verity_static.go diff --git a/squashfs/verity_test.go b/pkg/squashfs/verity_test.go similarity index 100% rename from squashfs/verity_test.go rename to pkg/squashfs/verity_test.go diff --git a/pkg/types/types.go b/pkg/types/types.go new file mode 100644 index 0000000..ccdf95c --- /dev/null +++ b/pkg/types/types.go @@ -0,0 +1,32 @@ +package types + +import ( + "io" + + "machinerun.io/atomfs/verity" +) + +type Filesystem interface { + // Make a filesystem image. + Make(tempdir string, rootfs string, eps *ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) + + // Mount a filesystem as container root, without host root privileges. + GuestMount(fsFile string, mountpoint string) error + + Mount(fs, mountpoint, rootHash string) error + + HostMount(fs string, mountpoint string, rootHash string) error + + Umount(mountpoint string) error + + VerityDataLocation() uint64 + + ExtractSingle(fsFile string, extractDir string) error +} + +type FilesystemType string + +const ( + Squashfs FilesystemType = "squashfs" + Erofs FilesystemType = "erofs" +) diff --git a/verity/mediatype.go b/verity/mediatype.go new file mode 100644 index 0000000..6333e79 --- /dev/null +++ b/verity/mediatype.go @@ -0,0 +1,10 @@ +package verity + +type VerityMetadata bool + +const ( + VeritySuffix = "verity" + + VerityMetadataPresent VerityMetadata = true + VerityMetadataMissing VerityMetadata = false +) diff --git a/squashfs/verity.go b/verity/verity.go similarity index 95% rename from squashfs/verity.go rename to verity/verity.go index 6642f86..b81275b 100644 --- a/squashfs/verity.go +++ b/verity/verity.go @@ -1,4 +1,4 @@ -package squashfs +package verity // #cgo pkg-config: libcryptsetup devmapper --static // #include @@ -77,7 +77,7 @@ import ( "github.com/martinjungblut/go-cryptsetup" "github.com/pkg/errors" "golang.org/x/sys/unix" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/mount" ) const VerityRootHashAnnotation = "io.stackeroci.stacker.squashfs_verity_root_hash" @@ -130,7 +130,7 @@ func isCryptsetupEINVAL(err error) bool { var cryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") -func appendVerityData(file string) (string, error) { +func AppendVerityData(file string) (string, error) { fi, err := os.Lstat(file) if err != nil { return "", errors.WithStack(err) @@ -179,27 +179,8 @@ func appendVerityData(file string) (string, error) { return fmt.Sprintf("%x", rootHash), errors.WithStack(err) } -func verityDataLocation(sblock *superblock) (uint64, error) { - squashLen := sblock.size - - // squashfs is padded out to the nearest 4k - if squashLen%4096 != 0 { - squashLen = squashLen + (4096 - squashLen%4096) - } - - return squashLen, nil -} - func verityName(p string) string { - return fmt.Sprintf("%s-%s", p, veritySuffix) -} - -func fileChanged(a os.FileInfo, path string) bool { - b, err := os.Lstat(path) - if err != nil { - return true - } - return !os.SameFile(a, b) + return fmt.Sprintf("%s-%s", p, VeritySuffix) } // Mount a filesystem as container root, without host root @@ -454,7 +435,7 @@ func Umount(mountpoint string) error { // was this a verity mount or a regular loopback mount? (if it's a // regular loopback mount, we detached it above, so need to do anything // special here; verity doesn't play as nicely) - if strings.HasSuffix(theMount.Source, veritySuffix) { + if strings.HasSuffix(theMount.Source, VeritySuffix) { // find the loop device that backs the verity device deviceNo, err := findLoopBackingVerity(theMount.Source) if err != nil {