Skip to content

Commit

Permalink
re-add support for TFS version 4 encodings
Browse files Browse the repository at this point in the history 
This commit returns support for reading and optionally writing TFS version 4
images. 'ops image' commands that read an existing image will read either
version 4 or 5 images, and commands that create images may either accept a
"-4" or "--tfsv4" commandline option or a "TFSv4" config bool to create a
version 4 image.

This support is intended to help ease the transition towards use of TFS
version 5.
  • Loading branch information
@wjhun
wjhun committed Aug 24, 2023
1 parent 1605ee4 commit 5ec5e33
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 47 deletions.
11 changes: 11 additions & 0 deletions cmd/flags_build_image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type BuildImageCommandFlags struct {
DisableArgsCopy bool
CmdEnvs []string
ImageName string
TFSv4 bool
Mounts []string
TargetRoot string
IPAddress string
Expand Down Expand Up @@ -55,6 +56,10 @@ func (flags *BuildImageCommandFlags) MergeToConfig(c *types.Config) (err error)
c.RunConfig.ImageName = flags.ImageName
}

if flags.TFSv4 {
c.TFSv4 = true
}

setNanosBaseImage(c)

if c.RunConfig.ImageName == "" && c.Program != "" {
Expand Down Expand Up @@ -146,6 +151,11 @@ func NewBuildImageCommandFlags(cmdFlags *pflag.FlagSet) (flags *BuildImageComman
exitWithError(err.Error())
}

flags.TFSv4, err = cmdFlags.GetBool("tfsv4")
if err != nil {
exitWithError(err.Error())
}

flags.TargetRoot, err = cmdFlags.GetString("target-root")
if err != nil {
exitWithError(err.Error())
Expand Down Expand Up @@ -209,6 +219,7 @@ func PersistBuildImageCommandFlags(cmdFlags *pflag.FlagSet) {
cmdFlags.StringArrayP("envs", "e", nil, "env arguments")
cmdFlags.StringP("target-root", "r", "", "target root")
cmdFlags.StringP("imagename", "i", "", "image name")
cmdFlags.BoolP("tfsv4", "4", false, "use TFSv4")
cmdFlags.StringArray("mounts", nil, "mount <volume_id:mount_path>")
cmdFlags.StringArrayP("args", "a", nil, "command line arguments")
cmdFlags.BoolP("disable-args-copy", "", false, "disable copying of files passed as arguments")
Expand Down
2 changes: 2 additions & 0 deletions cmd/flags_pkg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ func TestPkgFlagsMergeToConfig(t *testing.T) {
RunConfig: types.RunConfig{
Memory: "2G",
},
TFSv4: true,
}

err = pkgFlags.MergeToConfig(c)
Expand Down Expand Up @@ -137,6 +138,7 @@ func TestPkgFlagsMergeToConfig(t *testing.T) {
Memory: "2G",
ImageName: lepton.GetOpsHome() + "/images/ops",
},
TFSv4: true,
}

assert.Equal(t, expectedConfig, c)
Expand Down
26 changes: 16 additions & 10 deletions fs/mkfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,15 @@ var uefiFileBootaa64 = []byte{

// MkfsCommand wraps mkfs calls
type MkfsCommand struct {
bootPath string
uefiPath string
label string
manifest *Manifest
partitions bool
size int64
outPath string
rootTfs *tfs
bootPath string
uefiPath string
label string
manifest *Manifest
partitions bool
size int64
outPath string
rootTfs *tfs
oldEncoding bool
}

// NewMkfsCommand returns an instance of MkfsCommand
Expand Down Expand Up @@ -210,6 +211,11 @@ func (m *MkfsCommand) SetLabel(label string) {
m.label = label
}

// SetOldEncoding forces use of TFS version 4
func (m *MkfsCommand) SetOldEncoding() {
m.oldEncoding = true
}

// Execute runs mkfs command
func (m *MkfsCommand) Execute() error {
if m.outPath == "" {
Expand Down Expand Up @@ -269,7 +275,7 @@ func (m *MkfsCommand) Execute() error {
if manifest != nil {
manifest.finalize()
if manifest.boot != nil {
_, err = tfsWrite(outFile, outOffset, bootFSSize, "", manifest.boot)
_, err = tfsWrite(outFile, outOffset, bootFSSize, "", manifest.boot, m.oldEncoding)
if err != nil {
return fmt.Errorf("cannot write boot filesystem: %v", err)
}
Expand All @@ -279,7 +285,7 @@ func (m *MkfsCommand) Execute() error {
} else {
root = mkFS()
}
m.rootTfs, err = tfsWrite(outFile, outOffset, 0, m.label, root)
m.rootTfs, err = tfsWrite(outFile, outOffset, 0, m.label, root, m.oldEncoding)
if err != nil {
return fmt.Errorf("cannot write root filesystem: %v", err)
}
Expand Down
111 changes: 75 additions & 36 deletions fs/tfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

const tfsMagic = "NVMTFS"
const tfsVersion = 5
const oldTfsVersion = 4

const logExtensionSize = 1024 * sectorSize

Expand Down Expand Up @@ -62,28 +63,33 @@ type tfs struct {
root *map[string]interface{}
}

func (t *tfs) logInit() error {
func (t *tfs) logInit(oldEncoding bool) error {
if (t.size != 0) && (t.allocated+sectorSize > t.size) {
return fmt.Errorf("available space (%d bytes) too small, required %d", t.size-t.allocated, sectorSize)
}
t.currentExt = t.newLogExt(true)
t.currentExt = t.newLogExt(true, oldEncoding)
t.allocated += sectorSize
return t.logExtend()
}

func (t *tfs) newLogExt(initial bool) *tlogExt {
func (t *tfs) newLogExt(initial bool, oldEncoding bool) *tlogExt {
var extSize uint
if initial {
extSize = sectorSize
} else {
extSize = logExtensionSize
}
logExt := &tlogExt{
offset: t.allocated,
buffer: make([]byte, 0, extSize),
offset: t.allocated,
oldEncoding: oldEncoding,
buffer: make([]byte, 0, extSize),
}
logExt.buffer = append(logExt.buffer, tfsMagic...)
logExt.buffer = appendVarint(logExt.buffer, tfsVersion)
version := uint(tfsVersion)
if oldEncoding {
version = oldTfsVersion
}
logExt.buffer = appendVarint(logExt.buffer, version)
logExt.buffer = appendVarint(logExt.buffer, extSize/sectorSize)
if initial {
logExt.buffer = append(logExt.buffer, t.uuid[:]...)
Expand All @@ -97,7 +103,7 @@ func (t *tfs) logExtend() error {
if (t.size != 0) && (t.allocated+logExtensionSize > t.size) {
return fmt.Errorf("available space (%d bytes) too small, required %d", t.size-t.allocated, logExtensionSize)
}
logExt := t.newLogExt(false)
logExt := t.newLogExt(false, t.currentExt.oldEncoding)
t.currentExt.linkTo(logExt.offset)
t.allocated += logExtensionSize
err := t.currentExt.flush(t.imgFile, t.imgOffset)
Expand All @@ -121,7 +127,12 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
if err != nil {
return 0, err
}
if version != tfsVersion {
var oldEncoding bool
if version == 4 {
oldEncoding = true
} else if version == tfsVersion {
oldEncoding = false
} else {
return 0, fmt.Errorf("TFS version mismatch: expected %d, found %d", tfsVersion, version)
}
_size, err := getVarint(buffer, &offset)
Expand Down Expand Up @@ -170,7 +181,7 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
}
if length == tupleTotalLen {
var decoded uint64
_, err := t.decodeValue(buffer[offset:offset+uint64(length)], &decoded)
_, err := t.decodeValue(buffer[offset:offset+uint64(length)], &decoded, oldEncoding)
if err != nil {
return 0, err
}
Expand All @@ -194,7 +205,7 @@ func (t *tfs) readLogExt(offset, size uint64) (uint64, error) {
t.decoder.tupleRemain -= length
if t.decoder.tupleRemain == 0 {
var decoded uint64
_, err := t.decodeValue(t.staging, &decoded)
_, err := t.decodeValue(t.staging, &decoded, oldEncoding)
if err != nil {
return 0, err
}
Expand Down Expand Up @@ -252,20 +263,38 @@ func (t *tfs) writeDirEntries(dir map[string]interface{}) error {
func (t *tfs) encodeValue(value interface{}) error {
str, isStr := value.(string)
if isStr {
t.encodeString(str, typeString)
strType := byte(typeString)
if t.currentExt.oldEncoding {
strType = typeBuffer
}
t.encodeString(str, strType)
return nil
}
strSlice, isStrSlice := value.([]string)
if isStrSlice {
vector := make([]interface{}, len(strSlice))
if !t.currentExt.oldEncoding {
vector := make([]interface{}, len(strSlice))
for i, str := range strSlice {
vector[i] = str
}
return t.encodeVector(vector)
}
tuple := make(map[string]interface{})
for i, str := range strSlice {
vector[i] = str
tuple[strconv.Itoa(i)] = str
}
return t.encodeVector(vector)
return t.encodeTuple(tuple)
}
slice, isSlice := value.([]interface{})
if isSlice {
return t.encodeVector(slice)
if !t.currentExt.oldEncoding {
return t.encodeVector(slice)
}
tuple := make(map[string]interface{})
for i, val := range slice {
tuple[strconv.Itoa(i)] = val
}
return t.encodeTuple(tuple)
}
tuple, isTuple := value.(map[string]interface{})
if isTuple {
Expand Down Expand Up @@ -323,19 +352,19 @@ func (t *tfs) encodeVector(vector []interface{}) error {
return nil
}

func (t *tfs) decodeValue(buffer []byte, offset *uint64) (interface{}, error) {
func (t *tfs) decodeValue(buffer []byte, offset *uint64, oldEncoding bool) (interface{}, error) {
var entry, dataType byte
length, err := getHeader(buffer, offset, &entry, &dataType)
length, err := getHeader(buffer, offset, &entry, &dataType, oldEncoding)
if err != nil {
return nil, err
}
switch dataType {
case typeTuple:
return t.decodeTuple(buffer, offset, entry, length)
return t.decodeTuple(buffer, offset, entry, length, oldEncoding)
case typeBuffer:
return t.decodeBuf(buffer, offset, entry, length)
case typeVector:
return t.decodeVector(buffer, offset, entry, length)
return t.decodeVector(buffer, offset, entry, length, oldEncoding)
case typeInteger:
return t.decodeInteger(buffer, offset, entry, length)
case typeString:
Expand All @@ -345,7 +374,7 @@ func (t *tfs) decodeValue(buffer []byte, offset *uint64) (interface{}, error) {
}
}

func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uint) (*[]interface{}, error) {
func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uint, oldEncoding bool) (*[]interface{}, error) {
var vector *[]interface{}
if entry == entryImmediate {
newVector := make([]interface{}, length)
Expand All @@ -362,7 +391,7 @@ func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uin
}
}
for i := 0; i < int(length); i++ {
value, err := t.decodeValue(buffer, offset)
value, err := t.decodeValue(buffer, offset, oldEncoding)
if err != nil {
return vector, err
}
Expand All @@ -371,7 +400,7 @@ func (t *tfs) decodeVector(buffer []byte, offset *uint64, entry byte, length uin
return vector, nil
}

func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint) (*map[string]interface{}, error) {
func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint, oldEncoding bool) (*map[string]interface{}, error) {
var tuple *map[string]interface{}
if entry == entryImmediate {
newTuple := make(map[string]interface{})
Expand All @@ -389,7 +418,7 @@ func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint
}
for i := 0; i < int(length); i++ {
var nameEntry, nameType byte
n, err := getHeader(buffer, offset, &nameEntry, &nameType)
n, err := getHeader(buffer, offset, &nameEntry, &nameType, oldEncoding)
if err != nil {
return tuple, err
}
Expand All @@ -400,7 +429,7 @@ func (t *tfs) decodeTuple(buffer []byte, offset *uint64, entry byte, length uint
if err != nil {
return tuple, err
}
value, err := t.decodeValue(buffer, offset)
value, err := t.decodeValue(buffer, offset, oldEncoding)
if err != nil {
return tuple, err
}
Expand Down Expand Up @@ -543,12 +572,17 @@ func (t *tfs) pushHeader(entry byte, dataType byte, length int) {
len64 := uint64(length)
bitCount := uint(64 - bits.LeadingZeros64(len64))
var words uint
if bitCount > 3 {
words = ((bitCount - 3) + (7 - 1)) / 7
immBits := uint(3)
if t.currentExt.oldEncoding {
immBits = 5
}
var first = (entry << 7) | (dataType << 4) | byte(len64>>(words*7))
if bitCount > immBits {
words = ((bitCount - immBits) + (7 - 1)) / 7
}
var first = (entry << 7) | (dataType << (immBits + 1)) |
byte(len64>>(words*7))
if words != 0 {
first |= 1 << 3
first |= 1 << immBits
}
t.staging = append(t.staging, first)
i := words
Expand Down Expand Up @@ -921,8 +955,9 @@ func (r *tfsFileReader) selectExtent(index int) {
}

type tlogExt struct {
offset uint64
buffer []byte
offset uint64
oldEncoding bool
buffer []byte
}

func (e *tlogExt) linkTo(extOffset uint64) {
Expand Down Expand Up @@ -958,16 +993,20 @@ func appendVarint(buffer []byte, x uint) []byte {
return buffer
}

func getHeader(buffer []byte, offset *uint64, entry *byte, dataType *byte) (uint, error) {
func getHeader(buffer []byte, offset *uint64, entry *byte, dataType *byte, oldEncoding bool) (uint, error) {
if int(*offset) >= len(buffer) {
return 0, fmt.Errorf("getHeader(): buffer length %d exhausted", len(buffer))
}
b := buffer[*offset]
*offset++
*entry = b >> 7
*dataType = (b >> 4) & 0x7
length := uint(b & 0x7)
if (b & (1 << 3)) != 0 {
immBits := uint(3)
if oldEncoding {
immBits = 5
}
*dataType = (b >> (immBits + 1)) & ((1 << (6 - immBits)) - 1)
length := uint(b & ((1 << immBits) - 1))
if (b & (1 << immBits)) != 0 {
for {
if int(*offset) >= len(buffer) {
return 0, fmt.Errorf("getHeader(): buffer length %d exhausted", len(buffer))
Expand Down Expand Up @@ -1042,15 +1081,15 @@ func newTfs(imgFile *os.File, imgOffset uint64, fsSize uint64) *tfs {
}

// tfsWrite writes filesystem metadata and contents to image file
func tfsWrite(imgFile *os.File, imgOffset uint64, fsSize uint64, label string, root map[string]interface{}) (*tfs, error) {
func tfsWrite(imgFile *os.File, imgOffset uint64, fsSize uint64, label string, root map[string]interface{}, oldEncoding bool) (*tfs, error) {
tfs := newTfs(imgFile, imgOffset, fsSize)
tfs.label = label
rand.Seed(time.Now().UnixNano())
_, err := rand.Read(tfs.uuid[:])
if err != nil {
return nil, fmt.Errorf("error generating random uuid: %v", err)
}
err = tfs.logInit()
err = tfs.logInit(oldEncoding)
if err != nil {
return nil, fmt.Errorf("cannot create filesystem log: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion fs/tfs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ func TestLogExt(t *testing.T) {
size: 512,
}
t.Run("initial log ext", func(t *testing.T) {
ext := tfs.newLogExt(true)
ext := tfs.newLogExt(true, false)
if string(ext.buffer[:6]) != "NVMTFS" {
t.Errorf("invalid TFS magic: got %d want 'NVMTFS'", ext.buffer[:6])
}
Expand Down
Loading

0 comments on commit 5ec5e33

Please sign in to comment.