feat: drill fixed width through multi

pkg/appendable/appendable.go

@@ -127,12 +127,9 @@ type IndexMeta struct {
 	FieldWidth uint16
 }
 
-func (m *IndexMeta) MarshalBinary() ([]byte, error) {
-	buf := make([]byte, 2+2+len(m.FieldName)+2)
-	binary.LittleEndian.PutUint16(buf[0:], uint16(m.FieldType))
-
+func DetermineFieldWidth(t FieldType) uint16 {
 	var width uint16
-	switch m.FieldType {
+	switch t {
 	case FieldTypeFloat64, FieldTypeInt64:
 		width = 8
 	case FieldTypeBoolean:
@@ -143,7 +140,13 @@ func (m *IndexMeta) MarshalBinary() ([]byte, error) {
 		width = 0xFFFF
 	}
 
-	binary.LittleEndian.PutUint16(buf[2:], width)
+	return width
+}
+
+func (m *IndexMeta) MarshalBinary() ([]byte, error) {
+	buf := make([]byte, 2+2+len(m.FieldName)+2)
+	binary.LittleEndian.PutUint16(buf[0:], uint16(m.FieldType))
+	binary.LittleEndian.PutUint16(buf[2:], DetermineFieldWidth(m.FieldType))
 	binary.LittleEndian.PutUint16(buf[4:], uint16(len(m.FieldName)))
 	copy(buf[6:], m.FieldName)
 	return buf, nil

pkg/appendable/index_file.go

@@ -185,10 +185,16 @@ func (i *IndexFile) FindOrCreateIndex(name string, fieldType FieldType) (*btree.
 	metadata := &IndexMeta{}
 	metadata.FieldName = name
 	metadata.FieldType = fieldType
+	fieldWidth := DetermineFieldWidth(fieldType)
+	metadata.FieldWidth = fieldWidth
+
 	buf, err := metadata.MarshalBinary()
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal metadata: %w", err)
 	}
+
+	next.FixedWidth = fieldWidth
+
 	return next, next.SetMetadata(buf)
 }
 

pkg/btree/bptree.go

@@ -13,6 +13,7 @@ import (
 type MetaPage interface {
 	Root() (MemoryPointer, error)
 	SetRoot(MemoryPointer) error
+	GetWidth() uint16
 }
 
 type BPTree struct {
@@ -155,7 +156,7 @@ func (t *BPTree) readNode(ptr MemoryPointer) (*BPTreeNode, error) {
 	if _, err := t.tree.Seek(int64(ptr.Offset), io.SeekStart); err != nil {
 		return nil, err
 	}
-	node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser}
+	node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()}
 	if _, err := node.ReadFrom(t.tree); err != nil {
 		return nil, err
 	}
@@ -245,7 +246,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error {
 	}
 	if root == nil {
 		// special case, create the root as the first node
-		node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser}
+		node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()}
 		node.Keys = []ReferencedValue{key}
 		node.leafPointers = []MemoryPointer{value}
 		buf, err := node.MarshalBinary()
@@ -291,7 +292,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error {
 			midKey := n.Keys[mid]
 
 			// n is the left node, m the right node
-			m := &BPTreeNode{Data: t.Data, DataParser: t.DataParser}
+			m := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()}
 			if n.Leaf() {
 				m.leafPointers = n.leafPointers[mid:]
 				m.Keys = n.Keys[mid:]
@@ -345,7 +346,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error {
 				// the parent will be written to disk in the next iteration
 			} else {
 				// the root split, so create a new root
-				p := &BPTreeNode{Data: t.Data, DataParser: t.DataParser}
+				p := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()}
 				p.Keys = []ReferencedValue{midKey}
 				p.internalPointers = []uint64{
 					uint64(noffset), uint64(moffset),

pkg/btree/bptree_test.go

@@ -26,6 +26,10 @@ func (m *testMetaPage) Root() (MemoryPointer, error) {
 	return m.root, nil
 }
 
+func (m *testMetaPage) GetWidth() uint16 {
+	return ^uint16(0)
+}
+
 func (m *testMetaPage) write() error {
 	buf := make([]byte, 8)
 	binary.LittleEndian.PutUint64(buf, m.root.Offset)

pkg/btree/multi.go

@@ -21,8 +21,9 @@ const N = 16
  * math.MaxUint64.
  */
 type LinkedMetaPage struct {
-	rws    ReadWriteSeekPager
-	offset uint64
+	rws        ReadWriteSeekPager
+	FixedWidth uint16
+	offset     uint64
 }
 
 func (m *LinkedMetaPage) Root() (MemoryPointer, error) {
@@ -40,6 +41,10 @@ func (m *LinkedMetaPage) SetRoot(mp MemoryPointer) error {
 	return binary.Write(m.rws, binary.LittleEndian, mp)
 }
 
+func (m *LinkedMetaPage) GetWidth() uint16 {
+	return m.FixedWidth
+}
+
 // BPTree returns a B+ tree that uses this meta page as the root
 // of the tree. If data is not nil, then it will be used as the
 // data source for the tree.

pkg/btree/node.go

@@ -64,6 +64,7 @@ type BPTreeNode struct {
 	leafPointers     []MemoryPointer
 	internalPointers []uint64
 	Keys             []ReferencedValue
+	FixedWidth       uint16
 }
 
 func (n *BPTreeNode) Leaf() bool {
@@ -85,10 +86,8 @@ func (n *BPTreeNode) Size() int64 {
 	size := 4 // number of keys
 	for _, k := range n.Keys {
 		size += 12
-		if k.ElideValue {
-			size += 4
-		} else {
-			size += 4 + len(k.Value)
+		if n.FixedWidth != ^uint16(0) {
+			size += len(k.Value)
 		}
 	}
 	for range n.leafPointers {
@@ -117,12 +116,7 @@ func (n *BPTreeNode) MarshalBinary() ([]byte, error) {
 		binary.LittleEndian.PutUint64(buf[ct:ct+8], k.DataPointer.Offset)
 		binary.LittleEndian.PutUint32(buf[ct+8:ct+12], k.DataPointer.Length)
 		ct += 12
-		if k.ElideValue {
-			binary.LittleEndian.PutUint32(buf[ct:ct+4], ^uint32(0))
-			ct += 4
-		} else {
-			binary.LittleEndian.PutUint32(buf[ct:ct+4], uint32(len(k.Value)))
-			ct += 4
+		if n.FixedWidth != ^uint16(0) {
 			m := copy(buf[ct:ct+len(k.Value)], k.Value)
 			if m != len(k.Value) {
 				return nil, fmt.Errorf("failed to copy key: %w", io.ErrShortWrite)
@@ -174,16 +168,13 @@ func (n *BPTreeNode) UnmarshalBinary(buf []byte) error {
 		n.Keys[i].DataPointer.Length = binary.LittleEndian.Uint32(buf[m+8 : m+12])
 		m += 12
 
-		l := binary.LittleEndian.Uint32(buf[m : m+4])
-		m += 4
-		if l == ^uint32(0) {
+		if n.FixedWidth == ^uint16(0) {
 			// read the key out of the memory pointer stored at this position
-			n.Keys[i].ElideValue = true
 			dp := n.Keys[i].DataPointer
 			n.Keys[i].Value = n.DataParser.Parse(n.Data[dp.Offset : dp.Offset+uint64(dp.Length)]) // resolving the data-file
 		} else {
-			n.Keys[i].Value = buf[m : m+int(l)]
-			m += int(l)
+			n.Keys[i].Value = buf[m : m+int(n.FixedWidth)]
+			m += int(n.FixedWidth)
 		}
 	}
 	for i := range n.leafPointers {

pkg/mocks/main.go

@@ -15,6 +15,10 @@ type testMetaPage struct {
 	root btree.MemoryPointer
 }
 
+func (m *testMetaPage) GetWidth() uint16 {
+	return ^uint16(0)
+}
+
 func (m *testMetaPage) SetRoot(mp btree.MemoryPointer) error {
 	m.root = mp
 	return m.write()