diff --git a/pkg/appendable/appendable.go b/pkg/appendable/appendable.go index f10cd084..5cad844c 100644 --- a/pkg/appendable/appendable.go +++ b/pkg/appendable/appendable.go @@ -127,12 +127,9 @@ type IndexMeta struct { FieldWidth uint16 } -func (m *IndexMeta) MarshalBinary() ([]byte, error) { - buf := make([]byte, 2+2+len(m.FieldName)+2) - binary.LittleEndian.PutUint16(buf[0:], uint16(m.FieldType)) - +func DetermineFieldWidth(t FieldType) uint16 { var width uint16 - switch m.FieldType { + switch t { case FieldTypeFloat64, FieldTypeInt64: width = 8 case FieldTypeBoolean: @@ -143,7 +140,13 @@ func (m *IndexMeta) MarshalBinary() ([]byte, error) { width = 0xFFFF } - binary.LittleEndian.PutUint16(buf[2:], width) + return width +} + +func (m *IndexMeta) MarshalBinary() ([]byte, error) { + buf := make([]byte, 2+2+len(m.FieldName)+2) + binary.LittleEndian.PutUint16(buf[0:], uint16(m.FieldType)) + binary.LittleEndian.PutUint16(buf[2:], DetermineFieldWidth(m.FieldType)) binary.LittleEndian.PutUint16(buf[4:], uint16(len(m.FieldName))) copy(buf[6:], m.FieldName) return buf, nil diff --git a/pkg/appendable/index_file.go b/pkg/appendable/index_file.go index 23692ed4..ec96f6ef 100644 --- a/pkg/appendable/index_file.go +++ b/pkg/appendable/index_file.go @@ -185,10 +185,16 @@ func (i *IndexFile) FindOrCreateIndex(name string, fieldType FieldType) (*btree. metadata := &IndexMeta{} metadata.FieldName = name metadata.FieldType = fieldType + fieldWidth := DetermineFieldWidth(fieldType) + metadata.FieldWidth = fieldWidth + buf, err := metadata.MarshalBinary() if err != nil { return nil, fmt.Errorf("failed to marshal metadata: %w", err) } + + next.FixedWidth = fieldWidth + return next, next.SetMetadata(buf) } diff --git a/pkg/btree/bptree.go b/pkg/btree/bptree.go index cd3346b5..8c2e3fbf 100644 --- a/pkg/btree/bptree.go +++ b/pkg/btree/bptree.go @@ -13,6 +13,7 @@ import ( type MetaPage interface { Root() (MemoryPointer, error) SetRoot(MemoryPointer) error + GetWidth() uint16 } type BPTree struct { @@ -155,7 +156,7 @@ func (t *BPTree) readNode(ptr MemoryPointer) (*BPTreeNode, error) { if _, err := t.tree.Seek(int64(ptr.Offset), io.SeekStart); err != nil { return nil, err } - node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser} + node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()} if _, err := node.ReadFrom(t.tree); err != nil { return nil, err } @@ -245,7 +246,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { } if root == nil { // special case, create the root as the first node - node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser} + node := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()} node.Keys = []ReferencedValue{key} node.leafPointers = []MemoryPointer{value} buf, err := node.MarshalBinary() @@ -291,7 +292,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { midKey := n.Keys[mid] // n is the left node, m the right node - m := &BPTreeNode{Data: t.Data, DataParser: t.DataParser} + m := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()} if n.Leaf() { m.leafPointers = n.leafPointers[mid:] m.Keys = n.Keys[mid:] @@ -345,7 +346,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { // the parent will be written to disk in the next iteration } else { // the root split, so create a new root - p := &BPTreeNode{Data: t.Data, DataParser: t.DataParser} + p := &BPTreeNode{Data: t.Data, DataParser: t.DataParser, FixedWidth: t.meta.GetWidth()} p.Keys = []ReferencedValue{midKey} p.internalPointers = []uint64{ uint64(noffset), uint64(moffset), diff --git a/pkg/btree/bptree_test.go b/pkg/btree/bptree_test.go index 593c4367..92aa1cb9 100644 --- a/pkg/btree/bptree_test.go +++ b/pkg/btree/bptree_test.go @@ -26,6 +26,10 @@ func (m *testMetaPage) Root() (MemoryPointer, error) { return m.root, nil } +func (m *testMetaPage) GetWidth() uint16 { + return ^uint16(0) +} + func (m *testMetaPage) write() error { buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, m.root.Offset) diff --git a/pkg/btree/multi.go b/pkg/btree/multi.go index f62cc968..a9377542 100644 --- a/pkg/btree/multi.go +++ b/pkg/btree/multi.go @@ -21,8 +21,9 @@ const N = 16 * math.MaxUint64. */ type LinkedMetaPage struct { - rws ReadWriteSeekPager - offset uint64 + rws ReadWriteSeekPager + FixedWidth uint16 + offset uint64 } func (m *LinkedMetaPage) Root() (MemoryPointer, error) { @@ -40,6 +41,10 @@ func (m *LinkedMetaPage) SetRoot(mp MemoryPointer) error { return binary.Write(m.rws, binary.LittleEndian, mp) } +func (m *LinkedMetaPage) GetWidth() uint16 { + return m.FixedWidth +} + // BPTree returns a B+ tree that uses this meta page as the root // of the tree. If data is not nil, then it will be used as the // data source for the tree. diff --git a/pkg/btree/node.go b/pkg/btree/node.go index b18508d0..975744de 100644 --- a/pkg/btree/node.go +++ b/pkg/btree/node.go @@ -64,6 +64,7 @@ type BPTreeNode struct { leafPointers []MemoryPointer internalPointers []uint64 Keys []ReferencedValue + FixedWidth uint16 } func (n *BPTreeNode) Leaf() bool { @@ -85,10 +86,8 @@ func (n *BPTreeNode) Size() int64 { size := 4 // number of keys for _, k := range n.Keys { size += 12 - if k.ElideValue { - size += 4 - } else { - size += 4 + len(k.Value) + if n.FixedWidth != ^uint16(0) { + size += len(k.Value) } } for range n.leafPointers { @@ -117,12 +116,7 @@ func (n *BPTreeNode) MarshalBinary() ([]byte, error) { binary.LittleEndian.PutUint64(buf[ct:ct+8], k.DataPointer.Offset) binary.LittleEndian.PutUint32(buf[ct+8:ct+12], k.DataPointer.Length) ct += 12 - if k.ElideValue { - binary.LittleEndian.PutUint32(buf[ct:ct+4], ^uint32(0)) - ct += 4 - } else { - binary.LittleEndian.PutUint32(buf[ct:ct+4], uint32(len(k.Value))) - ct += 4 + if n.FixedWidth != ^uint16(0) { m := copy(buf[ct:ct+len(k.Value)], k.Value) if m != len(k.Value) { return nil, fmt.Errorf("failed to copy key: %w", io.ErrShortWrite) @@ -174,16 +168,13 @@ func (n *BPTreeNode) UnmarshalBinary(buf []byte) error { n.Keys[i].DataPointer.Length = binary.LittleEndian.Uint32(buf[m+8 : m+12]) m += 12 - l := binary.LittleEndian.Uint32(buf[m : m+4]) - m += 4 - if l == ^uint32(0) { + if n.FixedWidth == ^uint16(0) { // read the key out of the memory pointer stored at this position - n.Keys[i].ElideValue = true dp := n.Keys[i].DataPointer n.Keys[i].Value = n.DataParser.Parse(n.Data[dp.Offset : dp.Offset+uint64(dp.Length)]) // resolving the data-file } else { - n.Keys[i].Value = buf[m : m+int(l)] - m += int(l) + n.Keys[i].Value = buf[m : m+int(n.FixedWidth)] + m += int(n.FixedWidth) } } for i := range n.leafPointers { diff --git a/pkg/mocks/main.go b/pkg/mocks/main.go index 2518df28..f8f501f9 100644 --- a/pkg/mocks/main.go +++ b/pkg/mocks/main.go @@ -15,6 +15,10 @@ type testMetaPage struct { root btree.MemoryPointer } +func (m *testMetaPage) GetWidth() uint16 { + return ^uint16(0) +} + func (m *testMetaPage) SetRoot(mp btree.MemoryPointer) error { m.root = mp return m.write()