Skip to content

Commit

Permalink
Bug: iterator not iterating sequentially (#126)
Browse files Browse the repository at this point in the history 
* bug

* removing index file

* step 1 -- convert all reference values to bigendian

* step 2 -- convert all others to little endian

* remove mock.index from tracing
  • Loading branch information
@friendlymatthew
friendlymatthew authored Feb 22, 2024
1 parent b689183 commit 545bb1f
Show file tree
Hide file tree
Showing 12 changed files with 189 additions and 45 deletions.
12 changes: 6 additions & 6 deletions pkg/appendable/appendable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (m *FileMeta) MarshalBinary() ([]byte, error) {
buf := make([]byte, 10)
buf[0] = byte(m.Version)
buf[1] = byte(m.Format)
binary.BigEndian.PutUint64(buf[2:], m.ReadOffset)
binary.LittleEndian.PutUint64(buf[2:], m.ReadOffset)
return buf, nil
}

Expand All @@ -117,7 +117,7 @@ func (m *FileMeta) UnmarshalBinary(buf []byte) error {
return fmt.Errorf("unrecognized file format: %v", buf[1])
}

m.ReadOffset = binary.BigEndian.Uint64(buf[2:])
m.ReadOffset = binary.LittleEndian.Uint64(buf[2:])
return nil
}

Expand All @@ -128,8 +128,8 @@ type IndexMeta struct {

func (m *IndexMeta) MarshalBinary() ([]byte, error) {
buf := make([]byte, 2+len(m.FieldName)+2)
binary.BigEndian.PutUint16(buf[0:], uint16(m.FieldType))
binary.BigEndian.PutUint16(buf[2:], uint16(len(m.FieldName)))
binary.LittleEndian.PutUint16(buf[0:], uint16(m.FieldType))
binary.LittleEndian.PutUint16(buf[2:], uint16(len(m.FieldName)))
copy(buf[4:], m.FieldName)
return buf, nil
}
Expand All @@ -138,8 +138,8 @@ func (m *IndexMeta) UnmarshalBinary(buf []byte) error {
if len(buf) < 4 {
return fmt.Errorf("invalid metadata size: %d", len(buf))
}
m.FieldType = FieldType(binary.BigEndian.Uint16(buf[0:]))
nameLength := binary.BigEndian.Uint16(buf[2:])
m.FieldType = FieldType(binary.LittleEndian.Uint16(buf[0:]))
nameLength := binary.LittleEndian.Uint16(buf[2:])
if len(buf) < 4+int(nameLength) {
return fmt.Errorf("invalid metadata size: %d", len(buf))
}
Expand Down
48 changes: 48 additions & 0 deletions pkg/btree/bptree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,54 @@ func (t *BPTree) readNode(ptr MemoryPointer) (*BPTreeNode, error) {
return node, nil
}

func (t *BPTree) first() (ReferencedValue, error) {
rootNode, _, err := t.root()

if err != nil {
return ReferencedValue{}, err
}

currNode, err := t.readNode(rootNode.Pointer(0))
if err != nil {
return ReferencedValue{}, err
}

for !currNode.leaf() {
childPointer := currNode.Pointer(0)
currNode, err = t.readNode(childPointer)

if err != nil {
return ReferencedValue{}, err
}
}

return currNode.Keys[0], nil
}

func (t *BPTree) last() (ReferencedValue, error) {
rootNode, _, err := t.root()

if err != nil {
return ReferencedValue{}, err
}

currNode, err := t.readNode(rootNode.Pointer(rootNode.NumPointers() - 1))
if err != nil {
return ReferencedValue{}, err
}

for !currNode.leaf() {
childPointer := currNode.Pointer(currNode.NumPointers() - 1)
currNode, err = t.readNode(childPointer)

if err != nil {
return ReferencedValue{}, err
}
}

return currNode.Keys[0], nil
}

// traverse returns the path from root to leaf in reverse order (leaf first)
// the last element is always the node passed in
func (t *BPTree) traverse(key ReferencedValue, node *BPTreeNode, ptr MemoryPointer) ([]TraversalRecord, error) {
Expand Down
102 changes: 99 additions & 3 deletions pkg/btree/bptree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package btree
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"math"
"math/rand"
Expand All @@ -27,7 +28,7 @@ func (m *testMetaPage) Root() (MemoryPointer, error) {

func (m *testMetaPage) write() error {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, m.root.Offset)
binary.LittleEndian.PutUint64(buf, m.root.Offset)
if _, err := m.pf.Seek(4096, io.SeekStart); err != nil {
return err
}
Expand Down Expand Up @@ -222,15 +223,15 @@ func TestBPTree_SequentialInsertionTest(t *testing.T) {
tree := NewBPTree(p, newTestMetaPage(t, p))
for i := 0; i < 256; i++ {
buf := make([]byte, 8)
binary.LittleEndian.PutUint64(buf, uint64(i))
binary.BigEndian.PutUint64(buf, uint64(i))
if err := tree.Insert(ReferencedValue{Value: buf}, MemoryPointer{Offset: uint64(i), Length: uint32(len(buf))}); err != nil {
t.Fatal(err)
}
}

for i := 0; i < 256; i++ {
buf := make([]byte, 8)
binary.LittleEndian.PutUint64(buf, uint64(i))
binary.BigEndian.PutUint64(buf, uint64(i))
k, v, err := tree.Find(ReferencedValue{Value: buf})
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -470,3 +471,98 @@ func TestBPTree_Iteration_SinglePage(t *testing.T) {
}
})
}

func TestBPTree_Iteration_FirstLast(t *testing.T) {
b := buftest.NewSeekableBuffer()
p, err := NewPageFile(b)
if err != nil {
t.Fatal(err)
}
tree := NewBPTree(p, newTestMetaPage(t, p))
start := 10.0
increments := []float64{0.01, 0.05, 0.3}
currentIncrementIndex := 0

for i := start; i < 256; i += increments[currentIncrementIndex] {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, math.Float64bits(i))

if err := tree.Insert(ReferencedValue{Value: buf}, MemoryPointer{Offset: uint64(i * 100), Length: uint32(len(buf))}); err != nil {
t.Fatal(err)
}

if int(i*100)%10 == 0 && currentIncrementIndex < len(increments)-1 {
currentIncrementIndex++
}
}

t.Run("find first and iter", func(t *testing.T) {
first, err := tree.first()
if err != nil {
t.Fatal(err)
}
firstBuf := make([]byte, 8)
binary.BigEndian.PutUint64(firstBuf, math.Float64bits(10))

if !bytes.Equal(first.Value, firstBuf) {
t.Fatal("expected 10 as first reference value")
}

iter, err := tree.Iter(first)
if err != nil {
t.Fatal(err)
}

i := 0
var p *float64 = nil
for ; iter.Next(); i++ {
k := iter.Key()
var c float64
reader := bytes.NewReader(k.Value)
err := binary.Read(reader, binary.BigEndian, &c)
if err != nil {
fmt.Println("binary.Read failed:", err)
return
}

if p != nil && *p > c {
t.Errorf("expected a non-decreasing traversal but got prev: %v, curr: %v", *p, c)
return
}
p = &c
}

})

t.Run("find last and iter", func(t *testing.T) {
last, err := tree.last()
if err != nil {
t.Fatal(err)
}

iter, err := tree.Iter(last)
if err != nil {
t.Fatal(err)
}

i := 0
var r *float64 = nil
for ; iter.Prev(); i++ {
k := iter.Key()
var l float64
reader := bytes.NewReader(k.Value)
err := binary.Read(reader, binary.BigEndian, &l)
if err != nil {
fmt.Println("binary.Read failed:", err)
return
}

if r != nil && !(l <= *r) {
t.Errorf("expected a non-increasing traversal but got curr: %v, prev: %v", l, *r)
return
}
r = &l
}

})
}
14 changes: 7 additions & 7 deletions pkg/btree/multi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ func (m *LinkedMetaPage) Root() (MemoryPointer, error) {
return MemoryPointer{}, err
}
var mp MemoryPointer
return mp, binary.Read(m.rws, binary.BigEndian, &mp)
return mp, binary.Read(m.rws, binary.LittleEndian, &mp)
}

func (m *LinkedMetaPage) SetRoot(mp MemoryPointer) error {
if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil {
return err
}
return binary.Write(m.rws, binary.BigEndian, mp)
return binary.Write(m.rws, binary.LittleEndian, mp)
}

// BPTree returns a B+ tree that uses this meta page as the root
Expand All @@ -62,7 +62,7 @@ func (m *LinkedMetaPage) Metadata() ([]byte, error) {
return nil, err
}
// the first four bytes represents the length
length := binary.BigEndian.Uint32(buf[:4])
length := binary.LittleEndian.Uint32(buf[:4])
return buf[4 : 4+length], nil
}

Expand All @@ -82,7 +82,7 @@ func (m *LinkedMetaPage) SetMetadata(data []byte) error {
return err
}
buf := append(make([]byte, 4), data...)
binary.BigEndian.PutUint32(buf, uint32(len(data)))
binary.LittleEndian.PutUint32(buf, uint32(len(data)))
if _, err := m.rws.Write(buf); err != nil {
return err
}
Expand All @@ -102,7 +102,7 @@ func (m *LinkedMetaPage) Next() (*LinkedMetaPage, error) {
return nil, err
}
var next MemoryPointer
if err := binary.Read(m.rws, binary.BigEndian, &next); err != nil {
if err := binary.Read(m.rws, binary.LittleEndian, &next); err != nil {
return nil, err
}
return &LinkedMetaPage{rws: m.rws, offset: next.Offset}, nil
Expand All @@ -128,7 +128,7 @@ func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) {
if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil {
return nil, err
}
if err := binary.Write(m.rws, binary.BigEndian, next.offset); err != nil {
if err := binary.Write(m.rws, binary.LittleEndian, next.offset); err != nil {
return nil, err
}
return next, nil
Expand Down Expand Up @@ -158,7 +158,7 @@ func (m *LinkedMetaPage) Exists() (bool, error) {
func (m *LinkedMetaPage) Reset() error {
// write a full page of zeros
emptyPage := make([]byte, m.rws.PageSize())
binary.BigEndian.PutUint64(emptyPage[12:20], ^uint64(0))
binary.LittleEndian.PutUint64(emptyPage[12:20], ^uint64(0))
if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil {
return err
}
Expand Down
32 changes: 16 additions & 16 deletions pkg/btree/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,22 +101,22 @@ func (n *BPTreeNode) MarshalBinary() ([]byte, error) {
buf := make([]byte, n.Size())
// set the first bit to 1 if it's a leaf
if n.leaf() {
binary.BigEndian.PutUint32(buf[:4], uint32(-size))
binary.LittleEndian.PutUint32(buf[:4], uint32(-size))
} else {
binary.BigEndian.PutUint32(buf[:4], uint32(size))
binary.LittleEndian.PutUint32(buf[:4], uint32(size))
}
if size == 0 {
panic("writing empty node")
}
ct := 4
for _, k := range n.Keys {
if k.DataPointer.Length > 0 {
binary.BigEndian.PutUint32(buf[ct:ct+4], ^uint32(0))
binary.BigEndian.PutUint64(buf[ct+4:ct+12], k.DataPointer.Offset)
binary.BigEndian.PutUint32(buf[ct+12:ct+16], k.DataPointer.Length)
binary.LittleEndian.PutUint32(buf[ct:ct+4], ^uint32(0))
binary.LittleEndian.PutUint64(buf[ct+4:ct+12], k.DataPointer.Offset)
binary.LittleEndian.PutUint32(buf[ct+12:ct+16], k.DataPointer.Length)
ct += 4 + 12
} else {
binary.BigEndian.PutUint32(buf[ct:ct+4], uint32(len(k.Value)))
binary.LittleEndian.PutUint32(buf[ct:ct+4], uint32(len(k.Value)))
m := copy(buf[ct+4:ct+4+len(k.Value)], k.Value)
if m != len(k.Value) {
return nil, fmt.Errorf("failed to copy key: %w", io.ErrShortWrite)
Expand All @@ -125,12 +125,12 @@ func (n *BPTreeNode) MarshalBinary() ([]byte, error) {
}
}
for _, p := range n.leafPointers {
binary.BigEndian.PutUint64(buf[ct:ct+8], p.Offset)
binary.BigEndian.PutUint32(buf[ct+8:ct+12], p.Length)
binary.LittleEndian.PutUint64(buf[ct:ct+8], p.Offset)
binary.LittleEndian.PutUint32(buf[ct+8:ct+12], p.Length)
ct += 12
}
for _, p := range n.internalPointers {
binary.BigEndian.PutUint64(buf[ct:ct+8], p)
binary.LittleEndian.PutUint64(buf[ct:ct+8], p)
ct += 8
}
if ct != int(n.Size()) {
Expand All @@ -149,7 +149,7 @@ func (n *BPTreeNode) WriteTo(w io.Writer) (int64, error) {
}

func (n *BPTreeNode) UnmarshalBinary(buf []byte) error {
size := int32(binary.BigEndian.Uint32(buf[:4]))
size := int32(binary.LittleEndian.Uint32(buf[:4]))
leaf := size < 0
if leaf {
n.leafPointers = make([]MemoryPointer, -size)
Expand All @@ -164,11 +164,11 @@ func (n *BPTreeNode) UnmarshalBinary(buf []byte) error {

m := 4
for i := range n.Keys {
l := binary.BigEndian.Uint32(buf[m : m+4])
l := binary.LittleEndian.Uint32(buf[m : m+4])
if l == ^uint32(0) {
// read the key out of the memory pointer stored at this position
n.Keys[i].DataPointer.Offset = binary.BigEndian.Uint64(buf[m+4 : m+12])
n.Keys[i].DataPointer.Length = binary.BigEndian.Uint32(buf[m+12 : m+16])
n.Keys[i].DataPointer.Offset = binary.LittleEndian.Uint64(buf[m+4 : m+12])
n.Keys[i].DataPointer.Length = binary.LittleEndian.Uint32(buf[m+12 : m+16])
dp := n.Keys[i].DataPointer
n.Keys[i].Value = n.DataParser.Parse(n.Data[dp.Offset : dp.Offset+uint64(dp.Length)]) // resolving the data-file
m += 4 + 12
Expand All @@ -178,12 +178,12 @@ func (n *BPTreeNode) UnmarshalBinary(buf []byte) error {
}
}
for i := range n.leafPointers {
n.leafPointers[i].Offset = binary.BigEndian.Uint64(buf[m : m+8])
n.leafPointers[i].Length = binary.BigEndian.Uint32(buf[m+8 : m+12])
n.leafPointers[i].Offset = binary.LittleEndian.Uint64(buf[m : m+8])
n.leafPointers[i].Length = binary.LittleEndian.Uint32(buf[m+8 : m+12])
m += 12
}
for i := range n.internalPointers {
n.internalPointers[i] = binary.BigEndian.Uint64(buf[m : m+8])
n.internalPointers[i] = binary.LittleEndian.Uint64(buf[m : m+8])
m += 8
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/btree/pagefile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func NewPageFile(rws io.ReadWriteSeeker) (*PageFile, error) {
}
} else {
for i := 0; i < len(pf.freePageIndexes); i++ {
offset := int64(binary.BigEndian.Uint64(buf[i*8 : (i+1)*8]))
offset := int64(binary.LittleEndian.Uint64(buf[i*8 : (i+1)*8]))
if offset != 0 {
pf.freePageIndexes[pf.freePageHead] = offset
pf.freePageHead = (pf.freePageHead + 1) % len(pf.freePageIndexes)
Expand Down Expand Up @@ -89,7 +89,7 @@ func (pf *PageFile) writeFreePageIndices() error {
tail := (pf.freePageHead - pf.freePageCount + len(pf.freePageIndexes)) % len(pf.freePageIndexes)
for i := 0; i < pf.freePageCount; i++ {
offset := pf.freePageIndexes[tail+i]
binary.BigEndian.PutUint64(buf[i*8:(i+1)*8], uint64(offset))
binary.LittleEndian.PutUint64(buf[i*8:(i+1)*8], uint64(offset))
}
if _, err := pf.ReadWriteSeeker.Seek(0, io.SeekStart); err != nil {
return err
Expand Down
Loading

0 comments on commit 545bb1f

Please sign in to comment.