Skip to content
This repository has been archived by the owner on Aug 13, 2019. It is now read-only.

Commit

Permalink
add two more benchmarks
Browse files Browse the repository at this point in the history
Signed-off-by: naivewong <[email protected]>
  • Loading branch information
naivewong committed Aug 13, 2019
1 parent 9a858bc commit dadf7ee
Show file tree
Hide file tree
Showing 3 changed files with 665 additions and 54 deletions.
4 changes: 3 additions & 1 deletion index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -1065,8 +1065,10 @@ func (dec *Decoder) Postings(b []byte, version int) (int, Postings, error) {
l := d.Get()
if version == FormatV3 {
return n, newPrefixCompressedPostings(l), d.Err()
} else {
} else if version == FormatV1 || version == FormatV2 {
return n, newBigEndianPostings(l), d.Err()
} else {
return n, EmptyPostings(), d.Err()
}
}

Expand Down
102 changes: 52 additions & 50 deletions index/postings.go
Original file line number Diff line number Diff line change
Expand Up @@ -692,15 +692,15 @@ func (it *bigEndianPostings) Err() error {
}

type prefixCompressedPostings struct {
bs []byte
cur uint64
inside bool
idx int // The current offset inside the bs.
footerAddr int
key uint64
numBlock int
blockIdx int // The current block idx.
nextBlock int // The starting offset of the next block.
bs []byte
cur uint64
initialized bool
idx int // The current offset inside the bs.
footerAddr int
key uint64
numBlock int
blockIdx int // The current block idx.
nextBlock int // The starting offset of the next block.
}

func newPrefixCompressedPostings(bstream []byte) *prefixCompressedPostings {
Expand All @@ -716,19 +716,19 @@ func (it *prefixCompressedPostings) At() uint64 {
}

func (it *prefixCompressedPostings) Next() bool {
if it.inside { // Already entered the block.
if it.initialized { // Already entered the block.
if it.idx < it.nextBlock {
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx:]))
it.idx += 2
return true
}
it.blockIdx += 1 // Go to the next block.
it.blockIdx++ // Go to the next block.
}
// Currently not entered any block.
if it.idx < it.footerAddr {
it.key = binary.BigEndian.Uint64(it.bs[it.idx:])
it.idx += 8
it.inside = true
it.initialized = true
it.nextBlock = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+1)<<2):]))
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx:]))
it.idx += 2
Expand All @@ -744,55 +744,60 @@ func (it *prefixCompressedPostings) seekInBlock(x uint64) bool {
j := sort.Search(num, func(i int) bool {
return binary.BigEndian.Uint16(it.bs[it.idx+(i<<1):]) >= curVal
})
if j == num {
// Fast-path to the next block.
// The first element in next block should be >= x.
it.idx = it.nextBlock
it.blockIdx += 1
if it.idx >= it.footerAddr {
return false
}
it.key = binary.BigEndian.Uint64(it.bs[it.idx:])
it.idx += 8
it.inside = true
it.nextBlock = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+1)<<2):]))
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx:]))
it.idx += 2
if j < num {
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx+(j<<1):]))
it.idx += (j + 1) << 1
return true
}
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx+(j<<1):]))
it.idx += (j + 1) << 1
// Fast-path to the next block.
// The first element in next block should be >= x.
it.idx = it.nextBlock
it.blockIdx++
if it.idx >= it.footerAddr {
return false
}
it.key = binary.BigEndian.Uint64(it.bs[it.idx:])
it.idx += 8
it.initialized = true
it.nextBlock = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+1)<<2):]))
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx:]))
it.idx += 2
return true
}

func (it *prefixCompressedPostings) Seek(x uint64) bool {
if it.cur >= x {
return true
}
curKey := (x >> 16) << 16
if it.inside && it.key == curKey {
curKey := x & 0xffffffffffff0000
if it.initialized && it.key == curKey {
// Fast path for x in current block.
return it.seekInBlock(x)
} else {
i := sort.Search(it.numBlock-it.blockIdx, func(i int) bool {
off := int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+i)<<2):]))
k := binary.BigEndian.Uint64(it.bs[off:])
return k >= curKey
})
if i == it.numBlock-it.blockIdx {
return false
}
it.blockIdx += i
if i > 0 {
it.idx = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx)<<2):]))
}
}
i := sort.Search(it.numBlock-it.blockIdx, func(i int) bool {
off := int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+i)<<2):]))
k := binary.BigEndian.Uint64(it.bs[off:])
return k >= curKey
})
if i == it.numBlock-it.blockIdx {
it.idx = it.footerAddr
return false
}
it.blockIdx += i
if i > 0 {
it.idx = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx)<<2):]))
}
it.key = binary.BigEndian.Uint64(it.bs[it.idx:])
it.idx += 8

it.inside = true
it.initialized = true

it.nextBlock = int(binary.BigEndian.Uint32(it.bs[it.footerAddr+((it.blockIdx+1)<<2):]))
if it.key != curKey {
it.cur = it.key | uint64(binary.BigEndian.Uint16(it.bs[it.idx:]))
it.idx += 2
return true
}
return it.seekInBlock(x)
}

Expand All @@ -815,34 +820,31 @@ func writePrefixCompressedPostings(e *encoding.Encbuf, arr []uint64) {
return
}
key := uint64(0)
mask := uint64((1 << uint(16)) - 1) // Mask for the elements in the block.
invertedMask := ^mask
mask := uint64(0xFFFF) // Mask for the elements in the block.
invertedMask := uint64(0xFFFFFFFFFFFF0000)
var (
curKey uint64
curVal uint64
idx int // Index of current element in arr.
startingOffs []uint32 // The starting offsets of each block.
vals []uint16 // The converted values in the current block.
startOff = len(e.Get())
c = make([]byte, 2)
)
e.PutBE32(0) // Footer starting offset.
e.PutBE32(0) // Number of blocks.
for idx < len(arr) {
for idx := range arr {
curKey = arr[idx] & invertedMask // Key of block.
curVal = arr[idx] & mask // Value inside block.
if curKey != key {
// Move to next block.
if idx != 0 {
// We don't need to store the starting offset of the first block because it won't be used.
startingOffs = append(startingOffs, uint32(len(e.B)))
writePrefixCompressedPostingsBlock(e, vals, key, c)
vals = vals[:0]
}
key = curKey
}
vals = append(vals, uint16(curVal))
idx += 1
}
startingOffs = append(startingOffs, uint32(len(e.B)))
writePrefixCompressedPostingsBlock(e, vals, key, c)
Expand Down
Loading

0 comments on commit dadf7ee

Please sign in to comment.