Skip to content

Commit

Permalink
Support store option for the geoshape field (#1714)
Browse files Browse the repository at this point in the history 
* Support `store` option for the geoshape field

It includes,
-Enabling store option for geoshape field.
-Adapting the NewCircle apis from geo/geojson.
-Updating the UTs to reflect the signature changes.

* go mod tidy after bleve_index_api, scorch_segment_api, zapx updates

* Fix updated function name within DiskStatsReporter

* Update zapx@v15 to latest

Co-authored-by: Abhinav Dangeti <[email protected]>
  • Loading branch information
@sreekanth-cb @abhinavdangeti
sreekanth-cb and abhinavdangeti authored Jul 22, 2022
1 parent d946733 commit ec0d3aa
Show file tree
Hide file tree
Showing 15 changed files with 142 additions and 89 deletions.
60 changes: 50 additions & 10 deletions document/field_geoshape.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/blevesearch/bleve/v2/geo"
"github.com/blevesearch/bleve/v2/size"
index "github.com/blevesearch/bleve_index_api"
"github.com/blevesearch/geo/geojson"
)

var reflectStaticSizeGeoShapeField int
Expand All @@ -40,6 +41,7 @@ type GeoShapeField struct {
options index.FieldIndexingOptions
numPlainTextBytes uint64
length int
encodedValue []byte
value []byte

frequencies index.TokenFrequencies
Expand Down Expand Up @@ -80,8 +82,8 @@ func (n *GeoShapeField) Analyze() {
tokens := make(analysis.TokenStream, 0)
tokens = append(tokens, &analysis.Token{
Start: 0,
End: len(n.value),
Term: n.value,
End: len(n.encodedValue),
Term: n.encodedValue,
Position: 1,
Type: analysis.AlphaNumeric,
})
Expand Down Expand Up @@ -109,7 +111,8 @@ func (n *GeoShapeField) Value() []byte {
}

func (n *GeoShapeField) GoString() string {
return fmt.Sprintf("&document.GeoShapeField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
return fmt.Sprintf("&document.GeoShapeField{Name:%s, Options: %s, Value: %s}",
n.name, n.options, n.value)
}

func (n *GeoShapeField) NumPlainTextBytes() uint64 {
Expand All @@ -122,17 +125,34 @@ func NewGeoShapeField(name string, arrayPositions []uint64,
coordinates, typ, DefaultGeoShapeIndexingOptions)
}

func NewGeoShapeFieldFromBytes(name string, arrayPositions []uint64,
value []byte) *GeoShapeField {
return &GeoShapeField{
name: name,
arrayPositions: arrayPositions,
value: value,
options: DefaultGeoShapeIndexingOptions,
numPlainTextBytes: uint64(len(value)),
}
}

func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
coordinates [][][][]float64, typ string,
options index.FieldIndexingOptions) *GeoShapeField {
shape, value, err := geo.NewGeoJsonShape(coordinates, typ)
shape, encodedValue, err := geo.NewGeoJsonShape(coordinates, typ)
if err != nil {
return nil
}

// extra glue bytes to work around the term splitting logic from interfering
// the custom encoding of the geoshape coordinates inside the docvalues.
value = append(geo.GlueBytes, append(value, geo.GlueBytes...)...)
encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)

// get the byte value for the geoshape.
value, err := shape.Value()
if err != nil {
return nil
}

options = options | DefaultGeoShapeIndexingOptions

Expand All @@ -141,6 +161,7 @@ func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
name: name,
arrayPositions: arrayPositions,
options: options,
encodedValue: encodedValue,
value: value,
numPlainTextBytes: uint64(len(value)),
}
Expand All @@ -149,14 +170,20 @@ func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64,
func NewGeometryCollectionFieldWithIndexingOptions(name string,
arrayPositions []uint64, coordinates [][][][][]float64, types []string,
options index.FieldIndexingOptions) *GeoShapeField {
shape, value, err := geo.NewGeometryCollection(coordinates, types)
shape, encodedValue, err := geo.NewGeometryCollection(coordinates, types)
if err != nil {
return nil
}

// extra glue bytes to work around the term splitting logic from interfering
// the custom encoding of the geoshape coordinates inside the docvalues.
value = append(geo.GlueBytes, append(value, geo.GlueBytes...)...)
encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)

// get the byte value for the geometryCollection.
value, err := shape.Value()
if err != nil {
return nil
}

options = options | DefaultGeoShapeIndexingOptions

Expand All @@ -165,22 +192,29 @@ func NewGeometryCollectionFieldWithIndexingOptions(name string,
name: name,
arrayPositions: arrayPositions,
options: options,
encodedValue: encodedValue,
value: value,
numPlainTextBytes: uint64(len(value)),
}
}

func NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64,
centerPoint []float64, radius float64,
centerPoint []float64, radius string,
options index.FieldIndexingOptions) *GeoShapeField {
shape, value, err := geo.NewGeoCircleShape(centerPoint, radius)
shape, encodedValue, err := geo.NewGeoCircleShape(centerPoint, radius)
if err != nil {
return nil
}

// extra glue bytes to work around the term splitting logic from interfering
// the custom encoding of the geoshape coordinates inside the docvalues.
value = append(geo.GlueBytes, append(value, geo.GlueBytes...)...)
encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...)

// get the byte value for the circle.
value, err := shape.Value()
if err != nil {
return nil
}

options = options | DefaultGeoShapeIndexingOptions

Expand All @@ -189,7 +223,13 @@ func NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64,
name: name,
arrayPositions: arrayPositions,
options: options,
encodedValue: encodedValue,
value: value,
numPlainTextBytes: uint64(len(value)),
}
}

// GeoShape is an implementation of the index.GeoShapeField interface.
func (n *GeoShapeField) GeoShape() (index.GeoJSON, error) {
return geojson.ParseGeoJSONShape(n.value)
}
22 changes: 19 additions & 3 deletions geo/geo_s2plugin_impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,10 @@ func (p *Point) Type() string {
return PointType
}

func (p *Point) Value() ([]byte, error) {
return json.Marshal(p)
}

func (p *Point) Intersects(s index.GeoJSON) (bool, error) {
// placeholder implementation
return false, nil
Expand Down Expand Up @@ -262,6 +266,10 @@ func (br *boundedRectangle) Type() string {
return "boundedRectangle"
}

func (br *boundedRectangle) Value() ([]byte, error) {
return json.Marshal(br)
}

func (p *boundedRectangle) Intersects(s index.GeoJSON) (bool, error) {
// placeholder implementation
return false, nil
Expand Down Expand Up @@ -300,6 +308,10 @@ func (bp *boundedPolygon) Type() string {
return "boundedPolygon"
}

func (bp *boundedPolygon) Value() ([]byte, error) {
return json.Marshal(bp)
}

func (p *boundedPolygon) Intersects(s index.GeoJSON) (bool, error) {
// placeholder implementation
return false, nil
Expand Down Expand Up @@ -342,6 +354,10 @@ func (p *pointDistance) Type() string {
return "pointDistance"
}

func (p *pointDistance) Value() ([]byte, error) {
return json.Marshal(p)
}

func NewPointDistance(centerLat, centerLon,
dist float64) *pointDistance {
return &pointDistance{centerLat: centerLat,
Expand Down Expand Up @@ -388,8 +404,8 @@ func NewGeometryCollection(coordinates [][][][][]float64,
// prefix the byte contents with certain glue bytes that
// can be used later while filering the doc values.
func NewGeoCircleShape(cp []float64,
radiusInMeter float64) (index.GeoJSON, []byte, error) {
return geojson.NewGeoCircleShape(cp, radiusInMeter)
radius string) (index.GeoJSON, []byte, error) {
return geojson.NewGeoCircleShape(cp, radius)
}

func NewGeoJsonShape(coordinates [][][][]float64, typ string) (
Expand Down Expand Up @@ -421,7 +437,7 @@ func NewGeoJsonMultiPolygon(points [][][][]float64) index.GeoJSON {
return geojson.NewGeoJsonMultiPolygon(points)
}

func NewGeoCircle(points []float64, radius float64) index.GeoJSON {
func NewGeoCircle(points []float64, radius string) index.GeoJSON {
return geojson.NewGeoCircle(points, radius)
}

Expand Down
14 changes: 4 additions & 10 deletions geo/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,13 +351,12 @@ func ExtractGeometryCollection(thing interface{}) ([][][][][]float64, []string,
// ExtractCircle takes an interface{} and tries it's best to
// interpret the center point coordinates and the radius for a
// given circle shape.
func ExtractCircle(thing interface{}) ([]float64, float64, bool) {
func ExtractCircle(thing interface{}) ([]float64, string, bool) {
thingVal := reflect.ValueOf(thing)
if !thingVal.IsValid() {
return nil, 0, false
return nil, "", false
}
var rv []float64
var radius float64
var radiusStr string

if thingVal.Kind() == reflect.Map {
Expand All @@ -366,26 +365,21 @@ func ExtractCircle(thing interface{}) ([]float64, float64, bool) {

if iter.Key().String() == "radius" {
radiusStr = iter.Value().Interface().(string)
r, err := ParseDistance(radiusStr)
if err != nil {
return nil, 0, false
}
radius = r
continue
}

if iter.Key().String() == "coordinates" {
lng, lat, found := ExtractGeoPoint(iter.Value().Interface())
if !found {
return nil, 0, false
return nil, radiusStr, false
}
rv = append(rv, lng)
rv = append(rv, lat)
}
}
}

return rv, radius, true
return rv, radiusStr, true
}

// ExtractGeoShapeCoordinates takes an interface{} and tries it's best to
Expand Down
16 changes: 8 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,23 @@ go 1.13
require (
github.com/RoaringBitmap/roaring v0.9.4
github.com/bits-and-blooms/bitset v1.2.0
github.com/blevesearch/bleve_index_api v1.0.2
github.com/blevesearch/geo v0.1.12
github.com/blevesearch/bleve_index_api v1.0.3
github.com/blevesearch/geo v0.1.13
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/blevesearch/go-porterstemmer v1.0.3
github.com/blevesearch/goleveldb v1.0.1
github.com/blevesearch/gtreap v0.1.1
github.com/blevesearch/scorch_segment_api/v2 v2.1.1
github.com/blevesearch/scorch_segment_api/v2 v2.1.2
github.com/blevesearch/segment v0.9.0
github.com/blevesearch/snowball v0.6.1
github.com/blevesearch/snowballstem v0.9.0
github.com/blevesearch/upsidedown_store_api v1.0.1
github.com/blevesearch/vellum v1.0.8
github.com/blevesearch/zapx/v11 v11.3.4
github.com/blevesearch/zapx/v12 v12.3.4
github.com/blevesearch/zapx/v13 v13.3.4
github.com/blevesearch/zapx/v14 v14.3.4
github.com/blevesearch/zapx/v15 v15.3.5-0.20220713163830-ae843e553177
github.com/blevesearch/zapx/v11 v11.3.5
github.com/blevesearch/zapx/v12 v12.3.5
github.com/blevesearch/zapx/v13 v13.3.5
github.com/blevesearch/zapx/v14 v14.3.5
github.com/blevesearch/zapx/v15 v15.3.5-0.20220722171731-5dd118e621d2
github.com/couchbase/moss v0.2.0
github.com/golang/protobuf v1.3.2
github.com/spf13/cobra v0.0.5
Expand Down
34 changes: 16 additions & 18 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
github.com/blevesearch/bleve_index_api v1.0.2 h1:rO736FwEPMVY1mGi7d4n7CgBB3+tB7uYN7QTjR+Ij+s=
github.com/blevesearch/bleve_index_api v1.0.2/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
github.com/blevesearch/geo v0.1.12 h1:GYKvTZv8uhGeN5fkOtEtuwNqq9/GijKsle/5MxkEOQQ=
github.com/blevesearch/geo v0.1.12/go.mod h1:8z6udmXe8Ek8uuX4qOIWKb50vY/OQ1SG+XhL5FrcHOU=
github.com/blevesearch/bleve_index_api v1.0.3 h1:DDSWaPXOZZJ2BB73ZTWjKxydAugjwywcqU+91AAqcAg=
github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4=
github.com/blevesearch/geo v0.1.13 h1:RsY1vfFm81iv1g+uoCQtsOFvKAhZnpOdTOK8JRA6pqw=
github.com/blevesearch/geo v0.1.13/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc=
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA=
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
Expand All @@ -20,9 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE=
github.com/blevesearch/scorch_segment_api/v2 v2.1.1 h1:J8UDudUpDJz21d/hCMIshCeRordwnDTftgXcSDMUx40=
github.com/blevesearch/scorch_segment_api/v2 v2.1.1/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE=
github.com/blevesearch/scorch_segment_api/v2 v2.1.2 h1:TAte9VZLWda5WAVlZTTZ+GCzEHqGJb4iB2aiZSA6Iv8=
github.com/blevesearch/scorch_segment_api/v2 v2.1.2/go.mod h1:rvoQXZGq8drq7vXbNeyiRzdEOwZkjkiYGf1822i6CRA=
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac=
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ=
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
Expand All @@ -33,16 +31,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15a
github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q=
github.com/blevesearch/vellum v1.0.8 h1:iMGh4lfxza4BnWO/UJTMPlI3HsK9YawjPv+TteVa9ck=
github.com/blevesearch/vellum v1.0.8/go.mod h1:+cpRi/tqq49xUYSQN2P7A5zNSNrS+MscLeeaZ3J46UA=
github.com/blevesearch/zapx/v11 v11.3.4 h1:MjYFN8fwDajRgeUsKMfW673zI6MI3twy0pCsUH/LXgc=
github.com/blevesearch/zapx/v11 v11.3.4/go.mod h1:HJ7qdfBxdziuymKvXbsBVhCK5pB98tdzQbc8pJV6tJo=
github.com/blevesearch/zapx/v12 v12.3.4 h1:OpPoHQjsjvDImDzwKZXTXubIPJz28EaRynJGJSS6mvU=
github.com/blevesearch/zapx/v12 v12.3.4/go.mod h1:uQrKrK9XjXAAsJfAIE8ViLqIKP/keA2DQhS1XXpjkwA=
github.com/blevesearch/zapx/v13 v13.3.4 h1:f646k6300VGRIR7eJ6lLtF8UC95NIWmF899j49o7PJA=
github.com/blevesearch/zapx/v13 v13.3.4/go.mod h1:Wl7hO1gT+IDvJb7i06g2iW5Qvw0KzncJPsBx7WGWhLA=
github.com/blevesearch/zapx/v14 v14.3.4 h1:/FVzSGFG5rbVWfPEqlcaJd8lZSJMQpTdmFhz/l2QI7w=
github.com/blevesearch/zapx/v14 v14.3.4/go.mod h1:b1YhRXXhAj9i+9aOwhRKCHUmJyYieK/QbDvPJDLddUk=
github.com/blevesearch/zapx/v15 v15.3.5-0.20220713163830-ae843e553177 h1:0/WYF9nS1HBgDc3z7ePdDz15CwMuNYZ4WeD7Kravm7M=
github.com/blevesearch/zapx/v15 v15.3.5-0.20220713163830-ae843e553177/go.mod h1:ii4ohMQC0TCUjYfq8OtrbABgeI1zljjyXBFpUe/dPDw=
github.com/blevesearch/zapx/v11 v11.3.5 h1:eBQWQ7huA+mzm0sAGnZDwgGGli7S45EO+N+ObFWssbI=
github.com/blevesearch/zapx/v11 v11.3.5/go.mod h1:5UdIa/HRMdeRCiLQOyFESsnqBGiip7vQmYReA9toevU=
github.com/blevesearch/zapx/v12 v12.3.5 h1:5pX2hU+R1aZihT7ac1dNWh1n4wqkIM9pZzWp0ANED9s=
github.com/blevesearch/zapx/v12 v12.3.5/go.mod h1:ANcthYRZQycpbRut/6ArF5gP5HxQyJqiFcuJCBju/ss=
github.com/blevesearch/zapx/v13 v13.3.5 h1:eJ3gbD+Nu8p36/O6lhfdvWQ4pxsGYSuTOBrLLPVWJ74=
github.com/blevesearch/zapx/v13 v13.3.5/go.mod h1:FV+dRnScFgKnRDIp08RQL4JhVXt1x2HE3AOzqYa6fjo=
github.com/blevesearch/zapx/v14 v14.3.5 h1:hEvVjZaagFCvOUJrlFQ6/Z6Jjy0opM3g7TMEo58TwP4=
github.com/blevesearch/zapx/v14 v14.3.5/go.mod h1:954A/eKFb+pg/ncIYWLWCKY+mIjReM9FGTGIO2Wu1cU=
github.com/blevesearch/zapx/v15 v15.3.5-0.20220722171731-5dd118e621d2 h1:kXBfopw0cwKD37xif91wzHqSkP9eyI9MAZv4MNxPR4w=
github.com/blevesearch/zapx/v15 v15.3.5-0.20220722171731-5dd118e621d2/go.mod h1:QMUh2hXCaYIWFKPYGavq/Iga2zbHWZ9DZAa9uFbWyvg=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
Expand Down
2 changes: 1 addition & 1 deletion index/scorch/merge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context,
switch segI := seg.(type) {
case segment.DiskStatsReporter:
totalBytesRead := segI.BytesRead() + prevBytesReadTotal
segI.SetBytesRead(totalBytesRead)
segI.ResetBytesRead(totalBytesRead)
seg = segI.(segment.Segment)
}

Expand Down
2 changes: 2 additions & 0 deletions index/scorch/snapshot_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,8 @@ func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) {
rvd.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value))
case 'g':
rvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value))
case 's':
rvd.AddField(document.NewGeoShapeFieldFromBytes(name, arrayPos, value))
}

return true
Expand Down
6 changes: 6 additions & 0 deletions index_impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,13 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
value = []float64{lon, lat}
}
}
case index.GeoShapeField:
v, err := docF.GeoShape()
if err == nil {
value = v
}
}

if value != nil {
hit.AddFieldValue(docF.Name(), value)
}
Expand Down
4 changes: 2 additions & 2 deletions mapping/field.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,12 +321,12 @@ func (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{},
}

if shape == geo.CircleType {
center, radiusInMeter, found := geo.ExtractCircle(propertyMightBeGeoShape)
center, radius, found := geo.ExtractCircle(propertyMightBeGeoShape)
if found {
fieldName := getFieldName(pathString, path, fm)
options := fm.Options()
field := document.NewGeoCircleFieldWithIndexingOptions(fieldName,
indexes, center, radiusInMeter, options)
indexes, center, radius, options)
context.doc.AddField(field)

if !fm.IncludeInAll {
Expand Down
Loading

0 comments on commit ec0d3aa

Please sign in to comment.