Skip to content

Commit

Permalink
palomar: basic query parsing, handle 'from:'
Browse files Browse the repository at this point in the history
  • Loading branch information
bnewbold committed Sep 1, 2023
1 parent a9e1c05 commit cafef6e
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 19 deletions.
2 changes: 2 additions & 0 deletions cmd/palomar/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ var searchPostCmd = &cli.Command{
}
res, err := search.DoSearchPosts(
context.Background(),
identity.DefaultDirectory(), // TODO: parse PLC arg
escli,
cctx.String("es-post-index"),
strings.Join(cctx.Args().Slice(), " "),
Expand Down Expand Up @@ -259,6 +260,7 @@ var searchProfileCmd = &cli.Command{
} else {
res, err := search.DoSearchProfiles(
context.Background(),
identity.DefaultDirectory(), // TODO: parse PLC arg
escli,
cctx.String("es-profile-index"),
strings.Join(cctx.Args().Slice(), " "),
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49
github.com/goccy/go-json v0.10.2
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/gorilla/websocket v1.5.0
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hashicorp/golang-lru v0.5.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
Expand Down
70 changes: 70 additions & 0 deletions search/parse_query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package search

import (
"context"
"fmt"
"log/slog"
"strings"

"github.com/bluesky-social/indigo/atproto/identity"
"github.com/bluesky-social/indigo/atproto/syntax"

"github.com/google/shlex"
)

// takes a query string and pulls out some facet patterns ("from:handle.net") as filters
func ParseQuery(ctx context.Context, dir identity.Directory, raw string) (string, []map[string]interface{}) {
var filters []map[string]interface{}
parts, err := shlex.Split(raw)
if err != nil {
// pass-through if failed to parse
return raw, filters
}
keep := make([]string, len(parts))
for _, p := range parts {
if !strings.ContainsRune(p, ':') || strings.ContainsRune(p, ' ') {
// simple: quoted (whitespace), or just a token
keep = append(keep, p)
continue
}
if strings.HasPrefix(p, "did:") {
filters = append(filters, map[string]interface{}{
"term": map[string]interface{}{"did": p},
})
continue
}
if strings.HasPrefix(p, "from:") && len(p) > 6 {
handle, err := syntax.ParseHandle(p[5:])
if err != nil {
keep = append(keep, p)
continue
}
id, err := dir.LookupHandle(ctx, handle)
if err != nil {
if err != identity.ErrHandleNotFound {
slog.Error("failed to resolve handle", "err", err)
}
continue
}
filters = append(filters, map[string]interface{}{
"term": map[string]interface{}{"did": id.DID.String()},
})
continue
}
keep = append(keep, p)
}

out := ""
for _, p := range keep {
if strings.ContainsRune(p, ' ') {
out += fmt.Sprintf(" \"%s\"", p)
} else {
if out == "" {
out = p
} else {
out += " " + p
}
}
}
return out, filters
}
43 changes: 43 additions & 0 deletions search/parse_query_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package search

import (
"context"
"testing"

"github.com/bluesky-social/indigo/atproto/identity"
"github.com/bluesky-social/indigo/atproto/syntax"

"github.com/stretchr/testify/assert"
)

func TestParseQuery(t *testing.T) {
ctx := context.Background()
assert := assert.New(t)
dir := identity.NewMockDirectory()
dir.Insert(identity.Identity{
Handle: syntax.Handle("known.example.com"),
DID: syntax.DID("did:plc:abc222"),
})

var q string
var f []map[string]interface{}

q, f = ParseQuery(ctx, &dir, "")
assert.Equal("", q)
assert.Empty(f)

p1 := "some +test \"with phrase\" -ok"
q, f = ParseQuery(ctx, &dir, p1)
assert.Equal(p1, q)
assert.Empty(f)

p2 := "missing from:missing.example.com"
q, f = ParseQuery(ctx, &dir, p2)
assert.Equal("missing", q)
assert.Empty(f)

p3 := "known from:known.example.com"
q, f = ParseQuery(ctx, &dir, p3)
assert.Equal("known", q)
assert.Equal(1, len(f))
}
56 changes: 39 additions & 17 deletions search/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"fmt"
"log/slog"

"github.com/bluesky-social/indigo/atproto/identity"

es "github.com/opensearch-project/opensearch-go/v2"
)

Expand Down Expand Up @@ -52,40 +54,53 @@ func checkParams(offset, size int) error {
return nil
}

func DoSearchPosts(ctx context.Context, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
func DoSearchPosts(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
if err := checkParams(offset, size); err != nil {
return nil, err
}
queryStr, filters := ParseQuery(ctx, dir, q)
basic := map[string]interface{}{
"simple_query_string": map[string]interface{}{
"query": queryStr,
"fields": []string{"everything"},
"flags": "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE",
"default_operator": "and",
"lenient": true,
"analyze_wildcard": false,
},
}
query := map[string]interface{}{
// TODO: filter to not show any created_at in the future
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": basic,
"filter": filters,
},
},
"sort": map[string]any{
"created_at": map[string]any{
"order": "desc",
},
},
"query": map[string]interface{}{
"match": map[string]interface{}{
"everything": map[string]interface{}{
"query": q,
},
},
},
"size": size,
"from": offset,
}

return doSearch(ctx, escli, index, query)
}

func DoSearchProfiles(ctx context.Context, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
if err := checkParams(offset, size); err != nil {
return nil, err
}
queryStr, filters := ParseQuery(ctx, dir, q)
basic := map[string]interface{}{
"match": map[string]interface{}{
"everything": map[string]interface{}{
"query": q,
},
"simple_query_string": map[string]interface{}{
"query": queryStr,
"fields": []string{"everything"},
"flags": "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE",
"default_operator": "and",
"lenient": true,
"analyze_wildcard": false,
},
}
query := map[string]interface{}{
Expand All @@ -96,7 +111,8 @@ func DoSearchProfiles(ctx context.Context, escli *es.Client, index, q string, of
map[string]interface{}{"term": map[string]interface{}{"has_avatar": true}},
map[string]interface{}{"term": map[string]interface{}{"has_banner": true}},
},
"boost": 1.0,
"filter": filters,
"boost": 1.0,
},
},
"size": size,
Expand Down Expand Up @@ -148,7 +164,10 @@ func doSearch(ctx context.Context, escli *es.Client, index string, query interfa
if err := json.NewEncoder(&buf).Encode(query); err != nil {
log.Fatalf("Error encoding query: %s", err)
}
slog.Warn("sending query", "index", index, "query", query)
bod, err := json.Marshal(query)
if nil == err {
slog.Warn("sending query", "index", index, "query", string(bod))
}

// Perform the search request.
res, err := escli.Search(
Expand All @@ -158,7 +177,10 @@ func doSearch(ctx context.Context, escli *es.Client, index string, query interfa
escli.Search.WithTrackTotalHits(false), // expensive to track total hits
)
if err != nil {
log.Fatalf("Error getting response: %s", err)
return nil, fmt.Errorf("search query error: %w", err)
}
if res.IsError() {
return nil, fmt.Errorf("search query error, code=%d", res.StatusCode)
}
defer res.Body.Close()

Expand Down
4 changes: 2 additions & 2 deletions search/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ func (s *Server) processTooBigCommit(ctx context.Context, evt *comatproto.SyncSu
}

func (s *Server) SearchPosts(ctx context.Context, srch string, offset, size int) ([]PostSearchResult, error) {
resp, err := DoSearchPosts(ctx, s.escli, s.postIndex, srch, offset, size)
resp, err := DoSearchPosts(ctx, s.dir, s.escli, s.postIndex, srch, offset, size)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -408,7 +408,7 @@ func (s *Server) SearchProfiles(ctx context.Context, srch string, typeahead bool
if typeahead {
resp, err = DoSearchProfilesTypeahead(ctx, s.escli, s.profileIndex, srch)
} else {
resp, err = DoSearchProfiles(ctx, s.escli, s.profileIndex, srch, offset, size)
resp, err = DoSearchProfiles(ctx, s.dir, s.escli, s.profileIndex, srch, offset, size)
}
if err != nil {
return nil, err
Expand Down

0 comments on commit cafef6e

Please sign in to comment.