palomar: bit of progress

cmd/palomar/README.md

@@ -6,12 +6,9 @@ Almost all the code for this service is actually in the `search/` directory at t
 
 ## API
 
-### `/search/posts?q=QUERY`
-
-### `/search/profiles?q=QUERY`
-
-### `/search/profiles-typeahead?q=QUERY`
+### `/search/posts?q=QUERY&size=COUNT&offset=INT`
 
+### `/search/profiles?q=QUERY&typeahead=BOOL`
 
 ## Development Quickstart
 
@@ -22,10 +19,13 @@ Run an ephemeral opensearch instance on local port 9200, with SSL disabled, and
 
 In this directory, use HTTPie to create indices:
 
-    #http --verify no put https://admin:admin@localhost:9200/palomar_post < post_schema.json
-    #http --verify no put https://admin:admin@localhost:9200/palomar_profile < profile_schema.json
-    http put http://admin:admin@localhost:9200/palomar_post < post_schema.json
+    # may need to pass '--verify no' as first arg to 'http' in some cases
     http put http://admin:admin@localhost:9200/palomar_profile < profile_schema.json
+    http put http://admin:admin@localhost:9200/palomar_post < post_schema.json
+
+    # if needed to delete any existing first (DATA LOSS)
+    http delete http://admin:admin@localhost:9200/palomar_profile
+    http delete http://admin:admin@localhost:9200/palomar_post
 
 See [README.opensearch.md]() for more Opensearch operational tips.
 

cmd/palomar/main.go

@@ -1,15 +1,17 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	_ "github.com/joho/godotenv/autoload"
 
+	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/search"
 	"github.com/bluesky-social/indigo/util/cliutil"
 
@@ -83,13 +85,6 @@ func run(args []string) error {
 			Value:   "https://plc.directory",
 			EnvVars: []string{"ATP_PLC_HOST"},
 		},
-		// TODO(bnewbold): this is a temporary hack to fetch our own blobs
-		&cli.StringFlag{
-			Name:    "atp-pds-host",
-			Usage:   "method, hostname, and port of PDS instance",
-			Value:   "https://bsky.social",
-			EnvVars: []string{"ATP_PDS_HOST"},
-		},
 		&cli.IntFlag{
 			Name:    "max-metadb-connections",
 			EnvVars: []string{"MAX_METADB_CONNECTIONS"},
@@ -138,14 +133,25 @@ var runCmd = &cli.Command{
 			return fmt.Errorf("failed to get elasticsearch: %w", err)
 		}
 
+		// TODO: replace this with "bingo" resolver
+		base := identity.BaseDirectory{
+			PLCURL: cctx.String("atp-plc-host"),
+			HTTPClient: http.Client{
+				Timeout: time.Second * 15,
+			},
+			TryAuthoritativeDNS: false,
+		}
+		dir := identity.NewCacheDirectory(&base, 50000, time.Hour*24, time.Minute*2)
+
 		srv, err := search.NewServer(
 			db,
 			escli,
-			cctx.String("atp-plc-host"),
-			cctx.String("atp-pds-host"),
-			cctx.String("atp-bgs-host"),
-			cctx.String("es-profile-index"),
-			cctx.String("es-post-index"),
+			&dir,
+			search.Config{
+				BGSHost:      cctx.String("atp-bgs-host"),
+				ProfileIndex: cctx.String("es-profile-index"),
+				PostIndex:    cctx.String("es-post-index"),
+			},
 		)
 		if err != nil {
 			return err
@@ -193,63 +199,36 @@ var elasticCheckCmd = &cli.Command{
 	},
 }
 
-func queryIndex(cctx *cli.Context, index string) error {
-	escli, err := createEsClient(cctx)
-	if err != nil {
-		return err
-	}
-
-	var buf bytes.Buffer
-	var query map[string]interface{}
-	if cctx.Bool("typeahead") == true {
-		query = map[string]interface{}{
-			"query": map[string]interface{}{
-				"multi_match": map[string]interface{}{
-					"query": strings.Join(cctx.Args().Slice(), " "),
-					"type":  "bool_prefix",
-					"fields": []string{
-						"typeahead",
-						"typeahead._2gram",
-						"typeahead._3gram",
-					},
-				},
-			},
-		}
-	} else {
-		query = map[string]interface{}{
-			"query": map[string]interface{}{
-				"query_string": map[string]interface{}{
-					"query":         strings.Join(cctx.Args().Slice(), " "),
-					"default_field": "everything",
-				},
-			},
-		}
-	}
-	if err := json.NewEncoder(&buf).Encode(query); err != nil {
-		log.Fatalf("Error encoding query: %s", err)
+func printHits(resp *search.EsSearchResponse) {
+	fmt.Printf("%d hits in %d\n", len(resp.Hits.Hits), resp.Took)
+	for _, hit := range resp.Hits.Hits {
+		b, _ := json.Marshal(hit.Source)
+		fmt.Println(string(b))
 	}
-
-	// Perform the search request.
-	res, err := escli.Search(
-		escli.Search.WithContext(context.Background()),
-		escli.Search.WithIndex(index),
-		escli.Search.WithBody(&buf),
-		escli.Search.WithTrackTotalHits(true),
-		escli.Search.WithPretty(),
-	)
-	if err != nil {
-		log.Fatalf("Error getting response: %s", err)
-	}
-
-	fmt.Println(res)
-	return nil
+	return
 }
 
 var searchPostCmd = &cli.Command{
 	Name:  "search-post",
 	Usage: "run a simple query against posts index",
 	Action: func(cctx *cli.Context) error {
-		return queryIndex(cctx, cctx.String("es-post-index"))
+		escli, err := createEsClient(cctx)
+		if err != nil {
+			return err
+		}
+		res, err := search.DoSearchPosts(
+			context.Background(),
+			escli,
+			cctx.String("es-post-index"),
+			strings.Join(cctx.Args().Slice(), " "),
+			0,
+			20,
+		)
+		if err != nil {
+			return err
+		}
+		printHits(res)
+		return nil
 	},
 }
 
@@ -262,7 +241,36 @@ var searchProfileCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		return queryIndex(cctx, cctx.String("es-profile-index"))
+		escli, err := createEsClient(cctx)
+		if err != nil {
+			return err
+		}
+		if cctx.Bool("typeahead") {
+			res, err := search.DoSearchProfilesTypeahead(
+				context.Background(),
+				escli,
+				cctx.String("es-profile-index"),
+				strings.Join(cctx.Args().Slice(), " "),
+			)
+			if err != nil {
+				return err
+			}
+			printHits(res)
+		} else {
+			res, err := search.DoSearchProfiles(
+				context.Background(),
+				escli,
+				cctx.String("es-profile-index"),
+				strings.Join(cctx.Args().Slice(), " "),
+				0,
+				20,
+			)
+			if err != nil {
+				return err
+			}
+			printHits(res)
+		}
+		return nil
 	},
 }
 

search/handlers.go

@@ -1,12 +1,10 @@
 package search
 
 import (
-	"context"
 	"fmt"
 	"strconv"
 	"strings"
 
-	api "github.com/bluesky-social/indigo/api"
 	bsky "github.com/bluesky-social/indigo/api/bsky"
 	"github.com/labstack/echo/v4"
 	otel "go.opentelemetry.io/otel"
@@ -17,16 +15,6 @@ type ActorSearchResp struct {
 	DID string `json:"did"`
 }
 
-func (s *Server) handleFromDid(ctx context.Context, did string) (string, error) {
-	phr := &api.ProdHandleResolver{}
-	handle, _, err := api.ResolveDidToHandle(ctx, s.xrpcc, s.plc, phr, did)
-	if err != nil {
-		return "", err
-	}
-
-	return handle, nil
-}
-
 func (s *Server) handleSearchRequestPosts(e echo.Context) error {
 	ctx, span := otel.Tracer("search").Start(e.Request().Context(), "handleSearchRequestPosts")
 	defer span.End()
@@ -83,7 +71,38 @@ func (s *Server) handleSearchRequestProfiles(e echo.Context) error {
 		})
 	}
 
-	out, err := s.SearchProfiles(ctx, q)
+	offset := 0
+	if q := strings.TrimSpace(e.QueryParam("offset")); q != "" {
+		v, err := strconv.Atoi(q)
+		if err != nil {
+			return &echo.HTTPError{
+				Code:    400,
+				Message: fmt.Sprintf("invalid value for 'offset': %s", err),
+			}
+		}
+
+		offset = v
+	}
+
+	count := 30
+	if q := strings.TrimSpace(e.QueryParam("count")); q != "" {
+		v, err := strconv.Atoi(q)
+		if err != nil {
+			return &echo.HTTPError{
+				Code:    400,
+				Message: fmt.Sprintf("invalid value for 'count': %s", err),
+			}
+		}
+
+		count = v
+	}
+
+	typeahead := false
+	if q := strings.TrimSpace(e.QueryParam("typeahead")); q == "true" || q == "1" || q == "y" {
+		typeahead = true
+	}
+
+	out, err := s.SearchProfiles(ctx, q, typeahead, offset, count)
 	if err != nil {
 		return err
 	}