diff --git a/Makefile b/Makefile index b66fb2cd..cdc75745 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,17 @@ run: - air --pretty + air -- --pretty run-cli: go run ./src --cli --pretty debug: - air -- -v --pretty + air -- --pretty -v debug-cli: - go run ./srv -v --cli --pretty + go run ./srv --cli --pretty -v trace: - air -- -vv --pretty + air -- --pretty -vv trace-cli: - go run ./src -vv --cli --pretty + go run ./src --cli --pretty -vv install: go get ./... diff --git a/src/cli/climode.go b/src/cli/climode.go index 58215783..e564da82 100644 --- a/src/cli/climode.go +++ b/src/cli/climode.go @@ -61,13 +61,18 @@ func Run(flags Flags, db cache.DB, conf config.Config) { Bool("visit", flags.Visit). Msg("Started hearching") + categoryName, err := category.FromString(flags.Category) + if err != nil { + log.Fatal().Err(err).Msg("Invalid category") + } + options := engines.Options{ Pages: engines.Pages{ Start: flags.StartPage, Max: flags.MaxPages, }, VisitPages: flags.Visit, - Category: category.FromString[flags.Category], + Category: categoryName, UserAgent: flags.UserAgent, Locale: flags.Locale, SafeSearch: flags.SafeSearch, @@ -76,7 +81,7 @@ func Run(flags Flags, db cache.DB, conf config.Config) { start := time.Now() - results, foundInDB := search.Search(flags.Query, options, db, conf.Settings, conf.Categories, conf.Server.Proxy.Salt) + results, foundInDB := search.Search(flags.Query, options, db, conf.Categories[options.Category], conf.Settings, conf.Server.Proxy.Salt) if !flags.Silent { printResults(results) @@ -87,5 +92,5 @@ func Run(flags Flags, db cache.DB, conf config.Config) { Dur("duration", time.Since(start)). Msg("Found results") - search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Settings, conf.Categories, results, foundInDB, conf.Server.Proxy.Salt) + search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Categories[options.Category], conf.Settings, results, foundInDB, conf.Server.Proxy.Salt) } diff --git a/src/cli/setup.go b/src/cli/setup.go index 297af758..347be328 100644 --- a/src/cli/setup.go +++ b/src/cli/setup.go @@ -36,7 +36,7 @@ func Setup() Flags { // ^FATAL } - if category.SafeFromString(cli.Category) == category.UNDEFINED { + if _, err := category.FromString(cli.Category); err != nil { log.Fatal().Msg("cli.Setup(): invalid category flag") // ^FATAL } diff --git a/src/router/search.go b/src/router/search.go index 76b79bdc..5823603b 100644 --- a/src/router/search.go +++ b/src/router/search.go @@ -32,7 +32,7 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config. params := r.Form - query := getParamOrDefault(params, "q") + query := strings.TrimSpace(getParamOrDefault(params, "q")) pagesStartS := getParamOrDefault(params, "start", "1") pagesMaxS := getParamOrDefault(params, "pages", "1") visitPagesS := getParamOrDefault(params, "deep", "false") @@ -42,10 +42,12 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config. safeSearchS := getParamOrDefault(params, "safesearch", "false") mobileS := getParamOrDefault(params, "mobile", "false") - queryWithoutSpaces := strings.TrimSpace(query) - if queryWithoutSpaces == "" || "!"+string(category.FromQuery(query)) == queryWithoutSpaces { - // return "[]" JSON when the query is empty or contains only category name - return writeResponseJSON(w, http.StatusOK, []struct{}{}) + if query == "" { + // user error + return writeResponseJSON(w, http.StatusBadRequest, ErrorResponse{ + Message: "query cannot be empty or whitespace", + Value: "empty query", + }) } pagesMax, err := strconv.Atoi(pagesMaxS) @@ -106,12 +108,12 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config. }) } - categoryName := category.SafeFromString(categoryS) - if categoryName == category.UNDEFINED { + categoryName, err := category.FromString(categoryS) + if err != nil { // user error return writeResponseJSON(w, http.StatusBadRequest, ErrorResponse{ Message: "invalid category value", - Value: fmt.Sprintf("%v", category.UNDEFINED), + Value: fmt.Sprintf("%v", categoryName), }) } @@ -147,13 +149,10 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config. } // search for results in db and web, afterwards return JSON - results, foundInDB := search.Search(query, options, db, settings, categories, salt) - - // get category from query or options - cat := category.FromQueryWithFallback(query, options.Category) + results, foundInDB := search.Search(query, options, db, categories[options.Category], settings, salt) // send response as soon as possible - if cat == category.IMAGES { + if categoryName == category.IMAGES { resultsOutput := result.ConvertToImageOutput(results) err = writeResponseJSON(w, http.StatusOK, resultsOutput) } else { @@ -162,7 +161,7 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config. } // don't return immediately, we want to cache results and update them if necessary - search.CacheAndUpdateResults(query, options, db, ttlConf, settings, categories, results, foundInDB, salt) + search.CacheAndUpdateResults(query, options, db, ttlConf, categories[options.Category], settings, results, foundInDB, salt) // if writing response failed, return the error return err diff --git a/src/search/bang.go b/src/search/bang.go deleted file mode 100644 index 7a8e1f0b..00000000 --- a/src/search/bang.go +++ /dev/null @@ -1,75 +0,0 @@ -package search - -import ( - "strings" - - "github.com/hearchco/hearchco/src/anonymize" - "github.com/hearchco/hearchco/src/config" - "github.com/hearchco/hearchco/src/search/category" - "github.com/hearchco/hearchco/src/search/engines" - "github.com/rs/zerolog/log" -) - -func procBang(query string, setCategory category.Name, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category) (string, category.Name, config.CategoryTimings, []engines.Name) { - useSpec, specEng := procSpecificEngine(query, settings) - goodCat, cat := procCategory(query, setCategory) - if !goodCat && !useSpec && (query != "" && query[0] == '!') { - // cat is set to GENERAL - log.Debug(). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("search.procBang(): invalid bang (not category or engine shortcut)") - } - - query = trimBang(query) - - if useSpec { - return query, category.GENERAL, categories[category.GENERAL].Timings, []engines.Name{specEng} - } else { - return query, cat, categories[cat].Timings, categories[cat].Engines - } -} - -// takes the bang out of the query performs TrimSpace -func trimBang(query string) string { - query = strings.TrimSpace(query) - - if query == "" || query[0] != '!' { - return query - } - - sp := strings.SplitN(query, " ", 2) - if len(sp) == 1 { - // only the bang is present - return "" - } - - return strings.TrimSpace(sp[1]) -} - -func procSpecificEngine(query string, settings map[engines.Name]config.Settings) (bool, engines.Name) { - if query == "" || query[0] != '!' { - return false, engines.UNDEFINED - } - sp := strings.SplitN(query, " ", 2) - bangWord := sp[0][1:] - for key, val := range settings { - if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) { - return true, key - } - } - - return false, engines.UNDEFINED -} - -// returns category in the query if a valid category is present -func procCategory(query string, setCategory category.Name) (bool, category.Name) { - cat := category.FromQuery(query) - if cat != "" { - return true, cat - } else if setCategory == "" { - return false, category.GENERAL - } else { - return false, setCategory - } -} diff --git a/src/search/cache.go b/src/search/cache.go index fbb11f58..325f2b70 100644 --- a/src/search/cache.go +++ b/src/search/cache.go @@ -4,7 +4,6 @@ import ( "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/cache" "github.com/hearchco/hearchco/src/config" - "github.com/hearchco/hearchco/src/search/category" "github.com/hearchco/hearchco/src/search/engines" "github.com/hearchco/hearchco/src/search/result" "github.com/rs/zerolog/log" @@ -12,7 +11,7 @@ import ( func CacheAndUpdateResults( query string, options engines.Options, db cache.DB, - ttlConf config.TTL, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category, + ttlConf config.TTL, categoryConf config.Category, settings map[engines.Name]config.Settings, results []result.Result, foundInDB bool, salt string, ) { @@ -46,7 +45,7 @@ func CacheAndUpdateResults( Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Updating results...") - newResults := PerformSearch(query, options, settings, categories, salt) + newResults := PerformSearch(query, options, categoryConf, settings, salt) uerr := db.Set(query, newResults, ttlConf.Time) if uerr != nil { // Error in updating cache is not returned, just logged diff --git a/src/search/category/category.go b/src/search/category/category.go index 2ea0520a..3fe47457 100644 --- a/src/search/category/category.go +++ b/src/search/category/category.go @@ -1,10 +1,8 @@ package category -import ( - "strings" -) +import "fmt" -var FromString = map[string]Name{ +var catMap = map[string]Name{ "general": GENERAL, "images": IMAGES, "science": SCIENCE, @@ -13,36 +11,18 @@ var FromString = map[string]Name{ "slow": THOROUGH, } -// returns category -func FromQuery(query string) Name { - if query == "" || query[0] != '!' { - return "" - } - cat := strings.SplitN(query, " ", 2)[0][1:] - if val, ok := FromString[cat]; ok { - return val - } - return "" -} - -func SafeFromString(cat string) Name { +// converts a string to a category name if it exists +// if the string is empty, then GENERAL is returned +// otherwise returns UNDEFINED +func FromString(cat string) (Name, error) { if cat == "" { - return "" + return GENERAL, nil } - ret, ok := FromString[cat] + + catName, ok := catMap[cat] if !ok { - return UNDEFINED + return UNDEFINED, fmt.Errorf("category %q is not defined", cat) } - return ret -} -func FromQueryWithFallback(query string, fallback Name) Name { - cat := FromQuery(query) - if cat != "" { - return cat - } else if fallback != "" { - return fallback - } else { - return GENERAL - } + return catName, nil } diff --git a/src/search/engines/_engines_test/tester.go b/src/search/engines/_engines_test/tester.go index b367ccea..1cb64947 100644 --- a/src/search/engines/_engines_test/tester.go +++ b/src/search/engines/_engines_test/tester.go @@ -13,14 +13,14 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult // TestCaseHasAnyResults for _, tc := range tchar { - if results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, ""); len(results) == 0 { + if results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, ""); len(results) == 0 { defer t.Errorf("Got no results for %q", tc.Query) } } // TestCaseContainsResults for _, tc := range tccr { - results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, "") + results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, "") if len(results) == 0 { defer t.Errorf("Got no results for %q", tc.Query) } else { @@ -43,7 +43,7 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult // TestCaseRankedResults for _, tc := range tcrr { - results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, "") + results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, "") if len(results) == 0 { defer t.Errorf("Got no results for %q", tc.Query) } else if len(results) < len(tc.ResultURL) { diff --git a/src/search/perform.go b/src/search/perform.go index fbfd3f5f..552ba1a9 100644 --- a/src/search/perform.go +++ b/src/search/perform.go @@ -9,55 +9,47 @@ import ( "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/config" "github.com/hearchco/hearchco/src/search/bucket" - "github.com/hearchco/hearchco/src/search/category" "github.com/hearchco/hearchco/src/search/engines" "github.com/hearchco/hearchco/src/search/rank" "github.com/hearchco/hearchco/src/search/result" "github.com/rs/zerolog/log" ) -func PerformSearch(query string, options engines.Options, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category, salt string) []result.Result { +func PerformSearch(query string, options engines.Options, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) []result.Result { + // check for empty query if query == "" { log.Trace().Msg("Empty search query.") return []result.Result{} } + // start searching searchTimer := time.Now() - - query, cat, timings, enginesToRun := procBang(query, options.Category, settings, categories) - // set the new category only within the scope of this function - options.Category = cat - query = url.QueryEscape(query) - - // check again after the bang is taken out - if query == "" { - log.Trace().Msg("Empty search query (with bang present).") - return []result.Result{} - } - log.Debug(). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("Searching") + Msg("Searching...") + // getting results from engines resTimer := time.Now() log.Debug().Msg("Waiting for results from engines...") - resultMap := runEngines(enginesToRun, query, options, settings, timings, salt) + resultMap := runEngines(categoryConf.Engines, url.QueryEscape(query), options, settings, categoryConf.Timings, salt) log.Debug(). Dur("duration", time.Since(resTimer)). Msg("Got results") + // ranking results rankTimer := time.Now() log.Debug().Msg("Ranking...") - results := rank.Rank(resultMap, categories[options.Category].Ranking) + results := rank.Rank(resultMap, categoryConf.Ranking) log.Debug(). Dur("duration", time.Since(rankTimer)). Msg("Finished ranking") + // finish searching log.Debug(). Dur("duration", time.Since(searchTimer)). Msg("Found results") diff --git a/src/search/search.go b/src/search/search.go index 3e0cf24c..0865ece7 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -4,13 +4,12 @@ import ( "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/cache" "github.com/hearchco/hearchco/src/config" - "github.com/hearchco/hearchco/src/search/category" "github.com/hearchco/hearchco/src/search/engines" "github.com/hearchco/hearchco/src/search/result" "github.com/rs/zerolog/log" ) -func Search(query string, options engines.Options, db cache.DB, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category, salt string) ([]result.Result, bool) { +func Search(query string, options engines.Options, db cache.DB, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) ([]result.Result, bool) { var results []result.Result var foundInDB bool gerr := db.Get(query, &results) @@ -39,7 +38,7 @@ func Search(query string, options engines.Options, db cache.DB, settings map[eng Msg("Nothing found in cache, doing a clean search") // the main line - results = PerformSearch(query, options, settings, categories, salt) + results = PerformSearch(query, options, categoryConf, settings, salt) result.Shorten(results, 2500) }