diff --git a/cmd/karpor/app/options/ai.go b/cmd/karpor/app/options/ai.go new file mode 100644 index 00000000..2603e5b2 --- /dev/null +++ b/cmd/karpor/app/options/ai.go @@ -0,0 +1,69 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "github.com/KusionStack/karpor/pkg/kubernetes/registry" + "github.com/spf13/pflag" +) + +type AIOptions struct { + AIBackend string + AIAuthToken string + AIBaseURL string + AIModel string + AITemperature float32 + AITopP float32 +} + +const ( + defaultBackend = "openai" + defaultModel = "gpt-3.5-turbo" + defaultTemperature = 1 + defaultTopP = 1 +) + +func NewAIOptions() *AIOptions { + return &AIOptions{} +} + +func (o *AIOptions) Validate() []error { + return nil +} + +func (o *AIOptions) ApplyTo(config *registry.ExtraConfig) error { + // Apply the AIOptions to the provided config + config.AIBackend = o.AIBackend + config.AIAuthToken = o.AIAuthToken + config.AIBaseURL = o.AIBaseURL + config.AIModel = o.AIModel + config.AITemperature = o.AITemperature + config.AITopP = o.AITopP + return nil +} + +// AddFlags adds flags for a specific Option to the specified FlagSet +func (o *AIOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.StringVar(&o.AIBackend, "ai-backend", defaultBackend, "The ai backend") + fs.StringVar(&o.AIAuthToken, "ai-auth-token", "", "The ai auth token") + fs.StringVar(&o.AIBaseURL, "ai-base-url", "", "The ai base url") + fs.StringVar(&o.AIModel, "ai-model", defaultModel, "The ai model") + fs.Float32Var(&o.AITemperature, "ai-temperature", defaultTemperature, "The ai temperature") + fs.Float32Var(&o.AITopP, "ai-top-p", defaultTopP, "The ai top-p") +} diff --git a/cmd/karpor/app/server.go b/cmd/karpor/app/server.go index 17d57d57..345f42fa 100644 --- a/cmd/karpor/app/server.go +++ b/cmd/karpor/app/server.go @@ -58,6 +58,7 @@ type Options struct { RecommendedOptions *options.RecommendedOptions SearchStorageOptions *options.SearchStorageOptions CoreOptions *options.CoreOptions + AIOptions *options.AIOptions StdOut io.Writer StdErr io.Writer @@ -74,6 +75,7 @@ func NewOptions(out, errOut io.Writer) (*Options, error) { ), SearchStorageOptions: options.NewSearchStorageOptions(), CoreOptions: options.NewCoreOptions(), + AIOptions: options.NewAIOptions(), StdOut: out, StdErr: errOut, } @@ -105,6 +107,9 @@ func NewServerCommand(ctx context.Context) *cobra.Command { expvar.Publish("StorageOptions", expvar.Func(func() interface{} { return o.SearchStorageOptions })) + expvar.Publish("AIOptions", expvar.Func(func() interface{} { + return o.AIOptions + })) expvar.Publish("Version", expvar.Func(func() interface{} { return version.GetVersion() })) @@ -140,6 +145,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { o.RecommendedOptions.AddFlags(fs) o.SearchStorageOptions.AddFlags(fs) o.CoreOptions.AddFlags(fs) + o.AIOptions.AddFlags(fs) } // Validate validates Options @@ -147,6 +153,7 @@ func (o *Options) Validate(args []string) error { errors := []error{} errors = append(errors, o.RecommendedOptions.Validate()...) errors = append(errors, o.SearchStorageOptions.Validate()...) + errors = append(errors, o.AIOptions.Validate()...) return utilerrors.NewAggregate(errors) } @@ -208,6 +215,9 @@ func (o *Options) Config() (*server.Config, error) { if err := o.CoreOptions.ApplyTo(config.ExtraConfig); err != nil { return nil, err } + if err := o.AIOptions.ApplyTo(config.ExtraConfig); err != nil { + return nil, err + } config.GenericConfig.BuildHandlerChainFunc = func(handler http.Handler, c *genericapiserver.Config) http.Handler { handler = genericapiserver.DefaultBuildHandlerChain(handler, c) diff --git a/go.mod b/go.mod index 0ca1836e..a3dc682f 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,12 @@ require ( github.com/go-chi/render v1.0.3 github.com/go-logr/logr v1.2.3 github.com/google/gofuzz v1.2.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.4.0 + github.com/hupe1980/go-huggingface v0.0.15 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 github.com/pkg/errors v0.9.1 + github.com/sashabaranov/go-openai v1.27.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 792ef9fe..948b2fe0 100644 --- a/go.sum +++ b/go.sum @@ -254,8 +254,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -275,6 +275,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/hupe1980/go-huggingface v0.0.15 h1:tTWmUGGunC/BYz4hrwS8SSVtMYVYjceG2uhL8HxeXvw= +github.com/hupe1980/go-huggingface v0.0.15/go.mod h1:IRvsik3+b9BJyw9hCfw1arI6gDObcVto1UA8f3kt8mM= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -395,6 +397,8 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sashabaranov/go-openai v1.27.0 h1:L3hO6650YUbKrbGUC6yCjsUluhKZ9h1/jcgbTItI8Mo= +github.com/sashabaranov/go-openai v1.27.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/pkg/core/handler/search/search.go b/pkg/core/handler/search/search.go index ee53b1ea..054b2640 100644 --- a/pkg/core/handler/search/search.go +++ b/pkg/core/handler/search/search.go @@ -19,6 +19,7 @@ import ( "strconv" "github.com/KusionStack/karpor/pkg/core/handler" + "github.com/KusionStack/karpor/pkg/core/manager/ai" "github.com/KusionStack/karpor/pkg/core/manager/search" "github.com/KusionStack/karpor/pkg/infra/search/storage" "github.com/KusionStack/karpor/pkg/util/ctxutil" @@ -34,7 +35,7 @@ import ( // @Tags search // @Produce json // @Param query query string true "The query to use for search. Required" -// @Param pattern query string true "The search pattern. Can be either sql or dsl. Required" +// @Param pattern query string true "The search pattern. Can be either sql, dsl or nl. Required" // @Param pageSize query string false "The size of the page. Default to 10" // @Param page query string false "The current page to fetch. Default to 1" // @Success 200 {array} runtime.Object "Array of runtime.Object" @@ -45,7 +46,7 @@ import ( // @Failure 429 {string} string "Too Many Requests" // @Failure 500 {string} string "Internal Server Error" // @Router /rest-api/v1/search [get] -func SearchForResource(searchMgr *search.SearchManager, searchStorage storage.SearchStorage) http.HandlerFunc { +func SearchForResource(searchMgr *search.SearchManager, aiMgr *ai.AIManager, searchStorage storage.SearchStorage) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // Extract the context and logger from the request. ctx := r.Context() @@ -71,9 +72,43 @@ func SearchForResource(searchMgr *search.SearchManager, searchStorage storage.Se searchPage = 1 } + query := searchQuery + + if searchPattern == storage.NLPatternType { + if err := ai.CheckAIManager(aiMgr); err != nil { + handler.FailureRender(ctx, w, r, err) + return + } + res, err := aiMgr.ConvertTextToSQL(searchQuery) + if err != nil { + handler.FailureRender(ctx, w, r, err) + return + } + searchQuery = res + } + logger.Info("Searching for resources...", "page", searchPage, "pageSize", searchPageSize) res, err := searchStorage.Search(ctx, searchQuery, searchPattern, &storage.Pagination{Page: searchPage, PageSize: searchPageSize}) + if err != nil { + if searchPattern == storage.NLPatternType { + fixedQuery, fixErr := aiMgr.FixSQL(query, searchQuery, err.Error()) + if fixErr != nil { + handler.FailureRender(ctx, w, r, err) + return + } + searchQuery = fixedQuery + res, err = searchStorage.Search(ctx, searchQuery, searchPattern, &storage.Pagination{Page: searchPage, PageSize: searchPageSize}) + if err != nil { + handler.FailureRender(ctx, w, r, err) + return + } + } else { + handler.FailureRender(ctx, w, r, err) + return + } + } + if err != nil { handler.FailureRender(ctx, w, r, err) return @@ -97,6 +132,7 @@ func SearchForResource(searchMgr *search.SearchManager, searchStorage storage.Se Deleted: res.Deleted, }) } + rt.SQLQuery = searchQuery rt.Total = res.Total rt.CurrentPage = searchPage rt.PageSize = searchPageSize diff --git a/pkg/core/manager/ai/manager.go b/pkg/core/manager/ai/manager.go new file mode 100644 index 00000000..2333abfa --- /dev/null +++ b/pkg/core/manager/ai/manager.go @@ -0,0 +1,47 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "github.com/KusionStack/karpor/pkg/infra/ai" + "github.com/KusionStack/karpor/pkg/kubernetes/registry" +) + +type AIManager struct { + client ai.AIProvider +} + +// NewAIManager returns a new AIManager object +func NewAIManager(c registry.ExtraConfig) (*AIManager, error) { + if c.AIAuthToken == "" { + return nil, ErrMissingAuthToken + } + aiClient := ai.NewClient(c.AIBackend) + if err := aiClient.Configure(ai.ConvertToAIConfig(c)); err != nil { + return nil, err + } + + return &AIManager{ + client: aiClient, + }, nil +} + +// CheckAIManager check if the AI manager is created +func CheckAIManager(aiMgr *AIManager) error { + if aiMgr == nil { + return ErrMissingAuthToken + } + return nil +} diff --git a/pkg/core/manager/ai/search.go b/pkg/core/manager/ai/search.go new file mode 100644 index 00000000..7f675a7d --- /dev/null +++ b/pkg/core/manager/ai/search.go @@ -0,0 +1,47 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "context" + "fmt" + + "github.com/KusionStack/karpor/pkg/infra/ai" +) + +// ConvertTextToSQL converts natural language text to an SQL query +func (a *AIManager) ConvertTextToSQL(query string) (string, error) { + servicePrompt := ai.ServicePromptMap[ai.Text2sqlType] + prompt := fmt.Sprintf(servicePrompt, query) + res, err := a.client.Generate(context.Background(), prompt) + if err != nil { + return "", err + } + if IsInvalidQuery(res) { + return "", ErrInvalidQuery + } + return ExtractSelectSQL(res), nil +} + +// FixSQL fix the error SQL +func (a *AIManager) FixSQL(sql string, query string, sqlErr string) (string, error) { + servicePrompt := ai.ServicePromptMap[ai.SQLFixType] + prompt := fmt.Sprintf(servicePrompt, query, sql, sqlErr) + res, err := a.client.Generate(context.Background(), prompt) + if err != nil { + return "", err + } + return ExtractSelectSQL(res), nil +} diff --git a/pkg/core/manager/ai/types.go b/pkg/core/manager/ai/types.go new file mode 100644 index 00000000..eedd6285 --- /dev/null +++ b/pkg/core/manager/ai/types.go @@ -0,0 +1,22 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import "errors" + +var ( + ErrMissingAuthToken = errors.New("auth token is required") + ErrInvalidQuery = errors.New("query is invalid") +) diff --git a/pkg/core/manager/ai/util.go b/pkg/core/manager/ai/util.go new file mode 100644 index 00000000..275849b0 --- /dev/null +++ b/pkg/core/manager/ai/util.go @@ -0,0 +1,32 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "regexp" + "strings" +) + +// IsInvalidQuery check if the query is invalid +func IsInvalidQuery(sql string) bool { + return strings.Contains(strings.ToLower(sql), "error") +} + +// ExtractSelectSQL extracts SQL statements that start with "SELECT * FROM" +func ExtractSelectSQL(sql string) string { + res := regexp.MustCompile(`(?i)SELECT \* FROM [^;]+`) + match := res.FindString(sql) + return match +} diff --git a/pkg/core/manager/ai/util_test.go b/pkg/core/manager/ai/util_test.go new file mode 100644 index 00000000..e1e2cd85 --- /dev/null +++ b/pkg/core/manager/ai/util_test.go @@ -0,0 +1,72 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestExtractSelectSQL tests the correctness of the ExtractSelectSQL function. +func TestExtractSelectSQL(t *testing.T) { + testCases := []struct { + name string + sql string + expected string + }{ + { + name: "NormalCase", + sql: "Q: 所有kind=namespace " + + "Schema_links: [kind, namespace] " + + "SQL: select * from resources where kind='namespace';", + expected: "select * from resources where kind='namespace'", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := ExtractSelectSQL(tc.sql) + require.Equal(t, tc.expected, actual) + }) + } +} + +// TestIsInvalidQuery tests the IsInvalidQuery function. +func TestIsInvalidQuery(t *testing.T) { + testCases := []struct { + name string + sql string + expected bool + }{ + { + name: "ValidQueryWithoutError", + sql: "select * from resources where kind='namespace';", + expected: false, + }, + { + name: "InvalidQuery", + sql: "Error", + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsInvalidQuery(tc.sql) + require.Equal(t, tc.expected, actual) + }) + } +} diff --git a/pkg/core/manager/search/types.go b/pkg/core/manager/search/types.go index d51caeeb..deea50c7 100644 --- a/pkg/core/manager/search/types.go +++ b/pkg/core/manager/search/types.go @@ -35,6 +35,7 @@ type UniResource struct { type UniResourceList struct { metav1.TypeMeta Items []UniResource `json:"items"` + SQLQuery string `json:"sqlQuery"` Total int `json:"total"` CurrentPage int `json:"currentPage"` PageSize int `json:"pageSize"` diff --git a/pkg/core/route/route.go b/pkg/core/route/route.go index d172c984..4816a100 100644 --- a/pkg/core/route/route.go +++ b/pkg/core/route/route.go @@ -15,6 +15,7 @@ package route import ( + "errors" "expvar" docs "github.com/KusionStack/karpor/api/openapispec" @@ -31,6 +32,7 @@ import ( summaryhandler "github.com/KusionStack/karpor/pkg/core/handler/summary" topologyhandler "github.com/KusionStack/karpor/pkg/core/handler/topology" healthhandler "github.com/KusionStack/karpor/pkg/core/health" + aimanager "github.com/KusionStack/karpor/pkg/core/manager/ai" clustermanager "github.com/KusionStack/karpor/pkg/core/manager/cluster" insightmanager "github.com/KusionStack/karpor/pkg/core/manager/insight" resourcegroupmanager "github.com/KusionStack/karpor/pkg/core/manager/resourcegroup" @@ -43,6 +45,7 @@ import ( "github.com/go-chi/chi/v5/middleware" httpswagger "github.com/swaggo/http-swagger/v2" genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/klog/v2" ) // NewCoreRoute creates and configures an instance of chi.Mux with the given @@ -89,6 +92,14 @@ func NewCoreRoute( if err != nil { return nil, err } + aiMgr, err := aimanager.NewAIManager(*extraConfig) + if err != nil { + if errors.Is(err, aimanager.ErrMissingAuthToken) { + klog.Warning("Auth token is empty.") + } else { + return nil, err + } + } clusterMgr := clustermanager.NewClusterManager() searchMgr := searchmanager.NewSearchManager() @@ -96,6 +107,7 @@ func NewCoreRoute( // Set up the API routes for version 1 of the API. router.Route("/rest-api/v1", func(r chi.Router) { setupRestAPIV1(r, + aiMgr, clusterMgr, insightMgr, resourceGroupMgr, @@ -122,6 +134,7 @@ func NewCoreRoute( // resource type and setting up proper handlers. func setupRestAPIV1( r chi.Router, + aiMgr *aimanager.AIManager, clusterMgr *clustermanager.ClusterManager, insightMgr *insightmanager.InsightManager, resourceGroupMgr *resourcegroupmanager.ResourceGroupManager, @@ -146,7 +159,7 @@ func setupRestAPIV1( }) r.Route("/search", func(r chi.Router) { - r.Get("/", searchhandler.SearchForResource(searchMgr, searchStorage)) + r.Get("/", searchhandler.SearchForResource(searchMgr, aiMgr, searchStorage)) }) r.Route("/insight", func(r chi.Router) { diff --git a/pkg/infra/ai/azureopenai.go b/pkg/infra/ai/azureopenai.go new file mode 100644 index 00000000..70c80b7c --- /dev/null +++ b/pkg/infra/ai/azureopenai.go @@ -0,0 +1,67 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "context" + "errors" + + "github.com/sashabaranov/go-openai" +) + +type AzureAIClient struct { + client *openai.Client + model string + temperature float32 +} + +func (c *AzureAIClient) Configure(cfg AIConfig) error { + if cfg.BaseURL == "" { + return errors.New("base url was not provided") + } + + defaultConfig := openai.DefaultAzureConfig(cfg.AuthToken, cfg.BaseURL) + + client := openai.NewClientWithConfig(defaultConfig) + if client == nil { + return errors.New("error creating Azure OpenAI client") + } + + c.client = client + c.model = cfg.Model + c.temperature = cfg.Temperature + return nil +} + +func (c *AzureAIClient) Generate(ctx context.Context, prompt string) (string, error) { + resp, err := c.client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{ + Model: c.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleUser, + Content: prompt, + }, + }, + Temperature: c.temperature, + }) + if err != nil { + return "", err + } + + if len(resp.Choices) == 0 { + return "", errors.New("no completion choices returned from response") + } + return resp.Choices[0].Message.Content, nil +} diff --git a/pkg/infra/ai/huggingface.go b/pkg/infra/ai/huggingface.go new file mode 100644 index 00000000..bfe09a93 --- /dev/null +++ b/pkg/infra/ai/huggingface.go @@ -0,0 +1,54 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "context" + + "github.com/hupe1980/go-huggingface" +) + +type HuggingfaceClient struct { + client *huggingface.InferenceClient + model string + temperature float32 +} + +func (c *HuggingfaceClient) Configure(cfg AIConfig) error { + client := huggingface.NewInferenceClient(cfg.AuthToken) + + c.client = client + c.model = cfg.Model + c.temperature = cfg.Temperature + return nil +} + +func (c *HuggingfaceClient) Generate(ctx context.Context, prompt string) (string, error) { + resp, err := c.client.TextGeneration(ctx, &huggingface.TextGenerationRequest{ + Inputs: prompt, + Parameters: huggingface.TextGenerationParameters{ + Temperature: huggingface.PTR(float64(c.temperature)), + }, + Options: huggingface.Options{ + WaitForModel: huggingface.PTR(true), + }, + Model: c.model, + }) + if err != nil { + return "", err + } + + return resp[0].GeneratedText[len(prompt):], nil +} diff --git a/pkg/infra/ai/openai.go b/pkg/infra/ai/openai.go new file mode 100644 index 00000000..1150776b --- /dev/null +++ b/pkg/infra/ai/openai.go @@ -0,0 +1,69 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "context" + "errors" + + "github.com/sashabaranov/go-openai" +) + +type OpenAIClient struct { + client *openai.Client + model string + temperature float32 + topP float32 +} + +func (c *OpenAIClient) Configure(cfg AIConfig) error { + defaultConfig := openai.DefaultConfig(cfg.AuthToken) + if cfg.BaseURL != "" { + defaultConfig.BaseURL = cfg.BaseURL + } + + client := openai.NewClientWithConfig(defaultConfig) + if client == nil { + return errors.New("error creating OpenAI client") + } + + c.client = client + c.model = cfg.Model + c.temperature = cfg.Temperature + c.topP = cfg.TopP + return nil +} + +func (c *OpenAIClient) Generate(ctx context.Context, prompt string) (string, error) { + resp, err := c.client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{ + Model: c.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleUser, + Content: prompt, + }, + }, + Temperature: c.temperature, + TopP: c.topP, + }) + if err != nil { + return "", err + } + + if len(resp.Choices) == 0 { + return "", errors.New("no completion choices returned from response") + } + return resp.Choices[0].Message.Content, nil +} diff --git a/pkg/infra/ai/prompts.go b/pkg/infra/ai/prompts.go new file mode 100644 index 00000000..7d2ba872 --- /dev/null +++ b/pkg/infra/ai/prompts.go @@ -0,0 +1,105 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +const ( + defaultPrompt = "You are a helpful assistant." + + text2sqlPrompt = ` + You are an AI specialized in writing SQL queries. + Please convert the text: "%s" to sql. + If the text is not accurate enough, please output "Error". + The output tokens only need to give the SQL first, the other thought process please do not give. + The SQL should begin with "select * from" and end with ";". + + 1. The database now only supports one table resources. + + Table resources, columns = [cluster, apiVersion, kind, + namespace, name, creationTimestamp, deletionTimestamp, ownerReferences, + resourceVersion, labels.[key], annotations.[key], content] + + 2. find the schema_links for generating SQL queries for each question based on the database schema. + If there are Chinese expressions, please translate them into English. + + Follow are some examples. + + Q: find the kind which is not equal to pod + A: Let’s think step by step. In the question "find the kind column which is not equal to pod", we are asked: + "find the kind" so we need column = [kind]. + Based on the columns, the set of possible cell values are = [pod]. + So the Schema_links are: + Schema_links: [kind, pod] + + Q: find the kind Deployment which created before January 1, 2024, at 18:00:00 + A: Let’s think step by step. In the question "find the kind Deployment which created before January 1, 2024, at 18:00:00", we are asked: + "find the kind Deployment" so we need column = [kind]. + "created before" so we need column = [creationTimestamp]. + Based on the columns, the set of possible cell values are = [Deployment, 2024-01-01T18:00:00Z]. + So the Schema_links are: + Schema_links: [[kind, Deployment], [creationTimestamp, 2024-01-01T18:00:00Z]] + + Q: find the kind Namespace which which created + A: Let’s think step by step. In the question "find the kind", we are asked: + "find the kind Namespace " so we need column = [kind] + "created before" so we need column = [creationTimestamp] + Based on the columns, the set of possible cell values are = [kind, creationTimestamp]. + There is no creationTimestamp corresponding cell values, so the text is not accurate enough. + So the Schema_links are: + Schema_links: error + + 3. Use the the schema links to generate the SQL queries for each of the questions. + + Follow are some examples. + + Q: find the kind which is not equal to pod + Schema_links: [kind, pod] + SQL: select * from resources where kind!='Pod'; + + Q: find the kind Deployment which created before January 1, 2024, at 18:00:00 + Schema_links: [[kind, Deployment], [creationTimestamp, 2024-01-01T18:00:00Z]] + SQL: select * from resources where kind='Deployment' and creationTimestamp < '2024-01-01T18:00:00Z'; + + Q: find the namespace which does not contain banan + Schema_links: [namespace, banan] + SQL: select * from resources where namespace notlike 'banan_'; + + Q: find the kind Namespace which which created + Schema_links: error + Error; + + Please convert the text to sql. + ` + + sqlFixPrompt = ` + You are an AI specialized in writing SQL queries. + Please convert the text: "%s" to sql. + The SQL should begin with "select * from". + + The database now only supports one table resources. + + Table resources, columns = [cluster, apiVersion, kind, + namespace, name, creationTimestamp, deletionTimestamp, ownerReferences, + resourceVersion, labels.[key], annotations.[key], content] + + After we executed SQL: "%s", we observed the following error "%s". + Please fix the SQL. + ` +) + +var ServicePromptMap = map[string]string{ + "default": defaultPrompt, + "Text2sql": text2sqlPrompt, + "SqlFix": sqlFixPrompt, +} diff --git a/pkg/infra/ai/types.go b/pkg/infra/ai/types.go new file mode 100644 index 00000000..d4321ddf --- /dev/null +++ b/pkg/infra/ai/types.go @@ -0,0 +1,77 @@ +// Copyright The Karpor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ai + +import ( + "context" + + "github.com/KusionStack/karpor/pkg/kubernetes/registry" +) + +const ( + AzureProvider = "azureopenai" + HuggingFaceProvider = "huggingface" + OpenAIProvider = "openai" +) + +const ( + Text2sqlType = "Text2sql" + SQLFixType = "SqlFix" +) + +var clients = map[string]AIProvider{ + AzureProvider: &AzureAIClient{}, + HuggingFaceProvider: &HuggingfaceClient{}, + OpenAIProvider: &OpenAIClient{}, +} + +// AIProvider is an interface all AI clients. +type AIProvider interface { + // Configure sets up the AI service with the provided configuration. + Configure(config AIConfig) error + // Generate generates a response from the AI service based on + // the provided prompt and service type. + Generate(ctx context.Context, prompt string) (string, error) +} + +// AIConfig represents the configuration settings for an AI client. +type AIConfig struct { + Name string + AuthToken string + BaseURL string + Model string + Temperature float32 + TopP float32 +} + +func ConvertToAIConfig(c registry.ExtraConfig) AIConfig { + return AIConfig{ + Name: c.AIBackend, + AuthToken: c.AIAuthToken, + BaseURL: c.AIBaseURL, + Model: c.AIModel, + Temperature: c.AITemperature, + TopP: c.AITopP, + } +} + +// NewClient returns a new AIProvider object +func NewClient(name string) AIProvider { + if client, exists := clients[name]; exists { + return client + } + // default client + return &OpenAIClient{} +} diff --git a/pkg/infra/search/storage/elasticsearch/search.go b/pkg/infra/search/storage/elasticsearch/search.go index f5c141ac..02d93a96 100644 --- a/pkg/infra/search/storage/elasticsearch/search.go +++ b/pkg/infra/search/storage/elasticsearch/search.go @@ -47,7 +47,7 @@ func (s *Storage) Search(ctx context.Context, queryStr, patternType string, pagi if err != nil { return nil, errors.Wrap(err, "search by DSL failed") } - case storage.SQLPatternType: + case storage.SQLPatternType, storage.NLPatternType: sr, err = s.searchBySQL(ctx, queryStr, pagination) if err != nil { return nil, errors.Wrap(err, "search by SQL failed") diff --git a/pkg/infra/search/storage/types.go b/pkg/infra/search/storage/types.go index bde9e777..4b444d19 100644 --- a/pkg/infra/search/storage/types.go +++ b/pkg/infra/search/storage/types.go @@ -31,6 +31,7 @@ import ( const ( Equals = "=" + NLPatternType = "nl" DSLPatternType = "dsl" SQLPatternType = "sql" ) diff --git a/pkg/kubernetes/registry/types.go b/pkg/kubernetes/registry/types.go index af22e20f..45095f1f 100644 --- a/pkg/kubernetes/registry/types.go +++ b/pkg/kubernetes/registry/types.go @@ -40,9 +40,18 @@ type ExtraConfig struct { ElasticSearchPassword string ReadOnlyMode bool GithubBadge bool + EnableRBAC bool - EnableRBAC bool + // ServiceAccount configs ServiceAccountIssuer serviceaccount.TokenGenerator ServiceAccountMaxExpiration time.Duration ExtendExpiration bool + + // AI configs + AIBackend string + AIAuthToken string + AIBaseURL string + AIModel string + AITemperature float32 + AITopP float32 } diff --git a/ui/src/components/sqlSearch/index.tsx b/ui/src/components/sqlSearch/index.tsx index 66d9b166..b7df71a8 100644 --- a/ui/src/components/sqlSearch/index.tsx +++ b/ui/src/components/sqlSearch/index.tsx @@ -35,6 +35,11 @@ import { searchSqlPrefix, } from '@/utils/constants' import { useAxios } from '@/utils/request' +import { + cacheHistory, + deleteHistoryByItem, + getHistoryList, +} from '@/utils/tools' import styles from './styles.module.less' @@ -90,436 +95,426 @@ const focusHandlerExtension = EditorView.domEventHandlers({ type SqlSearchIProps = { sqlEditorValue: string - handleSearch: (val: string) => void -} - -function getHistoryList() { - return localStorage?.getItem('sqlEditorHistory') - ? JSON.parse(localStorage?.getItem('sqlEditorHistory')) - : [] + handleSqlSearch: (val: string) => void } -function deleteHistoryByItem(val: string) { - const lastHistory: any = localStorage.getItem('sqlEditorHistory') - const tmp = lastHistory ? JSON.parse(lastHistory) : [] - if (tmp?.length > 0 && tmp?.includes(val)) { - const newList = tmp?.filter(item => item !== val) - localStorage.setItem('sqlEditorHistory', JSON.stringify(newList)) - } -} +const SqlSearch = memo( + ({ sqlEditorValue, handleSqlSearch }: SqlSearchIProps) => { + const editorRef = useRef(null) + const { t, i18n } = useTranslation() + const clusterListRef = useRef(null) + const [clusterList, setClusterList] = useState([]) + const [historyCompletions, setHistoryCompletions] = useState< + { value: string }[] + >([]) + const historyCompletionsRef = useRef( + getHistoryList('sqlEditorHistory'), + ) + + function cacheSqlHistory(val: string) { + const result = cacheHistory('sqlEditorHistory', val) + historyCompletionsRef.current = result + setHistoryCompletions(historyCompletionsRef.current) + } -const SqlSearch = memo(({ sqlEditorValue, handleSearch }: SqlSearchIProps) => { - const editorRef = useRef(null) - const { t, i18n } = useTranslation() - const clusterListRef = useRef(null) - const [clusterList, setClusterList] = useState([]) - const [historyCompletions, setHistoryCompletions] = useState< - { value: string }[] - >([]) - const historyCompletionsRef = useRef(getHistoryList()) - - function cacheHistory(val: string) { - const lastHistory: any = localStorage.getItem('sqlEditorHistory') - const tmp = lastHistory ? JSON.parse(lastHistory) : [] - const newList = [val, ...tmp?.filter(item => item !== val)] - localStorage.setItem('sqlEditorHistory', JSON.stringify(newList)) - historyCompletionsRef.current = getHistoryList() - setHistoryCompletions(historyCompletionsRef.current) - } - - const { response } = useAxios({ - url: '/rest-api/v1/clusters', - option: { - params: { - orderBy: 'name', - ascending: true, + const { response } = useAxios({ + url: '/rest-api/v1/clusters', + option: { + params: { + orderBy: 'name', + ascending: true, + }, }, - }, - manual: false, - method: 'GET', - }) - - useEffect(() => { - if (response?.success) { - clusterListRef.current = response?.data?.items - setClusterList(response?.data?.items) - } - }, [response]) - - useEffect(() => { - getHistoryList() - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) - - function getCustomCompletions(regMatch, cusCompletions, pos) { - const filterTerm = regMatch[2] - const customCompletions = cusCompletions - .filter(completion => - completion.toLowerCase().includes(filterTerm.toLowerCase()), - ) - .map(completion => ({ - label: completion, - type: 'custom', - apply: completion, - boost: 0, - })) - - const from = pos - filterTerm?.length - if (customCompletions?.length > 0) { - return { from, options: customCompletions } - } - return null - } - - useEffect(() => { - if (editorRef.current) { - const contentEditableElement = editorRef.current.querySelector( - '.cm-content', - ) as any - if (contentEditableElement) { - contentEditableElement.style.outline = 'none' + manual: false, + method: 'GET', + }) + + useEffect(() => { + if (response?.success) { + clusterListRef.current = response?.data?.items + setClusterList(response?.data?.items) } - const customCompletionKeymap: KeyBinding[] = [ - { key: 'Tab', run: acceptCompletion }, - ] - const overrideKeymap = keymap.of( - customCompletionKeymap.concat( - completionKeymap.filter(b => b.key !== 'Enter'), - ), - ) - const mySQLHighlightStyle = HighlightStyle.define([ - { tag: tags.keyword, color: 'blue' }, - ]) - - const customCompletion = context => { - const { state, pos } = context - const beforeCursor = state.doc.sliceString(0, pos) - if (state.doc?.length === 0) { - const historyOptions: any[] = historyCompletionsRef?.current?.map( - record => ({ - label: record, - type: 'history', - apply: record, - }), - ) - return { - from: context.pos, - options: historyOptions, - filter: false, - } - } - - const whereMatch = /\b(where|or|and) (\S*)$/.exec(beforeCursor) - if (whereMatch) { - return getCustomCompletions(whereMatch, whereKeywords, pos) - } - - const kindMatch = /(kind\s*=\s*)(\S*)$/i.exec(beforeCursor) - if (kindMatch) { - return getCustomCompletions(kindMatch, kindCompletions, pos) - } + }, [response]) + + useEffect(() => { + getHistoryList('sqlEditorHistory') + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + function getCustomCompletions(regMatch, cusCompletions, pos) { + const filterTerm = regMatch[2] + const customCompletions = cusCompletions + .filter(completion => + completion.toLowerCase().includes(filterTerm.toLowerCase()), + ) + .map(completion => ({ + label: completion, + type: 'custom', + apply: completion, + boost: 0, + })) + + const from = pos - filterTerm?.length + if (customCompletions?.length > 0) { + return { from, options: customCompletions } + } + return null + } - if (whereKeywords?.some(item => beforeCursor?.endsWith(`${item} `))) { - const customCompletions = operatorKeywords.map(completion => ({ - label: completion, - type: 'custom', - validFor: () => false, - })) - return { from: pos, options: customCompletions } + useEffect(() => { + if (editorRef.current) { + const contentEditableElement = editorRef.current.querySelector( + '.cm-content', + ) as any + if (contentEditableElement) { + contentEditableElement.style.outline = 'none' } + const customCompletionKeymap: KeyBinding[] = [ + { key: 'Tab', run: acceptCompletion }, + ] + const overrideKeymap = keymap.of( + customCompletionKeymap.concat( + completionKeymap.filter(b => b.key !== 'Enter'), + ), + ) + const mySQLHighlightStyle = HighlightStyle.define([ + { tag: tags.keyword, color: 'blue' }, + ]) + + const customCompletion = context => { + const { state, pos } = context + const beforeCursor = state.doc.sliceString(0, pos) + if (state.doc?.length === 0) { + const historyOptions: any[] = historyCompletionsRef?.current?.map( + record => ({ + label: record, + type: 'history', + apply: record, + }), + ) + return { + from: context.pos, + options: historyOptions, + filter: false, + } + } - const clusterMatch = /(cluster\s*=\s*)(\S*)$/i.exec(beforeCursor) - if (clusterMatch) { - const clusterNameList = clusterListRef.current?.map( - item => `'${item?.metadata?.name}'`, - ) - return getCustomCompletions(clusterMatch, clusterNameList, pos) - } + const whereMatch = /\b(where|or|and) (\S*)$/.exec(beforeCursor) + if (whereMatch) { + return getCustomCompletions(whereMatch, whereKeywords, pos) + } - const word = context?.matchBefore(/\w*/) + const kindMatch = /(kind\s*=\s*)(\S*)$/i.exec(beforeCursor) + if (kindMatch) { + return getCustomCompletions(kindMatch, kindCompletions, pos) + } - if (!word || (word?.from === word?.to && !context?.explicit)) { - return null - } + if (whereKeywords?.some(item => beforeCursor?.endsWith(`${item} `))) { + const customCompletions = operatorKeywords.map(completion => ({ + label: completion, + type: 'custom', + validFor: () => false, + })) + return { from: pos, options: customCompletions } + } - const options = defaultKeywords - .filter(kw => kw.toLowerCase().includes(word.text?.toLowerCase())) - .map(stmt => ({ label: stmt, type: 'custom' })) - if (options?.length === 0) { - return null - } + const clusterMatch = /(cluster\s*=\s*)(\S*)$/i.exec(beforeCursor) + if (clusterMatch) { + const clusterNameList = clusterListRef.current?.map( + item => `'${item?.metadata?.name}'`, + ) + return getCustomCompletions(clusterMatch, clusterNameList, pos) + } - return completeFromList(options)(context) - } + const word = context?.matchBefore(/\w*/) - const completionPlugin = ViewPlugin.fromClass( - class { - constructor(view) { - this.addDeleteButtons(view) + if (!word || (word?.from === word?.to && !context?.explicit)) { + return null } - update(update) { - this.addDeleteButtons(update.view) + const options = defaultKeywords + .filter(kw => kw.toLowerCase().includes(word.text?.toLowerCase())) + .map(stmt => ({ label: stmt, type: 'custom' })) + if (options?.length === 0) { + return null } - addDeleteButtons(view) { - const compState: any = completionStatus(view.state) - if (compState === 'active') { - const completions: any = currentCompletions(view.state) - setTimeout(() => { - if (completions?.[0]?.type === 'history') { - view.dom - .querySelectorAll( - '.cm-tooltip.cm-tooltip-autocomplete > ul > li', - ) - .forEach((item, index) => { - if ( - item.querySelector( - '.cm-tooltip-autocomplete_item_label', - ) - ) { - return - } - if (item.querySelector('.delete-btn')) { - return - } - const labelSpan = document.createElement('span') - labelSpan.className = 'cm-tooltip-autocomplete_item_label' - labelSpan.innerText = completions?.[index]?.label - item.style.display = 'flex' - item.style.justifyContent = 'space-between' - item.style.alignItems = 'center' - labelSpan.style.flex = '1' - labelSpan.style.overflow = 'hidden' - labelSpan.style.textOverflow = 'ellipsis' - labelSpan.style.whiteSpace = 'nowrap' - labelSpan.style.padding = '0 10px' - labelSpan.style.fontSize = '14px' - const btn = document.createElement('span') - btn.innerText = '✕' - btn.className = 'delete-btn' - btn.style.border = 'none' - btn.style.fontSize = '20px' - btn.style.display = 'flex' - btn.style.justifyContent = 'center' - btn.style.alignItems = 'center' - btn.style.height = '100%' - btn.style.padding = '0 15px' - item.innerText = '' - item.appendChild(labelSpan) - item.appendChild(btn) - btn.addEventListener('mousedown', event => { - event.preventDefault() - event.stopPropagation() - const completionOption = completions?.[index] - historyCompletionsRef.current = - historyCompletionsRef?.current?.filter( - item => item !== completionOption?.label, + return completeFromList(options)(context) + } + + const completionPlugin = ViewPlugin.fromClass( + class { + constructor(view) { + this.addDeleteButtons(view) + } + + update(update) { + this.addDeleteButtons(update.view) + } + + addDeleteButtons(view) { + const compState: any = completionStatus(view.state) + if (compState === 'active') { + const completions: any = currentCompletions(view.state) + setTimeout(() => { + if (completions?.[0]?.type === 'history') { + view.dom + .querySelectorAll( + '.cm-tooltip.cm-tooltip-autocomplete > ul > li', + ) + .forEach((item, index) => { + if ( + item.querySelector( + '.cm-tooltip-autocomplete_item_label', ) - if (view) { - startCompletion(view) - deleteHistoryByItem(completionOption?.label) + ) { + return } + if (item.querySelector('.delete-btn')) { + return + } + const labelSpan = document.createElement('span') + labelSpan.className = + 'cm-tooltip-autocomplete_item_label' + labelSpan.innerText = completions?.[index]?.label + item.style.display = 'flex' + item.style.justifyContent = 'space-between' + item.style.alignItems = 'center' + labelSpan.style.flex = '1' + labelSpan.style.overflow = 'hidden' + labelSpan.style.textOverflow = 'ellipsis' + labelSpan.style.whiteSpace = 'nowrap' + labelSpan.style.padding = '0 10px' + labelSpan.style.fontSize = '14px' + const btn = document.createElement('span') + btn.innerText = '✕' + btn.className = 'delete-btn' + btn.style.border = 'none' + btn.style.fontSize = '20px' + btn.style.display = 'flex' + btn.style.justifyContent = 'center' + btn.style.alignItems = 'center' + btn.style.height = '100%' + btn.style.padding = '0 15px' + item.innerText = '' + item.appendChild(labelSpan) + item.appendChild(btn) + btn.addEventListener('mousedown', event => { + event.preventDefault() + event.stopPropagation() + const completionOption = completions?.[index] + historyCompletionsRef.current = + historyCompletionsRef?.current?.filter( + item => item !== completionOption?.label, + ) + if (view) { + startCompletion(view) + deleteHistoryByItem( + 'sqlEditorHistory', + completionOption?.label, + ) + } + }) }) - }) - } - }, 0) + } + }, 0) + } } - } - }, - ) - - const startState = EditorState.create({ - doc: '', - extensions: [ - completionPlugin, - placeholder(`${t('SearchUsingSQL')} ......`), - placeholderStyle, - new LanguageSupport(sql() as any), - highlightSpecialChars(), - syntaxHighlighting(mySQLHighlightStyle), - autocompletion({ - override: [customCompletion], - }), - focusHandlerExtension, - autocompletion(), - overrideKeymap, - EditorView.domEventHandlers({ - keydown: (event, view) => { - if (event.key === 'Enter') { - const completions = currentCompletions(view.state) - if (!completions || completions?.length === 0) { - event.preventDefault() - handleClick() - return true + }, + ) + + const startState = EditorState.create({ + doc: '', + extensions: [ + completionPlugin, + placeholder(`${t('SearchUsingSQL')} ......`), + placeholderStyle, + new LanguageSupport(sql() as any), + highlightSpecialChars(), + syntaxHighlighting(mySQLHighlightStyle), + autocompletion({ + override: [customCompletion], + }), + focusHandlerExtension, + autocompletion(), + overrideKeymap, + EditorView.domEventHandlers({ + keydown: (event, view) => { + if (event.key === 'Enter') { + const completions = currentCompletions(view.state) + if (!completions || completions?.length === 0) { + event.preventDefault() + handleClick() + return true + } + } + return false + }, + }), + EditorState.allowMultipleSelections.of(false), + EditorView.updateListener.of(update => { + if (update.docChanged) { + if (update.state.doc.lines > 1) { + update.view.dispatch({ + changes: { + from: update.startState.doc?.length, + to: update.state.doc?.length, + }, + }) } } - return false - }, - }), - EditorState.allowMultipleSelections.of(false), - EditorView.updateListener.of(update => { - if (update.docChanged) { - if (update.state.doc.lines > 1) { - update.view.dispatch({ - changes: { - from: update.startState.doc?.length, - to: update.state.doc?.length, - }, - }) - } - } - }), - ], - }) + }), + ], + }) - const view = new EditorView({ - state: startState, - parent: editorRef.current, - }) + const view = new EditorView({ + state: startState, + parent: editorRef.current, + }) - editorRef.current.view = view + editorRef.current.view = view - return () => { - if (editorRef.current?.view) { - // eslint-disable-next-line react-hooks/exhaustive-deps - editorRef.current?.view?.destroy() + return () => { + if (editorRef.current?.view) { + // eslint-disable-next-line react-hooks/exhaustive-deps + editorRef.current?.view?.destroy() + } } } - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [editorRef.current, historyCompletions, i18n?.language]) - - useEffect(() => { - if (sqlEditorValue && clusterList && editorRef.current?.view) { - editorRef.current?.view.dispatch({ - changes: { - from: 0, - to: editorRef.current?.view.state.doc?.length, - insert: sqlEditorValue, - }, - }) - } - }, [clusterList, editorRef.current?.view, sqlEditorValue]) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [editorRef.current, historyCompletions, i18n?.language]) + + useEffect(() => { + if (sqlEditorValue && clusterList && editorRef.current?.view) { + editorRef.current?.view.dispatch({ + changes: { + from: 0, + to: editorRef.current?.view.state.doc?.length, + insert: sqlEditorValue, + }, + }) + } + }, [clusterList, editorRef.current?.view, sqlEditorValue]) - const getContent = () => { - if (editorRef.current?.view) { - const content = editorRef.current.view.state.doc.toString() - return content + const getContent = () => { + if (editorRef.current?.view) { + const content = editorRef.current.view.state.doc.toString() + return content + } + return '' } - return '' - } - - function handleClick() { - const inputValue = getContent() - if (!inputValue) { - message.warning(t('PleaseEnterValidSQLStatement')) - return + + function handleClick() { + const inputValue = getContent() + if (!inputValue) { + message.warning(t('PleaseEnterValidSQLStatement')) + return + } + cacheSqlHistory(inputValue) + handleSqlSearch(inputValue) } - cacheHistory(inputValue) - handleSearch(inputValue) - } - - return ( -
-
-
{searchSqlPrefix}
-
-
ul { - box-sizing: border-box; - height: auto; - max-height: 40vh; - overflow-y: auto !important; - } - .cm-tooltip.cm-tooltip-autocomplete > ul > li { - background-color: #f5f5f5 !important; - margin: 5px 0 !important; - padding: 10px 0 !important; - border-radius: 6px !important; - width: auto !important; - box-sizing: border-box; - } - .cm-tooltip.cm-tooltip-autocomplete - > ul - > li[aria-selected='true'], - .cm-tooltip.cm-tooltip-autocomplete > ul > li:hover { - background-color: #97a9f5 !important; - color: white !important; - } - `} - /> -
- -
-
-
- + return ( +
+
+
{searchSqlPrefix}
+
+
ul { + box-sizing: border-box; + height: auto; + max-height: 40vh; + overflow-y: auto !important; + } + .cm-tooltip.cm-tooltip-autocomplete > ul > li { + background-color: #f5f5f5 !important; + margin: 5px 0 !important; + padding: 10px 0 !important; + border-radius: 6px !important; + width: auto !important; + box-sizing: border-box; + } + + .cm-tooltip.cm-tooltip-autocomplete + > ul + > li[aria-selected='true'], + .cm-tooltip.cm-tooltip-autocomplete > ul > li:hover { + background-color: #97a9f5 !important; + color: white !important; + } + `} + /> +
+ +
+
+
+ +
-
- ) -}) + ) + }, +) export default SqlSearch diff --git a/ui/src/locales/de.json b/ui/src/locales/de.json index df6d8c68..c6dee3a8 100644 --- a/ui/src/locales/de.json +++ b/ui/src/locales/de.json @@ -119,5 +119,8 @@ "LoginSuccess": "Erfolgreich eingeloggt", "Login": "Einloggen", "LogoutSuccess": "Erfolgreich abgemeldet", - "InputToken": "Geben Sie bitte den Token ein" + "InputToken": "Geben Sie bitte den Token ein", + "SearchByNaturalLanguage": "Suche mit natürlicher Sprache", + "CannotBeEmpty": "Darf nicht leer sein", + "DefaultTag": "Standard-Tag" } diff --git a/ui/src/locales/en.json b/ui/src/locales/en.json index 23bb3248..40b02524 100644 --- a/ui/src/locales/en.json +++ b/ui/src/locales/en.json @@ -119,5 +119,8 @@ "LoginSuccess": "Login Successful", "Login": "Login", "LogoutSuccess": "Successfully Logged Out", - "InputToken": "Please Enter the Token" + "InputToken": "Please Enter the Token", + "SearchByNaturalLanguage": "Search By Natural Language", + "CannotBeEmpty": "Cannot be empty", + "DefaultTag": "default tag" } diff --git a/ui/src/locales/pt.json b/ui/src/locales/pt.json index 112f8f10..67965cc0 100644 --- a/ui/src/locales/pt.json +++ b/ui/src/locales/pt.json @@ -119,5 +119,8 @@ "LoginSuccess": "Login bem-sucedido", "Login": "Login", "LogoutSuccess": "Sessão encerrada com sucesso", - "InputToken": "Por favor, insira o token" + "InputToken": "Por favor, insira o token", + "SearchByNaturalLanguage": "Procure por linguagem natural", + "CannotBeEmpty": "Não pode estar vazio", + "DefaultTag": "Tag padrão" } diff --git a/ui/src/locales/zh.json b/ui/src/locales/zh.json index 662434f7..bd61e526 100644 --- a/ui/src/locales/zh.json +++ b/ui/src/locales/zh.json @@ -119,5 +119,8 @@ "LoginSuccess": "登录成功", "Login": "登录", "LogoutSuccess": "登出成功", - "InputToken": "请输入 token" + "InputToken": "请输入 token", + "SearchByNaturalLanguage": "自然语言搜索", + "CannotBeEmpty": "不能为空", + "DefaultTag": "默认标签" } diff --git a/ui/src/pages/insightDetail/components/tagVariableSizeList/index.tsx b/ui/src/pages/insightDetail/components/tagVariableSizeList/index.tsx index cbf3b5b1..02014977 100644 --- a/ui/src/pages/insightDetail/components/tagVariableSizeList/index.tsx +++ b/ui/src/pages/insightDetail/components/tagVariableSizeList/index.tsx @@ -68,6 +68,7 @@ const TagVariableSizeList = ({ allTags, containerWidth }: IProps) => { return rows } + // eslint-disable-next-line react-hooks/exhaustive-deps const transformedData = useMemo(() => convertDataToRows(allTags), [allTags]) const itemSize = 30 // lineHeight const itemCount = transformedData?.length // row count diff --git a/ui/src/pages/result/index.tsx b/ui/src/pages/result/index.tsx index bb8e5484..468482dd 100644 --- a/ui/src/pages/result/index.tsx +++ b/ui/src/pages/result/index.tsx @@ -1,36 +1,113 @@ import React, { useState, useEffect } from 'react' -import { Pagination, Empty, Divider, Tooltip, Tag } from 'antd' +import { + Pagination, + Empty, + Divider, + Tooltip, + Input, + message, + AutoComplete, + Space, + Tag, +} from 'antd' import { useLocation, useNavigate } from 'react-router-dom' import { useTranslation } from 'react-i18next' -import { ClockCircleOutlined } from '@ant-design/icons' +import { ClockCircleOutlined, CloseOutlined } from '@ant-design/icons' import queryString from 'query-string' import classNames from 'classnames' import SqlSearch from '@/components/sqlSearch' import KarporTabs from '@/components/tabs/index' -import { utcDateToLocalDate } from '@/utils/tools' +import { + cacheHistory, + deleteHistoryByItem, + getHistoryList, + utcDateToLocalDate, +} from '@/utils/tools' import Loading from '@/components/loading' import { ICON_MAP } from '@/utils/images' import { searchSqlPrefix, tabsList } from '@/utils/constants' import { useAxios } from '@/utils/request' +// import useDebounce from '@/hooks/useDebounce' import styles from './styles.module.less' +const { Search } = Input +const { t } = useTranslation() +const Option = AutoComplete.Option + +export const CustomDropdown = props => { + const { options } = props + + return ( +
+ {options.map((option, index) => ( +
+ +
+ ))} +
+ ) +} + const Result = () => { const { t } = useTranslation() const location = useLocation() const navigate = useNavigate() const [pageData, setPageData] = useState() - const urlSearchParams = queryString.parse(location.search) - const [searchType, setSearchType] = useState('sql') + const urlSearchParams: any = queryString.parse(location.search) + const [searchType, setSearchType] = useState(urlSearchParams?.pattern) const [searchParams, setSearchParams] = useState({ pageSize: 20, page: 1, query: urlSearchParams?.query || '', total: 0, }) + const [naturalValue, setNaturalValue] = useState('') + const [sqlValue, setSqlValue] = useState('') + const [naturalOptions, setNaturalOptions] = useState( + getHistoryList('naturalHistory') || [], + ) + + function cacheNaturalHistory(key, val) { + const result = cacheHistory(key, val) + setNaturalOptions(result) + } + + useEffect(() => { + if (searchType === 'natural') { + setNaturalValue(urlSearchParams?.query) + handleNaturalSearch(urlSearchParams?.query) + } + if (searchType === 'sql') { + setSqlValue(urlSearchParams?.query) + handleSqlSearch(urlSearchParams?.query) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + if (urlSearchParams?.pattern) { + setSearchType(urlSearchParams?.pattern) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [urlSearchParams?.pattern, urlSearchParams?.query]) function handleTabChange(value: string) { setSearchType(value) + const urlString = queryString.stringify({ + pattern: value, + query: + value === 'natural' ? naturalValue : value === 'sql' ? sqlValue : '', + }) + navigate(`${location?.pathname}?${urlString}`, { replace: true }) } function handleChangePage(page: number, pageSize: number) { @@ -48,11 +125,23 @@ const Result = () => { useEffect(() => { if (response?.success) { - setPageData(response?.data?.items || {}) const objParams = { ...urlSearchParams, + pattern: 'sql', query: response?.successParams?.query || searchParams?.query, } + if (searchType === 'natural') { + let sqlVal + if (response?.data?.sqlQuery?.includes('WHERE')) { + sqlVal = `where ${response?.data?.sqlQuery?.split(' WHERE ')?.[1]}` + } + if (response?.data?.sqlQuery?.includes('where')) { + sqlVal = `where ${response?.data?.sqlQuery?.split(' where ')?.[1]}` + } + setSearchType('sql') + setSqlValue(sqlVal) + } + setPageData(response?.data?.items || {}) const urlString = queryString.stringify(objParams) navigate(`${location?.pathname}?${urlString}`, { replace: true }) } @@ -60,11 +149,19 @@ const Result = () => { }, [response]) function getPageData(params) { + const pattern = + searchType === 'natural' ? 'nl' : searchType === 'sql' ? 'sql' : '' + const query = + searchType === 'natural' + ? params?.query + : searchType === 'sql' + ? `${searchSqlPrefix} ${params?.query}` + : '' refetch({ option: { params: { - query: `${searchSqlPrefix} ${params?.query || searchParams?.query}`, - ...(searchType === 'sql' ? { pattern: 'sql' } : {}), + pattern, + query, page: params?.page || searchParams?.page, pageSize: params?.pageSize || searchParams?.pageSize, }, @@ -78,11 +175,8 @@ const Result = () => { }) } - useEffect(() => { - getPageData(searchParams) - }, []) // eslint-disable-line react-hooks/exhaustive-deps - - function handleSearch(inputValue) { + function handleSqlSearch(inputValue) { + setSqlValue(inputValue) setSearchParams({ ...searchParams, query: inputValue, @@ -130,6 +224,23 @@ const Result = () => { navigate(`/insightDetail/${nav}?${urlParams}`) } + function handleNaturalAutoCompleteChange(val) { + setNaturalValue(val) + } + + function handleNaturalSearch(value) { + if (!value && !naturalValue) { + message.warning(t('CannotBeEmpty')) + return + } + cacheNaturalHistory('naturalHistory', value) + getPageData({ + pageSize: searchParams?.pageSize, + page: 1, + query: value, + }) + } + function renderEmpty() { return (
{ ) } + const handleDelete = val => { + deleteHistoryByItem('naturalHistory', val) + const list = getHistoryList('naturalHistory') || [] + setNaturalOptions(list) + } + + const renderOption = val => { + return ( + + {val} + { + event?.stopPropagation() + handleDelete(val) + }} + /> + + ) + } + + const tmpOptions = naturalOptions?.map(val => ({ + value: val, + label: renderOption(val), + })) + return (
@@ -271,12 +413,39 @@ const Result = () => { onChange={handleTabChange} />
- + {searchType === 'sql' && ( + + )} + {searchType === 'natural' && ( +
+ { + if (option?.value) { + return ( + (option?.value as string) + ?.toUpperCase() + .indexOf(inputValue.toUpperCase()) !== -1 + ) + } + }} + > + + +
+ )}
{loading ? renderLoading() diff --git a/ui/src/pages/search/index.tsx b/ui/src/pages/search/index.tsx index ace766dd..dbe96df2 100644 --- a/ui/src/pages/search/index.tsx +++ b/ui/src/pages/search/index.tsx @@ -15,14 +15,21 @@ */ import React, { useCallback, useState } from 'react' -import { Tag } from 'antd' -import { DoubleLeftOutlined, DoubleRightOutlined } from '@ant-design/icons' +import { AutoComplete, Input, message, Space, Tag } from 'antd' +import { + DoubleLeftOutlined, + DoubleRightOutlined, + CloseOutlined, +} from '@ant-design/icons' import { useTranslation } from 'react-i18next' import { useNavigate } from 'react-router-dom' import KarporTabs from '@/components/tabs/index' import logoFull from '@/assets/img/logo-full.svg' import SqlSearch from '@/components/sqlSearch' import { defaultSqlExamples, tabsList } from '@/utils/constants' +import { deleteHistoryByItem, getHistoryList } from '@/utils/tools' + +const { Search } = Input import styles from './styles.module.less' @@ -32,6 +39,9 @@ const SearchPage = () => { const [searchType, setSearchType] = useState('sql') const [sqlEditorValue, setSqlEditorValue] = useState('') const [showAll, setShowAll] = useState(false) + const [naturalOptions, setNaturalOptions] = useState( + getHistoryList('naturalHistory') || [], + ) const toggleTags = () => { setShowAll(!showAll) @@ -45,7 +55,7 @@ const SearchPage = () => { setSqlEditorValue(str) } - const handleSearch = useCallback( + const handleSqlSearch = useCallback( inputValue => { navigate(`/search/result?query=${inputValue}&pattern=sql`) }, @@ -75,6 +85,45 @@ const SearchPage = () => { return renderSqlExamples(sqlExamples) } + function handleNaturalSearch(value) { + if (!value) { + message.warning(t('CannotBeEmpty')) + return + } + navigate(`/search/result?query=${value}&pattern=natural`) + } + + const handleDelete = val => { + deleteHistoryByItem('naturalHistory', val) + const list = getHistoryList('naturalHistory') || [] + setNaturalOptions(list) + } + + const renderOption = val => { + return ( + + {val} + { + event?.stopPropagation() + handleDelete(val) + }} + /> + + ) + } + + const tmpOptions = naturalOptions?.map(val => ({ + value: val, + label: renderOption(val), + })) + return (
@@ -88,12 +137,42 @@ const SearchPage = () => { onChange={handleTabChange} />
-
- -
+ + {searchType === 'sql' && ( +
+ +
+ )} + + {searchType === 'natural' && ( +
+ { + if (option?.value) { + return ( + (option?.value as string) + ?.toUpperCase() + .indexOf(inputValue.toUpperCase()) !== -1 + ) + } + }} + > + + +
+ )} +
{searchType === 'keyword' ? (
@@ -108,7 +187,7 @@ const SearchPage = () => {
- ) : ( + ) : searchType === 'natural' ? null : (
{renderSqlExamples(null)} {!showAll && ( diff --git a/ui/src/utils/constants.ts b/ui/src/utils/constants.ts index 11d42923..931a345b 100644 --- a/ui/src/utils/constants.ts +++ b/ui/src/utils/constants.ts @@ -81,6 +81,10 @@ export const defaultKeywords = [ export const tabsList = [ { label: 'KeywordSearch', value: 'keyword', disabled: true }, { label: 'SQLSearch', value: 'sql' }, + { + label: 'SearchByNaturalLanguage', + value: 'natural', + }, ] export const insightTabsList = [ diff --git a/ui/src/utils/tools.ts b/ui/src/utils/tools.ts index 75e91664..d7ef0494 100644 --- a/ui/src/utils/tools.ts +++ b/ui/src/utils/tools.ts @@ -199,3 +199,26 @@ export function getTextSizeByCanvas( canvas.remove() return width + 2 } + +export function getHistoryList(key) { + return localStorage?.getItem(key) + ? JSON.parse(localStorage?.getItem(key)) + : [] +} + +export function deleteHistoryByItem(key, val: string) { + const lastHistory: any = localStorage.getItem(key) + const tmp = lastHistory ? JSON.parse(lastHistory) : [] + if (tmp?.length > 0 && tmp?.includes(val)) { + const newList = tmp?.filter(item => item !== val) + localStorage.setItem(key, JSON.stringify(newList)) + } +} + +export function cacheHistory(key, val: string) { + const lastHistory: any = localStorage.getItem(key) + const tmp = lastHistory ? JSON.parse(lastHistory) : [] + const newList = [val, ...tmp?.filter(item => item !== val)] + localStorage.setItem(key, JSON.stringify(newList)) + return getHistoryList(key) +}