From ed007e3ff3047668b4769da5a0cf0e9f5ffe6bda Mon Sep 17 00:00:00 2001 From: Emily McMullan Date: Thu, 13 Jun 2024 10:36:58 -0400 Subject: [PATCH] support running multi-language apps Signed-off-by: Emily McMullan --- cmd/analyze.go | 372 ++++++++++++++++++++++++++++++------------------- 1 file changed, 232 insertions(+), 140 deletions(-) diff --git a/cmd/analyze.go b/cmd/analyze.go index b11f497..ed37df6 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -18,6 +18,7 @@ import ( "sort" "strings" + "github.com/devfile/alizer/pkg/apis/model" "github.com/devfile/alizer/pkg/apis/recognizer" "github.com/go-logr/logr" "github.com/konveyor-ecosystem/kantra/cmd/internal/hiddenfile" @@ -54,6 +55,9 @@ const ( goProvider = "go" pythonProvider = "python" nodeJSProvider = "javascript" + dotNetProvider = "dotnet" + // windows only + dotNetFrameworkProvider = "dotnetframework" ) // kantra analyze flags @@ -80,6 +84,7 @@ type analyzeCommand struct { contextLines int incidentSelector string depFolders []string + provider []string // tempDirs list of temporary dirs created, used for cleanup tempDirs []string @@ -154,16 +159,25 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { return err } foundProviders := []string{} + // file input means a binary was given which only the java provider can use if analyzeCmd.isFileInput { foundProviders = append(foundProviders, javaProvider) } else { - for _, c := range components { - log.Info("Got component", "component language", c.Languages, "path", c.Path) - for _, l := range c.Languages { - foundProviders = append(foundProviders, strings.ToLower(l.Name)) - } + foundProviders, err = analyzeCmd.setProviders(components, foundProviders) + if err != nil { + log.Error(err, "failed to set provider info") + return err + } + err = analyzeCmd.validateProviders(foundProviders) + if err != nil { + return err } } + providersMap, err := setProviderInitInfo(foundProviders) + if err != nil { + log.Error(err, "failed to set provider init info") + return err + } containerNetworkName, err := analyzeCmd.createContainerNetwork() if err != nil { log.Error(err, "failed to create container network") @@ -176,12 +190,12 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { return err } // allow for 5 retries of running provider in the case of port in use - providerPorts, err := analyzeCmd.RunProviders(cmd.Context(), containerNetworkName, containerVolName, foundProviders, 5) + _, err = analyzeCmd.RunProviders(cmd.Context(), containerNetworkName, containerVolName, providersMap, 5) if err != nil { log.Error(err, "failed to run provider") return err } - err = analyzeCmd.RunAnalysis(cmd.Context(), xmlOutputDir, containerVolName, foundProviders, providerPorts) + err = analyzeCmd.RunAnalysis(cmd.Context(), xmlOutputDir, containerVolName, providersMap) if err != nil { log.Error(err, "failed to run analysis") return err @@ -191,7 +205,7 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { log.Error(err, "failed to create json output file") return err } - err = analyzeCmd.GenerateStaticReport(cmd.Context()) + err = analyzeCmd.GenerateStaticReport(cmd.Context(), providersMap) if err != nil { log.Error(err, "failed to generate static report") return err @@ -222,6 +236,7 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { analyzeCommand.Flags().IntVar(&analyzeCmd.contextLines, "context-lines", 100, "number of lines of source code to include in the output for each incident") analyzeCommand.Flags().StringVar(&analyzeCmd.incidentSelector, "incident-selector", "", "an expression to select incidents based on custom variables. ex: (!package=io.konveyor.demo.config-utils)") analyzeCommand.Flags().StringArrayVarP(&analyzeCmd.depFolders, "dependency-folders", "d", []string{}, "directory for dependencies") + analyzeCommand.Flags().StringArrayVar(&analyzeCmd.provider, "provider", []string{}, "set provider to use") return analyzeCommand } @@ -363,7 +378,107 @@ func (a *analyzeCommand) CheckOverwriteOutput() error { return nil } -func (a *analyzeCommand) ListLabels(ctx context.Context) error { +// TODO add network and volume w/ interface +type ProviderInit struct { + port int + image string +} + +func (a *analyzeCommand) setProviders(components []model.Component, foundProviders []string) ([]string, error) { + if len(a.provider) > 0 { + for _, p := range a.provider { + foundProviders = append(foundProviders, p) + return foundProviders, nil + } + } + for _, c := range components { + a.log.V(5).Info("Got component", "component language", c.Languages, "path", c.Path) + for _, l := range c.Languages { + if l.Name == "C#" { + foundDotNetProv, err := setDotNetProvider(l.Frameworks) + if err != nil { + return nil, err + } + foundProviders = append(foundProviders, foundDotNetProv) + } else { + foundProviders = append(foundProviders, strings.ToLower(l.Name)) + } + } + } + return foundProviders, nil +} + +func setDotNetProvider(frameworks []string) (string, error) { + if len(frameworks) > 0 { + for _, f := range frameworks { + if f == "v4.5" || f == "v4.6" || f == "v4.7" || f == "v4.8" { + return dotNetFrameworkProvider, nil + } else { + return dotNetProvider, nil + } + } + } + return "", fmt.Errorf("unable to set dotnet provider") +} + +func setProviderInitInfo(foundProviders []string) (map[string]ProviderInit, error) { + providers := map[string]ProviderInit{} + for _, prov := range foundProviders { + port, err := freeport.GetFreePort() + if err != nil { + return nil, err + } + switch prov { + case javaProvider: + providers[javaProvider] = ProviderInit{ + port: port, + image: Settings.JavaProviderImage, + } + case goProvider: + providers[goProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + case pythonProvider: + providers[pythonProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + case nodeJSProvider: + providers[nodeJSProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + // TODO + // case dotNetProvider, dotNetFrameworkProvider: + // providers[nodeJSProvider] = ProviderInit{ + // port: port, + // image: "", + // } + } + } + return providers, nil +} + +func (a *analyzeCommand) validateProviders(providers []string) error { + validProvs := []string{ + javaProvider, + pythonProvider, + goProvider, + nodeJSProvider, + dotNetProvider, + dotNetFrameworkProvider, + } + for _, prov := range providers { + //validate other providers + if !slices.Contains(validProvs, prov) { + return fmt.Errorf("provider %v not supported. Use --providerOverride option", prov) + } + } + return nil +} + +func (a *analyzeCommand) ListLabels(ctx context.Context) error { return a.fetchLabels(ctx, a.listSources, a.listTargets, os.Stdout) } @@ -519,7 +634,7 @@ func (a *analyzeCommand) getDepsFolders() (map[string]string, []string) { return vols, dependencyFolders } -func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]int) (map[string]string, error) { +func (a *analyzeCommand) getConfigVolumes(providers map[string]ProviderInit) (map[string]string, error) { tempDir, err := os.MkdirTemp("", "analyze-config-") if err != nil { a.log.V(1).Error(err, "failed creating temp dir", "dir", tempDir) @@ -528,28 +643,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i a.log.V(1).Info("created directory for provider settings", "dir", tempDir) a.tempDirs = append(a.tempDirs, tempDir) - var foundJava bool - var foundGolang bool - var foundPython bool - var foundNode bool - switch providers[0] { - case javaProvider: - foundJava = true - case goProvider: - foundGolang = true - case pythonProvider: - foundPython = true - case nodeJSProvider: - foundNode = true - default: - return nil, fmt.Errorf("unable to find config for provider %v", providers[0]) - } - - // TODO (pgaikwad): binaries don't work with alizer right now, we need to revisit this - if !foundJava && a.isFileInput { - foundJava = true - } - otherProvsMountPath := SourceMountPath // when input is a file, it means it's probably a binary // only java provider can work with binaries, all others @@ -560,7 +653,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i javaConfig := provider.Config{ Name: javaProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[javaProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[javaProvider].port), InitConfig: []provider.InitConfig{ { Location: SourceMountPath, @@ -588,7 +681,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i goConfig := provider.Config{ Name: goProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[goProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[goProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.FullAnalysisMode, @@ -604,7 +697,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i pythonConfig := provider.Config{ Name: pythonProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[pythonProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[pythonProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.SourceOnlyAnalysisMode, @@ -619,7 +712,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i nodeJSConfig := provider.Config{ Name: nodeJSProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[nodeJSProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[nodeJSProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.SourceOnlyAnalysisMode, @@ -632,16 +725,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i }, } - vols, dependencyFolders := a.getDepsFolders() - if len(dependencyFolders) != 0 { - if providers[0] == pythonProvider { - pythonConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders - } - if providers[0] == nodeJSProvider { - nodeJSConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders - } - } - provConfig := []provider.Config{ { Name: "builtin", @@ -653,16 +736,24 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i }, }, } - - switch { - case foundJava: - provConfig = append(provConfig, javaConfig) - case foundGolang && a.mode == string(provider.FullAnalysisMode): - provConfig = append(provConfig, goConfig) - case foundPython: - provConfig = append(provConfig, pythonConfig) - case foundNode: - provConfig = append(provConfig, nodeJSConfig) + vols, dependencyFolders := a.getDepsFolders() + for prov, _ := range providers { + switch prov { + case javaProvider: + provConfig = append(provConfig, javaConfig) + case goProvider: + provConfig = append(provConfig, goConfig) + case pythonProvider: + if len(dependencyFolders) != 0 { + pythonConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders + } + provConfig = append(provConfig, pythonConfig) + case nodeJSProvider: + if len(dependencyFolders) != 0 { + nodeJSConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders + } + provConfig = append(provConfig, nodeJSConfig) + } } // Set proxy to providers @@ -688,7 +779,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i "failed to write provider config", "dir", tempDir, "file", "settings.json") return nil, err } - settingsVols := map[string]string{ tempDir: ConfigMountPath, } @@ -906,7 +996,7 @@ func (a *analyzeCommand) createContainerVolume() (string, error) { return volName, nil } -func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName string, volName string, providers []string, retry int) error { +func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName string, volName string, providers map[string]ProviderInit, retry int) error { if retry == 0 { return fmt.Errorf("too many provider container retry attempts") } @@ -919,65 +1009,71 @@ func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName return nil } -func (a *analyzeCommand) RunProviders(ctx context.Context, networkName string, volName string, providers []string, retry int) (map[string]int, error) { - providerPorts := map[string]int{} - port, err := freeport.GetFreePort() - if err != nil { - return nil, err - } - volumes := map[string]string{ - // application source code - volName: SourceMountPath, - } - vols, _ := a.getDepsFolders() - if len(vols) != 0 { - maps.Copy(volumes, vols) - } - - var providerImage string - switch providers[0] { - case javaProvider: - providerImage = Settings.JavaProviderImage - providerPorts[javaProvider] = port - case goProvider: - providerImage = Settings.GenericProviderImage - providerPorts[goProvider] = port - case pythonProvider: - providerImage = Settings.GenericProviderImage - providerPorts[pythonProvider] = port - case nodeJSProvider: - providerImage = Settings.GenericProviderImage - providerPorts[nodeJSProvider] = port - default: - return nil, fmt.Errorf("unable to run unsupported provider %v", providers[0]) - } - args := []string{fmt.Sprintf("--port=%v", port)} - a.log.Info("starting provider", "provider", providers[0]) - - con := container.NewContainer() - err = con.Run( - ctx, - container.WithImage(providerImage), - container.WithLog(a.log.V(1)), - container.WithVolumes(volumes), - container.WithContainerToolBin(Settings.PodmanBinary), - container.WithEntrypointArgs(args...), - container.WithDetachedMode(true), - container.WithCleanup(a.cleanup), - container.WithNetwork(networkName), - ) - if err != nil { - err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) - if err != nil { - return nil, err +func (a *analyzeCommand) RunProviders(ctx context.Context, networkName string, volName string, providers map[string]ProviderInit, retry int) (map[string]ProviderInit, error) { + runFirstProv := false + for prov, inits := range providers { + volumes := map[string]string{ + // application source code + volName: SourceMountPath, + } + vols, _ := a.getDepsFolders() + if len(vols) != 0 { + maps.Copy(volumes, vols) + } + args := []string{fmt.Sprintf("--port=%v", inits.port)} + // we have to start the fist provider separately to create the shared + // container network to then add other providers to the network + if !runFirstProv { + a.log.Info("starting first provider", "provider", prov) + con := container.NewContainer() + err := con.Run( + ctx, + container.WithImage(inits.image), + container.WithLog(a.log.V(1)), + container.WithVolumes(volumes), + container.WithContainerToolBin(Settings.PodmanBinary), + container.WithEntrypointArgs(args...), + container.WithDetachedMode(true), + container.WithCleanup(a.cleanup), + container.WithNetwork(networkName), + ) + if err != nil { + err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) + if err != nil { + return nil, err + } + } + a.providerContainerNames = append(a.providerContainerNames, con.Name) } + // start additional providers + if runFirstProv && len(providers) > 1 { + a.log.Info("starting provider", "provider", prov) + con := container.NewContainer() + err := con.Run( + ctx, + container.WithImage(inits.image), + container.WithLog(a.log.V(1)), + container.WithVolumes(volumes), + container.WithContainerToolBin(Settings.PodmanBinary), + container.WithEntrypointArgs(args...), + container.WithDetachedMode(true), + container.WithCleanup(a.cleanup), + container.WithNetwork(fmt.Sprintf("container:%v", a.providerContainerNames[0])), + ) + if err != nil { + err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) + if err != nil { + return nil, err + } + } + a.providerContainerNames = append(a.providerContainerNames, con.Name) + } + runFirstProv = true } - - a.providerContainerNames = append(a.providerContainerNames, con.Name) - return providerPorts, nil + return providers, nil } -func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, volName string, providers []string, ports map[string]int) error { +func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, volName string, providersMap map[string]ProviderInit) error { volumes := map[string]string{ // application source code volName: SourceMountPath, @@ -995,8 +1091,7 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v // for cleanup purposes a.tempDirs = append(a.tempDirs, xmlOutputDir) } - - configVols, err := a.getConfigVolumes(providers, ports) + configVols, err := a.getConfigVolumes(providersMap) if err != nil { a.log.V(1).Error(err, "failed to get config volumes for analysis") return err @@ -1011,43 +1106,33 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v } maps.Copy(volumes, ruleVols) } - args := []string{ fmt.Sprintf("--provider-settings=%s", ProviderSettingsMountPath), fmt.Sprintf("--output-file=%s", AnalysisOutputMountPath), fmt.Sprintf("--context-lines=%d", a.contextLines), } - // TODO update for running multiple apps - if providers[0] != javaProvider { + // default rulesets are only java rules + // may want to change this in the future + if _, ok := providersMap[javaProvider]; !ok { a.enableDefaultRulesets = false } - if a.enableDefaultRulesets { args = append(args, fmt.Sprintf("--rules=%s/", RulesetPath)) } - if a.incidentSelector != "" { args = append(args, fmt.Sprintf("--incident-selector=%s", a.incidentSelector)) } - if len(a.rules) > 0 { args = append(args, fmt.Sprintf("--rules=%s/", CustomRulePath)) } - if a.jaegerEndpoint != "" { args = append(args, "--enable-jaeger") args = append(args, "--jaeger-endpoint") args = append(args, a.jaegerEndpoint) } - - // python and node providers do not yet support dep analysis - if !a.analyzeKnownLibraries && (providers[0] != pythonProvider && providers[0] != nodeJSProvider) { - args = append(args, - fmt.Sprintf("--dep-label-selector=(!%s=open-source)", provider.DepSourceLabel)) - } if a.logLevel != nil { args = append(args, fmt.Sprintf("--verbose=%d", *a.logLevel)) } @@ -1056,14 +1141,17 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v args = append(args, fmt.Sprintf("--label-selector=%s", labelSelector)) } - switch true { - case a.mode == string(provider.FullAnalysisMode) && providers[0] == pythonProvider: - a.mode = string(provider.SourceOnlyAnalysisMode) - case a.mode == string(provider.FullAnalysisMode) && providers[0] == nodeJSProvider: - a.mode = string(provider.SourceOnlyAnalysisMode) - default: - a.log.Info("running dependency retrieval during analysis") - args = append(args, fmt.Sprintf("--dep-output-file=%s", DepsOutputMountPath)) + // as of now only java has dep capability + if _, ok := providersMap[javaProvider]; ok { + // need a smarter way of handling this with a java provider + other provider + if len(providersMap) == 1 { + if !a.analyzeKnownLibraries { + args = append(args, + fmt.Sprintf("--dep-label-selector=(!%s=open-source)", provider.DepSourceLabel)) + } + a.log.Info("running dependency retrieval during analysis") + args = append(args, fmt.Sprintf("--dep-output-file=%s", DepsOutputMountPath)) + } } analysisLogFilePath := filepath.Join(a.output, "analysis.log") @@ -1158,7 +1246,7 @@ func (a *analyzeCommand) CreateJSONOutput() error { return nil } -func (a *analyzeCommand) GenerateStaticReport(ctx context.Context) error { +func (a *analyzeCommand) GenerateStaticReport(ctx context.Context, provMap map[string]ProviderInit) error { if a.skipStaticReport { return nil } @@ -1172,9 +1260,13 @@ func (a *analyzeCommand) GenerateStaticReport(ctx context.Context) error { fmt.Sprintf("--output-path=%s", path.Join("/usr/local/static-report/output.js")), fmt.Sprintf("--application-name-list=%s", filepath.Base(a.input)), } - if a.mode == string(provider.FullAnalysisMode) { - staticReportArgs = append(staticReportArgs, - fmt.Sprintf("--deps-output-list=%s", DepsOutputMountPath)) + // as of now, only java provider has dep capability + // TODO smarter way to handle this + if _, ok := provMap[javaProvider]; ok { + if len(provMap) == 1 && a.mode == string(provider.FullAnalysisMode) { + staticReportArgs = append(staticReportArgs, + fmt.Sprintf("--deps-output-list=%s", DepsOutputMountPath)) + } } cpArgs := []string{"&& cp -r", "/usr/local/static-report", OutputPath}