From 165c4400b92dad453d1bf63e5ef5df11cbac7561 Mon Sep 17 00:00:00 2001 From: Emily McMullan Date: Thu, 13 Jun 2024 10:36:58 -0400 Subject: [PATCH] support running multi-language apps Signed-off-by: Emily McMullan --- cmd/analyze.go | 386 ++++++++++++++++++++++++++++++------------------- 1 file changed, 240 insertions(+), 146 deletions(-) diff --git a/cmd/analyze.go b/cmd/analyze.go index ab03a7f..ba46f61 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -18,6 +18,7 @@ import ( "sort" "strings" + "github.com/devfile/alizer/pkg/apis/model" "github.com/devfile/alizer/pkg/apis/recognizer" "github.com/go-logr/logr" "github.com/konveyor-ecosystem/kantra/cmd/internal/hiddenfile" @@ -54,6 +55,9 @@ const ( goProvider = "go" pythonProvider = "python" nodeJSProvider = "javascript" + dotNetProvider = "dotnet" + // windows only + dotNetFrameworkProvider = "dotnetframework" ) // kantra analyze flags @@ -81,6 +85,7 @@ type analyzeCommand struct { incidentSelector string depFolders []string overrideProviderSettings string + provider []string // tempDirs list of temporary dirs created, used for cleanup tempDirs []string @@ -129,14 +134,8 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { if val, err := cmd.Flags().GetBool(noCleanupFlag); err == nil { analyzeCmd.cleanup = !val } + var providersMap map[string]ProviderInit if analyzeCmd.overrideProviderSettings == "" { - // defer cleaning created resources here instead of PostRun - // if Run returns an error, PostRun does not run - defer func() { - if err := analyzeCmd.CleanAnalysisResources(cmd.Context()); err != nil { - log.Error(err, "failed to clean temporary directories") - } - }() if analyzeCmd.listSources || analyzeCmd.listTargets { err := analyzeCmd.ListLabels(cmd.Context()) if err != nil { @@ -156,16 +155,25 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { return err } foundProviders := []string{} + // file input means a binary was given which only the java provider can use if analyzeCmd.isFileInput { foundProviders = append(foundProviders, javaProvider) } else { - for _, c := range components { - log.Info("Got component", "component language", c.Languages, "path", c.Path) - for _, l := range c.Languages { - foundProviders = append(foundProviders, strings.ToLower(l.Name)) - } + foundProviders, err = analyzeCmd.setProviders(components, foundProviders) + if err != nil { + log.Error(err, "failed to set provider info") + return err + } + err = analyzeCmd.validateProviders(foundProviders) + if err != nil { + return err } } + providersMap, err = setProviderInitInfo(foundProviders) + if err != nil { + log.Error(err, "failed to set provider init info") + return err + } containerNetworkName, err := analyzeCmd.createContainerNetwork() if err != nil { log.Error(err, "failed to create container network") @@ -178,16 +186,23 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { return err } // allow for 5 retries of running provider in the case of port in use - providerPorts, err := analyzeCmd.RunProviders(cmd.Context(), containerNetworkName, containerVolName, foundProviders, 5) + _, err = analyzeCmd.RunProviders(cmd.Context(), containerNetworkName, containerVolName, providersMap, 5) if err != nil { log.Error(err, "failed to run provider") return err } - err = analyzeCmd.RunAnalysis(cmd.Context(), xmlOutputDir, containerVolName, foundProviders, providerPorts) + err = analyzeCmd.RunAnalysis(cmd.Context(), xmlOutputDir, containerVolName, providersMap) if err != nil { log.Error(err, "failed to run analysis") return err } + // defer cleaning created resources here instead of PostRun + // if Run returns an error, PostRun does not run + defer func() { + if err := analyzeCmd.CleanAnalysisResources(cmd.Context()); err != nil { + log.Error(err, "failed to clean temporary directories") + } + }() } else { err := analyzeCmd.RunAnalysisOverrideProviderSettings(cmd.Context()) if err != nil { @@ -200,7 +215,8 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { log.Error(err, "failed to create json output file") return err } - err = analyzeCmd.GenerateStaticReport(cmd.Context()) + + err = analyzeCmd.GenerateStaticReport(cmd.Context(), providersMap) if err != nil { log.Error(err, "failed to generate static report") return err @@ -232,6 +248,7 @@ func NewAnalyzeCmd(log logr.Logger) *cobra.Command { analyzeCommand.Flags().StringVar(&analyzeCmd.incidentSelector, "incident-selector", "", "an expression to select incidents based on custom variables. ex: (!package=io.konveyor.demo.config-utils)") analyzeCommand.Flags().StringArrayVarP(&analyzeCmd.depFolders, "dependency-folders", "d", []string{}, "directory for dependencies") analyzeCommand.Flags().StringVar(&analyzeCmd.overrideProviderSettings, "override-provider-settings", "", "override the provider settings, the analysis pod will be run on the host network and no providers with be started up") + analyzeCommand.Flags().StringArrayVar(&analyzeCmd.provider, "provider", []string{}, "set provider to use") return analyzeCommand } @@ -402,6 +419,106 @@ func (a *analyzeCommand) CheckOverwriteOutput() error { return nil } +// TODO add network and volume w/ interface +type ProviderInit struct { + port int + image string +} + +func (a *analyzeCommand) setProviders(components []model.Component, foundProviders []string) ([]string, error) { + if len(a.provider) > 0 { + for _, p := range a.provider { + foundProviders = append(foundProviders, p) + return foundProviders, nil + } + } + for _, c := range components { + a.log.V(5).Info("Got component", "component language", c.Languages, "path", c.Path) + for _, l := range c.Languages { + if l.Name == "C#" { + foundDotNetProv, err := setDotNetProvider(l.Frameworks) + if err != nil { + return nil, err + } + foundProviders = append(foundProviders, foundDotNetProv) + } else { + foundProviders = append(foundProviders, strings.ToLower(l.Name)) + } + } + } + return foundProviders, nil +} + +func setDotNetProvider(frameworks []string) (string, error) { + if len(frameworks) > 0 { + for _, f := range frameworks { + if f == "v4.5" || f == "v4.6" || f == "v4.7" || f == "v4.8" { + return dotNetFrameworkProvider, nil + } else { + return dotNetProvider, nil + } + } + } + return "", fmt.Errorf("unable to set dotnet provider") +} + +func setProviderInitInfo(foundProviders []string) (map[string]ProviderInit, error) { + providers := map[string]ProviderInit{} + for _, prov := range foundProviders { + port, err := freeport.GetFreePort() + if err != nil { + return nil, err + } + switch prov { + case javaProvider: + providers[javaProvider] = ProviderInit{ + port: port, + image: Settings.JavaProviderImage, + } + case goProvider: + providers[goProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + case pythonProvider: + providers[pythonProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + case nodeJSProvider: + providers[nodeJSProvider] = ProviderInit{ + port: port, + image: Settings.GenericProviderImage, + } + // TODO + // case dotNetProvider, dotNetFrameworkProvider: + // providers[nodeJSProvider] = ProviderInit{ + // port: port, + // image: "", + // } + } + } + return providers, nil +} + +func (a *analyzeCommand) validateProviders(providers []string) error { + validProvs := []string{ + javaProvider, + pythonProvider, + goProvider, + nodeJSProvider, + dotNetProvider, + dotNetFrameworkProvider, + } + for _, prov := range providers { + //validate other providers + if !slices.Contains(validProvs, prov) { + return fmt.Errorf("provider %v not supported. Use --providerOverride option", prov) + } + } + return nil +} + func (a *analyzeCommand) ListLabels(ctx context.Context) error { return a.fetchLabels(ctx, a.listSources, a.listTargets, os.Stdout) } @@ -558,7 +675,7 @@ func (a *analyzeCommand) getDepsFolders() (map[string]string, []string) { return vols, dependencyFolders } -func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]int) (map[string]string, error) { +func (a *analyzeCommand) getConfigVolumes(providers map[string]ProviderInit) (map[string]string, error) { tempDir, err := os.MkdirTemp("", "analyze-config-") if err != nil { a.log.V(1).Error(err, "failed creating temp dir", "dir", tempDir) @@ -567,28 +684,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i a.log.V(1).Info("created directory for provider settings", "dir", tempDir) a.tempDirs = append(a.tempDirs, tempDir) - var foundJava bool - var foundGolang bool - var foundPython bool - var foundNode bool - switch providers[0] { - case javaProvider: - foundJava = true - case goProvider: - foundGolang = true - case pythonProvider: - foundPython = true - case nodeJSProvider: - foundNode = true - default: - return nil, fmt.Errorf("unable to find config for provider %v", providers[0]) - } - - // TODO (pgaikwad): binaries don't work with alizer right now, we need to revisit this - if !foundJava && a.isFileInput { - foundJava = true - } - otherProvsMountPath := SourceMountPath // when input is a file, it means it's probably a binary // only java provider can work with binaries, all others @@ -599,7 +694,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i javaConfig := provider.Config{ Name: javaProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[javaProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[javaProvider].port), InitConfig: []provider.InitConfig{ { Location: SourceMountPath, @@ -627,7 +722,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i goConfig := provider.Config{ Name: goProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[goProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[goProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.FullAnalysisMode, @@ -643,7 +738,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i pythonConfig := provider.Config{ Name: pythonProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[pythonProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[pythonProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.SourceOnlyAnalysisMode, @@ -658,7 +753,7 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i nodeJSConfig := provider.Config{ Name: nodeJSProvider, - Address: fmt.Sprintf("0.0.0.0:%v", ports[nodeJSProvider]), + Address: fmt.Sprintf("0.0.0.0:%v", providers[nodeJSProvider].port), InitConfig: []provider.InitConfig{ { AnalysisMode: provider.SourceOnlyAnalysisMode, @@ -671,16 +766,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i }, } - vols, dependencyFolders := a.getDepsFolders() - if len(dependencyFolders) != 0 { - if providers[0] == pythonProvider { - pythonConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders - } - if providers[0] == nodeJSProvider { - nodeJSConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders - } - } - provConfig := []provider.Config{ { Name: "builtin", @@ -692,16 +777,24 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i }, }, } - - switch { - case foundJava: - provConfig = append(provConfig, javaConfig) - case foundGolang && a.mode == string(provider.FullAnalysisMode): - provConfig = append(provConfig, goConfig) - case foundPython: - provConfig = append(provConfig, pythonConfig) - case foundNode: - provConfig = append(provConfig, nodeJSConfig) + vols, dependencyFolders := a.getDepsFolders() + for prov, _ := range providers { + switch prov { + case javaProvider: + provConfig = append(provConfig, javaConfig) + case goProvider: + provConfig = append(provConfig, goConfig) + case pythonProvider: + if len(dependencyFolders) != 0 { + pythonConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders + } + provConfig = append(provConfig, pythonConfig) + case nodeJSProvider: + if len(dependencyFolders) != 0 { + nodeJSConfig.InitConfig[0].ProviderSpecificConfig["dependencyFolders"] = dependencyFolders + } + provConfig = append(provConfig, nodeJSConfig) + } } // Set proxy to providers @@ -727,7 +820,6 @@ func (a *analyzeCommand) getConfigVolumes(providers []string, ports map[string]i "failed to write provider config", "dir", tempDir, "file", "settings.json") return nil, err } - settingsVols := map[string]string{ tempDir: ConfigMountPath, } @@ -945,7 +1037,7 @@ func (a *analyzeCommand) createContainerVolume() (string, error) { return volName, nil } -func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName string, volName string, providers []string, retry int) error { +func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName string, volName string, providers map[string]ProviderInit, retry int) error { if retry == 0 { return fmt.Errorf("too many provider container retry attempts") } @@ -958,62 +1050,68 @@ func (a *analyzeCommand) retryProviderContainer(ctx context.Context, networkName return nil } -func (a *analyzeCommand) RunProviders(ctx context.Context, networkName string, volName string, providers []string, retry int) (map[string]int, error) { - providerPorts := map[string]int{} - port, err := freeport.GetFreePort() - if err != nil { - return nil, err - } - volumes := map[string]string{ - // application source code - volName: SourceMountPath, - } - vols, _ := a.getDepsFolders() - if len(vols) != 0 { - maps.Copy(volumes, vols) - } - - var providerImage string - switch providers[0] { - case javaProvider: - providerImage = Settings.JavaProviderImage - providerPorts[javaProvider] = port - case goProvider: - providerImage = Settings.GenericProviderImage - providerPorts[goProvider] = port - case pythonProvider: - providerImage = Settings.GenericProviderImage - providerPorts[pythonProvider] = port - case nodeJSProvider: - providerImage = Settings.GenericProviderImage - providerPorts[nodeJSProvider] = port - default: - return nil, fmt.Errorf("unable to run unsupported provider %v", providers[0]) - } - args := []string{fmt.Sprintf("--port=%v", port)} - a.log.Info("starting provider", "provider", providers[0]) - - con := container.NewContainer() - err = con.Run( - ctx, - container.WithImage(providerImage), - container.WithLog(a.log.V(1)), - container.WithVolumes(volumes), - container.WithContainerToolBin(Settings.PodmanBinary), - container.WithEntrypointArgs(args...), - container.WithDetachedMode(true), - container.WithCleanup(a.cleanup), - container.WithNetwork(networkName), - ) - if err != nil { - err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) - if err != nil { - return nil, err +func (a *analyzeCommand) RunProviders(ctx context.Context, networkName string, volName string, providers map[string]ProviderInit, retry int) (map[string]ProviderInit, error) { + runFirstProv := false + for prov, inits := range providers { + volumes := map[string]string{ + // application source code + volName: SourceMountPath, + } + vols, _ := a.getDepsFolders() + if len(vols) != 0 { + maps.Copy(volumes, vols) + } + args := []string{fmt.Sprintf("--port=%v", inits.port)} + // we have to start the fist provider separately to create the shared + // container network to then add other providers to the network + if !runFirstProv { + a.log.Info("starting first provider", "provider", prov) + con := container.NewContainer() + err := con.Run( + ctx, + container.WithImage(inits.image), + container.WithLog(a.log.V(1)), + container.WithVolumes(volumes), + container.WithContainerToolBin(Settings.PodmanBinary), + container.WithEntrypointArgs(args...), + container.WithDetachedMode(true), + container.WithCleanup(a.cleanup), + container.WithNetwork(networkName), + ) + if err != nil { + err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) + if err != nil { + return nil, err + } + } + a.providerContainerNames = append(a.providerContainerNames, con.Name) } + // start additional providers + if runFirstProv && len(providers) > 1 { + a.log.Info("starting provider", "provider", prov) + con := container.NewContainer() + err := con.Run( + ctx, + container.WithImage(inits.image), + container.WithLog(a.log.V(1)), + container.WithVolumes(volumes), + container.WithContainerToolBin(Settings.PodmanBinary), + container.WithEntrypointArgs(args...), + container.WithDetachedMode(true), + container.WithCleanup(a.cleanup), + container.WithNetwork(fmt.Sprintf("container:%v", a.providerContainerNames[0])), + ) + if err != nil { + err := a.retryProviderContainer(ctx, networkName, volName, providers, retry) + if err != nil { + return nil, err + } + } + a.providerContainerNames = append(a.providerContainerNames, con.Name) + } + runFirstProv = true } - - a.providerContainerNames = append(a.providerContainerNames, con.Name) - return providerPorts, nil + return providers, nil } func (a *analyzeCommand) RunAnalysisOverrideProviderSettings(ctx context.Context) error { @@ -1114,7 +1212,7 @@ func (a *analyzeCommand) RunAnalysisOverrideProviderSettings(ctx context.Context return nil } -func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, volName string, providers []string, ports map[string]int) error { +func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, volName string, providersMap map[string]ProviderInit) error { volumes := map[string]string{ // application source code volName: SourceMountPath, @@ -1132,8 +1230,7 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v // for cleanup purposes a.tempDirs = append(a.tempDirs, xmlOutputDir) } - - configVols, err := a.getConfigVolumes(providers, ports) + configVols, err := a.getConfigVolumes(providersMap) if err != nil { a.log.V(1).Error(err, "failed to get config volumes for analysis") return err @@ -1148,43 +1245,33 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v } maps.Copy(volumes, ruleVols) } - args := []string{ fmt.Sprintf("--provider-settings=%s", ProviderSettingsMountPath), fmt.Sprintf("--output-file=%s", AnalysisOutputMountPath), fmt.Sprintf("--context-lines=%d", a.contextLines), } - // TODO update for running multiple apps - if providers[0] != javaProvider { + // default rulesets are only java rules + // may want to change this in the future + if _, ok := providersMap[javaProvider]; !ok { a.enableDefaultRulesets = false } - if a.enableDefaultRulesets { args = append(args, fmt.Sprintf("--rules=%s/", RulesetPath)) } - if a.incidentSelector != "" { args = append(args, fmt.Sprintf("--incident-selector=%s", a.incidentSelector)) } - if len(a.rules) > 0 { args = append(args, fmt.Sprintf("--rules=%s/", CustomRulePath)) } - if a.jaegerEndpoint != "" { args = append(args, "--enable-jaeger") args = append(args, "--jaeger-endpoint") args = append(args, a.jaegerEndpoint) } - - // python and node providers do not yet support dep analysis - if !a.analyzeKnownLibraries && (providers[0] != pythonProvider && providers[0] != nodeJSProvider) { - args = append(args, - fmt.Sprintf("--dep-label-selector=(!%s=open-source)", provider.DepSourceLabel)) - } if a.logLevel != nil { args = append(args, fmt.Sprintf("--verbose=%d", *a.logLevel)) } @@ -1193,14 +1280,17 @@ func (a *analyzeCommand) RunAnalysis(ctx context.Context, xmlOutputDir string, v args = append(args, fmt.Sprintf("--label-selector=%s", labelSelector)) } - switch true { - case a.mode == string(provider.FullAnalysisMode) && providers[0] == pythonProvider: - a.mode = string(provider.SourceOnlyAnalysisMode) - case a.mode == string(provider.FullAnalysisMode) && providers[0] == nodeJSProvider: - a.mode = string(provider.SourceOnlyAnalysisMode) - default: - a.log.Info("running dependency retrieval during analysis") - args = append(args, fmt.Sprintf("--dep-output-file=%s", DepsOutputMountPath)) + // as of now only java has dep capability + if _, ok := providersMap[javaProvider]; ok { + // need a smarter way of handling this with a java provider + other provider + if len(providersMap) == 1 { + if !a.analyzeKnownLibraries { + args = append(args, + fmt.Sprintf("--dep-label-selector=(!%s=open-source)", provider.DepSourceLabel)) + } + a.log.Info("running dependency retrieval during analysis") + args = append(args, fmt.Sprintf("--dep-output-file=%s", DepsOutputMountPath)) + } } analysisLogFilePath := filepath.Join(a.output, "analysis.log") @@ -1295,7 +1385,7 @@ func (a *analyzeCommand) CreateJSONOutput() error { return nil } -func (a *analyzeCommand) GenerateStaticReport(ctx context.Context) error { +func (a *analyzeCommand) GenerateStaticReport(ctx context.Context, provMap map[string]ProviderInit) error { if a.skipStaticReport { return nil } @@ -1309,9 +1399,13 @@ func (a *analyzeCommand) GenerateStaticReport(ctx context.Context) error { fmt.Sprintf("--output-path=%s", path.Join("/usr/local/static-report/output.js")), fmt.Sprintf("--application-name-list=%s", filepath.Base(a.input)), } - if a.mode == string(provider.FullAnalysisMode) { - staticReportArgs = append(staticReportArgs, - fmt.Sprintf("--deps-output-list=%s", DepsOutputMountPath)) + // as of now, only java provider has dep capability + // TODO smarter way to handle this + if _, ok := provMap[javaProvider]; ok { + if len(provMap) == 1 && a.mode == string(provider.FullAnalysisMode) { + staticReportArgs = append(staticReportArgs, + fmt.Sprintf("--deps-output-list=%s", DepsOutputMountPath)) + } } cpArgs := []string{"&& cp -r", "/usr/local/static-report", OutputPath}