Merge pull request #75 from devtron-labs/release-candidate-v0.22.0

release: Release candidate v0.22.0

Wire.go

@@ -1,3 +1,6 @@
+//go:build wireinject
+// +build wireinject
+
 /*
  * Copyright (c) 2024. Devtron Inc.
  *
@@ -14,9 +17,6 @@
  * limitations under the License.
  */
 
-//go:build wireinject
-// +build wireinject
-
 package main
 
 import (
@@ -51,11 +51,10 @@ func InitializeApp() (*App, error) {
 		client.NewPubSubClientServiceImpl,
 		klarService.NewKlarServiceImpl,
 		wire.Bind(new(klarService.KlarService), new(*klarService.KlarServiceImpl)),
+		pubsub.NewNatsSubscriptionModeConfig,
 		pubsub.NewNatSubscription,
 		grafeasService.NewKlarServiceImpl,
 		wire.Bind(new(grafeasService.GrafeasService), new(*grafeasService.GrafeasServiceImpl)),
-		pubsub.NewTestPublishImpl,
-		wire.Bind(new(pubsub.TestPublish), new(*pubsub.TestPublishImpl)),
 
 		clairService.GetClairConfig,
 		clairService.NewClairServiceImpl,

api/RestHandler.go

@@ -24,44 +24,43 @@ import (
 	"github.com/devtron-labs/image-scanner/pkg/klarService"
 	"github.com/devtron-labs/image-scanner/pkg/security"
 	"github.com/devtron-labs/image-scanner/pkg/sql/bean"
+	"github.com/devtron-labs/image-scanner/pkg/sql/repository"
 	"github.com/devtron-labs/image-scanner/pkg/user"
-	"github.com/devtron-labs/image-scanner/pubsub"
 	"go.uber.org/zap"
 	"net/http"
 	"os"
+	"time"
 )
 
 type RestHandler interface {
 	ScanForVulnerability(w http.ResponseWriter, r *http.Request)
+	ScanForVulnerabilityEvent(scanConfig *common.ImageScanEvent) (*common.ScanEventResponse, error)
 }
 
 func NewRestHandlerImpl(logger *zap.SugaredLogger,
-	testPublish pubsub.TestPublish,
 	grafeasService grafeasService.GrafeasService,
 	userService user.UserService, imageScanService security.ImageScanService,
 	klarService klarService.KlarService,
 	clairService clairService.ClairService,
 	imageScanConfig *security.ImageScanConfig) *RestHandlerImpl {
 	return &RestHandlerImpl{
-		logger:           logger,
-		testPublish:      testPublish,
+		Logger:           logger,
 		grafeasService:   grafeasService,
 		userService:      userService,
-		imageScanService: imageScanService,
-		klarService:      klarService,
-		clairService:     clairService,
+		ImageScanService: imageScanService,
+		KlarService:      klarService,
+		ClairService:     clairService,
 		imageScanConfig:  imageScanConfig,
 	}
 }
 
 type RestHandlerImpl struct {
-	logger           *zap.SugaredLogger
-	testPublish      pubsub.TestPublish
+	Logger           *zap.SugaredLogger
 	grafeasService   grafeasService.GrafeasService
 	userService      user.UserService
-	imageScanService security.ImageScanService
-	klarService      klarService.KlarService
-	clairService     clairService.ClairService
+	ImageScanService security.ImageScanService
+	KlarService      klarService.KlarService
+	ClairService     clairService.ClairService
 	imageScanConfig  *security.ImageScanConfig
 }
 type Response struct {
@@ -88,64 +87,90 @@ func (impl *RestHandlerImpl) ScanForVulnerability(w http.ResponseWriter, r *http
 	var scanConfig common.ImageScanEvent
 	err := decoder.Decode(&scanConfig)
 	if err != nil {
-		impl.logger.Errorw("error in decode request", "error", err)
-		writeJsonResp(w, err, nil, http.StatusBadRequest)
+		impl.Logger.Errorw("error in decode request", "error", err)
+		WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
+	impl.Logger.Infow("imageScan event", "scanConfig", scanConfig)
+	result, err := impl.ScanForVulnerabilityEvent(&scanConfig)
+	if err != nil {
+		WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		return
+	}
+	impl.Logger.Debugw("save", "status", result)
+	WriteJsonResp(w, err, result, http.StatusOK)
+}
+
+func (impl *RestHandlerImpl) ScanForVulnerabilityEvent(scanConfig *common.ImageScanEvent) (*common.ScanEventResponse, error) {
 	if scanConfig.UserId == 0 {
 		scanConfig.UserId = 1 //setting user as system user in case of empty user data
 	}
-	impl.logger.Infow("image scan req", "req", scanConfig)
-	var result *common.ScanEventResponse
-	tool, err := impl.imageScanService.GetActiveTool()
+	impl.Logger.Infow("image scan req", "req", scanConfig)
+
+	tool, err := impl.ImageScanService.GetActiveTool()
 	if err != nil {
-		impl.logger.Errorw("err in image scanning", "err", err)
-		writeJsonResp(w, err, nil, http.StatusInternalServerError)
-		return
+		impl.Logger.Errorw("err in image scanning", "err", err)
+		return nil, err
 	}
-	executionHistory, executionHistoryDirPath, err := impl.imageScanService.RegisterScanExecutionHistoryAndState(&scanConfig, tool)
+	//creating execution history
+	scanEventJson, err := json.Marshal(scanConfig)
 	if err != nil {
-		impl.logger.Errorw("service err, RegisterScanExecutionHistoryAndState", "err", err)
-		writeJsonResp(w, err, nil, http.StatusInternalServerError)
-		return
+		impl.Logger.Errorw("error in marshalling scanEvent", "event", scanConfig, "err", err)
+		return nil, err
+	}
+	executionHistoryModel := &repository.ImageScanExecutionHistory{
+		Image:              scanConfig.Image,
+		ImageHash:          scanConfig.ImageDigest,
+		ExecutionTime:      time.Now(),
+		ExecutedBy:         scanConfig.UserId,
+		SourceMetadataJson: string(scanEventJson),
 	}
-	imageToBeScanned, err := impl.imageScanService.GetImageToBeScannedAndFetchCliEnv(&scanConfig)
+	executionHistory, executionHistoryDirPath, err := impl.ImageScanService.RegisterScanExecutionHistoryAndState(executionHistoryModel, tool)
 	if err != nil {
-		impl.logger.Errorw("service err, GetImageToBeScanned", "err", err)
-		writeJsonResp(w, err, nil, http.StatusInternalServerError)
-		return
+		impl.Logger.Errorw("service err, RegisterScanExecutionHistoryAndState", "err", err)
+		return nil, err
+	}
+	result, err := impl.ScanImageAsPerTool(scanConfig, tool, executionHistory, executionHistoryDirPath)
+	if err != nil {
+		impl.Logger.Errorw("service err, ScanImageAsPerTool", "err", err)
+		return nil, err
+	}
+	//deleting executionDirectoryPath with files as well
+	err = os.RemoveAll(executionHistoryDirPath)
+	if err != nil {
+		impl.Logger.Errorw("error in deleting executionHistoryDirectory", "err", err)
+		return nil, err
+	}
+	return result, nil
+}
+
+func (impl *RestHandlerImpl) ScanImageAsPerTool(scanConfig *common.ImageScanEvent, tool *repository.ScanToolMetadata,
+	executionHistory *repository.ImageScanExecutionHistory, executionHistoryDirPath string) (*common.ScanEventResponse, error) {
+	var result = &common.ScanEventResponse{}
+	imageToBeScanned, err := impl.ImageScanService.GetImageToBeScannedAndFetchCliEnv(scanConfig)
+	if err != nil {
+		impl.Logger.Errorw("service err, GetImageToBeScanned", "err", err)
+		return nil, err
 	}
 	scanConfig.Image = imageToBeScanned
 	if tool.Name == bean.ScanToolClair && tool.Version == bean.ScanToolVersion2 {
-		result, err = impl.klarService.Process(&scanConfig, executionHistory)
+		result, err = impl.KlarService.Process(scanConfig, executionHistory)
 		if err != nil {
-			impl.logger.Errorw("err in process msg", "err", err)
-			writeJsonResp(w, err, nil, http.StatusInternalServerError)
-			return
+			impl.Logger.Errorw("err in process msg", "err", err)
+			return nil, err
 		}
 	} else if tool.Name == bean.ScanToolClair && tool.Version == bean.ScanToolVersion4 {
-		result, err = impl.clairService.ScanImage(&scanConfig, tool, executionHistory)
+		result, err = impl.ClairService.ScanImage(scanConfig, tool, executionHistory)
 		if err != nil {
-			impl.logger.Errorw("err in process msg", "err", err)
-			writeJsonResp(w, err, nil, http.StatusInternalServerError)
-			return
+			impl.Logger.Errorw("err in process msg", "err", err)
+			return nil, err
 		}
 	} else {
-		err = impl.imageScanService.ScanImage(&scanConfig, tool, executionHistory, executionHistoryDirPath)
+		err = impl.ImageScanService.ScanImage(scanConfig, tool, executionHistory, executionHistoryDirPath)
 		if err != nil {
-			impl.logger.Errorw("err in process msg", "err", err)
-			writeJsonResp(w, err, nil, http.StatusInternalServerError)
-			return
+			impl.Logger.Errorw("err in process msg", "err", err)
+			return nil, err
 		}
 	}
-	//deleting executionDirectoryPath with files as well
-	err = os.RemoveAll(executionHistoryDirPath)
-	if err != nil {
-		impl.logger.Errorw("error in deleting executionHistoryDirectory", "err", err)
-		writeJsonResp(w, err, nil, http.StatusInternalServerError)
-		return
-	}
-
-	impl.logger.Debugw("save", "status", result)
-	writeJsonResp(w, err, result, http.StatusOK)
+	return result, nil
 }

api/apiError.go

@@ -18,14 +18,14 @@ package api
 
 import (
 	"encoding/json"
-	"github.com/devtron-labs/image-scanner/internal/util"
+	"github.com/devtron-labs/image-scanner/internals/util"
 	"github.com/juju/errors"
 	"net/http"
 )
 
-// use of writeJsonRespStructured is preferable. it api exists due to historical reason
-// err.message is used as internal message for ApiError object in resp
-func writeJsonResp(w http.ResponseWriter, err error, respBody interface{}, status int) {
+// use of WriteJsonRespStructured is preferable. it api exists due to historical reason
+// err.message is used as internals message for ApiError object in resp
+func WriteJsonResp(w http.ResponseWriter, err error, respBody interface{}, status int) {
 	response := ResponseV2{}
 	if err == nil {
 		response.Result = respBody
@@ -85,7 +85,7 @@ func writeJsonResp(w http.ResponseWriter, err error, respBody interface{}, statu
 }
 
 // use this method when we have specific api error to be conveyed to api User
-func writeJsonRespStructured(w http.ResponseWriter, err error, respBody interface{}, status int, apiErrors []*util.ApiError) {
+func WriteJsonRespStructured(w http.ResponseWriter, err error, respBody interface{}, status int, apiErrors []*util.ApiError) {
 	response := ResponseV2{}
 	response.Code = status
 	response.Status = http.StatusText(status)

common/bean.go

@@ -17,6 +17,7 @@
 package common
 
 import (
+	git "github.com/devtron-labs/common-lib/git-manager"
 	"github.com/optiopay/klar/clair"
 	"github.com/quay/claircore"
 	"strings"
@@ -67,33 +68,39 @@ type ImageScanRenderDto struct {
 }
 
 type ImageScanEvent struct {
-	Image            string `json:"image"`
-	ImageDigest      string `json:"imageDigest"`
-	AppId            int    `json:"appId"`
-	EnvId            int    `json:"envId"`
-	PipelineId       int    `json:"pipelineId"`
-	CiArtifactId     int    `json:"ciArtifactId"`
-	UserId           int    `json:"userId"`
-	AccessKey        string `json:"accessKey"`
-	SecretKey        string `json:"secretKey"`
-	Token            string `json:"token"`
-	AwsRegion        string `json:"awsRegion"`
-	DockerRegistryId string `json:"dockerRegistryId"`
-	DockerConnection string `json:"dockerConnection"`
-	DockerCert       string `json:"dockerCert"`
-	//CiProjectDetails []helper.CiProjectDetails `json:"ciProjectDetails"`
-	SourceType     SourceType    `json:"sourceType"`
-	SourceSubType  SourceSubType `json:"sourceSubType"`
-	CiWorkflowId   int           `json:"ciWorkflowId"`
-	CdWorkflowId   int           `json:"cdWorkflowId"`
-	ChartHistoryId int           `json:"chartHistoryId"`
-	ManifestData   *ManifestData `json:"manifestData"`
+	Image            string                 `json:"image"`
+	ImageDigest      string                 `json:"imageDigest"`
+	AppId            int                    `json:"appId"`
+	EnvId            int                    `json:"envId"`
+	PipelineId       int                    `json:"pipelineId"`
+	CiArtifactId     int                    `json:"ciArtifactId"`
+	UserId           int                    `json:"userId"`
+	AccessKey        string                 `json:"accessKey"`
+	SecretKey        string                 `json:"secretKey"`
+	Token            string                 `json:"token"`
+	AwsRegion        string                 `json:"awsRegion"`
+	DockerRegistryId string                 `json:"dockerRegistryId"`
+	DockerConnection string                 `json:"dockerConnection"`
+	DockerCert       string                 `json:"dockerCert"`
+	CiProjectDetails []git.CiProjectDetails `json:"ciProjectDetails"`
+	SourceType       SourceType             `json:"sourceType"`
+	SourceSubType    SourceSubType          `json:"sourceSubType"`
+	CiWorkflowId     int                    `json:"ciWorkflowId"`
+	CdWorkflowId     int                    `json:"cdWorkflowId"`
+	ChartHistoryId   int                    `json:"chartHistoryId"`
+	ManifestData     *ManifestData          `json:"manifestData"`
+	ReScan           bool                   `json:"reScan"`
+}
+
+func (r *ImageScanEvent) IsManifest() bool {
+	return r.SourceType == SourceTypeCode && r.SourceSubType == SourceSubTypeManifest
 }
 
 type ScanEventResponse struct {
 	RequestData         *ImageScanEvent            `json:"requestData"`
 	ResponseDataClairV4 []*claircore.Vulnerability `json:"responseDataClairV4"`
 	ResponseDataClairV2 []*clair.Vulnerability     `json:"ResponseDataClairV2"`
+	CodeScanRes         interface{}                `json:"codeScanResponse"`
 }
 
 const (

internal/step-lib/util/cli-util/handler.go → internals/step-lib/util/cli-util/handler.go

@@ -19,7 +19,7 @@ package cli_util
 import (
 	"context"
 	"github.com/devtron-labs/image-scanner/common"
-	common_util "github.com/devtron-labs/image-scanner/internal/step-lib/util/common-util"
+	common_util "github.com/devtron-labs/image-scanner/internals/step-lib/util/common-util"
 	"io"
 	"log"
 	"os"
@@ -40,7 +40,9 @@ func HandleCliRequest(baseCommand, outputFileName string, ctx context.Context, o
 	for arg, value := range args {
 		//assuming '-' or '--' is provided by user (if applicable)
 		argsSlice = append(argsSlice, arg)
-		argsSlice = append(argsSlice, value)
+		if value != "" {
+			argsSlice = append(argsSlice, value)
+		}
 	}
 	command := exec.CommandContext(ctx, common.SHELL_COMMAND, common.COMMAND_ARGS, baseCommand)
 	command.Env = append(command.Env, cliCommandEnv...)
@@ -50,7 +52,7 @@ func HandleCliRequest(baseCommand, outputFileName string, ctx context.Context, o
 		err, output = executeStaticCliRequest(command, outputFileName)
 	}
 	if err != nil {
-		log.Println("error in executing cli request", "err", err, "req", command)
+		log.Println("error in executing cli request", "err", err, "req", command, string(output))
 		return output, err
 	}
 	return output, nil
@@ -59,8 +61,8 @@ func HandleCliRequest(baseCommand, outputFileName string, ctx context.Context, o
 func executeStaticCliRequest(command *exec.Cmd, outputFileName string) (error, []byte) {
 	op, err := command.CombinedOutput()
 	if err != nil {
-		log.Println("error in running command", "err", err)
-		return err, nil
+		log.Println("error in running command", "err", err, "op", string(op))
+		return err, op
 	}
 	// If output is already stored in file, considering the output from file (file is created by tool over here)
 	if outputFileName != "" && op != nil {

internal/step-lib/util/http-util/handler.go → internals/step-lib/util/http-util/handler.go

@@ -19,7 +19,7 @@ package http_util
 import (
 	"context"
 	"fmt"
-	common_util "github.com/devtron-labs/image-scanner/internal/step-lib/util/common-util"
+	common_util "github.com/devtron-labs/image-scanner/internals/step-lib/util/common-util"
 	"io"
 	"log"
 	"net/http"

internal/util/ErrorUtil.go → internals/util/ErrorUtil.go

@@ -33,7 +33,7 @@ func (e *ApiError) Error() string {
 	return e.InternalMessage
 }
 
-// default internal will be set
+// default internals will be set
 func (e *ApiError) ErrorfInternal(format string, a ...interface{}) error {
 	return &ApiError{InternalMessage: fmt.Sprintf(format, a...)}
 }

pkg/clairService/ClairService.go

@@ -105,7 +105,7 @@ func (impl *ClairServiceImpl) ScanImage(scanEvent *common.ImageScanEvent, tool *
 	scanEventResponse := &common.ScanEventResponse{
 		RequestData: scanEvent,
 	}
-	isImageScanned, err := impl.ImageScanService.IsImageScanned(scanEvent.Image)
+	_, isImageScanned, err := impl.ImageScanService.IsImageScanned(scanEvent.Image, false)
 	if err != nil && err != pg.ErrNoRows {
 		impl.Logger.Errorw("error in fetching scan history ", "err", err, "image", scanEvent.Image)
 		return nil, err

pkg/klarService/KlarService.go

@@ -99,7 +99,7 @@ func (impl *KlarServiceImpl) Process(scanEvent *common.ImageScanEvent, execution
 		impl.logger.Errorw("error in getting docker registry by id", "err", err, "id", scanEvent.DockerRegistryId)
 		return nil, err
 	}
-	scanned, err := impl.imageScanService.IsImageScanned(scanEvent.Image)
+	_, scanned, err := impl.imageScanService.IsImageScanned(scanEvent.Image, false)
 	if err != nil && err != pg.ErrNoRows {
 		impl.logger.Errorw("error in fetching scan history ", "err", err)
 		return nil, err