Merge branch 'master' into update_security_compliance

Dockerfile

@@ -33,6 +33,7 @@ RUN go mod download
 
 RUN if [ "$INSTALL_TOOLS" == "yes" ] ; then \
         go install github.com/swaggo/swag/cmd/[email protected] && \
+        go install gotest.tools/[email protected] && \
         curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
         | sh -s -- -b $(go env GOPATH)/bin v1.50.0 ; \
     fi

README.md

@@ -16,6 +16,7 @@ System Patch Manager is one of the applications for [console.redhat.com](https:/
 - [Control by private API](#control-by-private-api)
 - [VMaaS](#vmaas)
 - [Monitoring](#monitoring)
+- [Profiling] (#profiling)
 
 ## Development environment
 
@@ -135,5 +136,21 @@ Copy Grafana board json config to the temporary file, e.g. `grafana.json` and ru
 ./scripts/grafana-json-to-yaml.sh grafana.json > ./dashboards/grafana-dashboard-insights-patchman-engine-general.configmap.yaml
 ~~~
 
+## Profiling
+App can be profiled using [/net/http/pprof](https://pkg.go.dev/net/http/pprof). Profiler is exposed on app's private port.
+### Local development
+- set `ENABLE_PROFILE=true` in the `cont/common.env`
+- `docker-compose up --build`
+- `go tool pprof http://localhost:{port}/debug/pprof/{heap|profile|block|mutex}`
+available ports:
+- 9000 - manager
+- 9002 - listener
+- 9003 - evaluator-upload
+- 9004 - evaluator-recalc
+### Admin API
+- set `ENABLE_PROFILE_{container_name}=true` in the ClowdApp
+- download the profile file using internal api `/api/patch/admin/pprof/{manager|listener|evaluator_upload|evaluator_recalc}/{heap|profile|block|mutex|trace}`
+- `go tool pprof <saved.file>`
+
 ## Deps backup
 [patchman-engine-deps](https://github.com/RedHatInsights/patchman-engine-deps)

VERSION

@@ -1 +1 @@
-v3.1.27
+v3.5.7

base/api/client.go

@@ -36,7 +36,7 @@ func (o *Client) Request(ctx *context.Context, method, url string,
 
 	httpResp, err := utils.CallAPI(o.HTTPClient, httpReq, o.Debug)
 	if err != nil {
-		return nil, errors.Wrap(err, "Request failed")
+		return httpResp, errors.Wrap(err, "Request failed")
 	}
 
 	bodyBytes, err := io.ReadAll(httpResp.Body)

base/core/gintesting.go

@@ -31,7 +31,7 @@ func InitRouterWithParams(handler gin.HandlerFunc, account int, method, path str
 	}
 	router.Use(func(c *gin.Context) {
 		// set default api version for tests to latest
-		c.Set(middlewares.KeyApiver, 3)
+		c.Set(middlewares.KeyApiver, LatestAPIVersion)
 		for _, kv := range contextKVs {
 			c.Set(kv.Key, kv.Value)
 		}

base/database/batch.go

@@ -108,7 +108,7 @@ func statementFromObjects(db *gorm.DB, objects reflect.Value) (*gorm.Statement,
 	}()
 	// Get a map of the first element to calculate field names and number of
 	// placeholders.
-	firstObjectFields, err := objectToMap(db, firstElem)
+	firstObjectFields, err := objectToMap(schema, firstElem)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +127,7 @@ func statementFromObjects(db *gorm.DB, objects reflect.Value) (*gorm.Statement,
 	for i := 0; i < objects.Len(); i++ {
 		// Retrieve ith element as an interface
 		r := objects.Index(i).Interface()
-		row, err := objectToMap(db, r)
+		row, err := objectToMap(schema, r)
 		if err != nil {
 			return nil, err
 		}
@@ -212,15 +212,14 @@ func parseClause(clauseOnConflict clause.Expression) string {
 
 // objectToMap takes any object of type <T> and returns a map with the gorm
 // field DB name as key and the value as value
-func objectToMap(db *gorm.DB, object interface{}) (map[string]interface{}, error) {
+func objectToMap(schema *schema.Schema, object interface{}) (map[string]interface{}, error) {
 	attributes := make(map[string]interface{})
 	now := bulkNow
 
 	// De-reference pointers (and it's values)
 	rv := reflect.ValueOf(object)
 	if rv.Kind() == reflect.Ptr {
 		rv = rv.Elem()
-		object = rv.Interface()
 	}
 	if rv.Kind() != reflect.Struct {
 		return nil, errors.New("value must be kind of Struct")
@@ -230,8 +229,7 @@ func objectToMap(db *gorm.DB, object interface{}) (map[string]interface{}, error
 		now = Db.NowFunc()
 	}
 
-	parsedSchema, _ := schema.Parse(object, &sync.Map{}, db.NamingStrategy)
-	for _, field := range parsedSchema.Fields {
+	for _, field := range schema.Fields {
 		// Exclude relational record because it's not directly contained in database columns
 		fieldVaule := rv.FieldByName(field.Name).Interface()
 		_, hasForeignKey := field.TagSettings["FOREIGNKEY"]

base/database/dbtest.go

@@ -1,7 +1,7 @@
 package database
 
 type TestTable struct {
-	ID    uint   `gorm:"primary_key"`
+	ID    uint   `gorm:"primaryKey"`
 	Name  string `gorm:"unique"`
 	Email string
 }

base/database/query.go

@@ -4,12 +4,13 @@ import (
 	"app/base"
 	"app/base/utils"
 	"fmt"
-	"github.com/pkg/errors"
 	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/pkg/errors"
 )
 
 type AttrParser func(string) (interface{}, error)
@@ -91,7 +92,7 @@ func getQueryFromTags(v reflect.Type) (AttrMap, []AttrName, error) {
 				res[k] = v
 			}
 		} else {
-			columnName := field.Name
+			columnName := strings.ToLower(field.Name)
 			if expr, has := field.Tag.Lookup("gorm"); has {
 				match := ColumnNameRe.FindStringSubmatch(expr)
 				if len(match) > 0 {

base/database/utils.go

@@ -2,30 +2,28 @@ package database
 
 import (
 	"app/base/models"
+	"app/base/rbac"
 	"app/base/types"
 	"app/base/utils"
+	"errors"
 	"fmt"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/jackc/pgconn"
-
 	log "github.com/sirupsen/logrus"
+	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 )
 
-const (
-	PgErrorDuplicateKey = "23505"
-)
-
-func Systems(tx *gorm.DB, accountID int) *gorm.DB {
-	return tx.Table("system_platform sp").Where("sp.rh_account_id = ?", accountID)
+func Systems(tx *gorm.DB, accountID int, groups map[string]string) *gorm.DB {
+	tx = tx.Table("system_platform sp").Where("sp.rh_account_id = ?", accountID)
+	return InventoryHostsJoin(tx, groups)
 }
 
-func SystemAdvisories(tx *gorm.DB, accountID int) *gorm.DB {
-	return Systems(tx, accountID).
+func SystemAdvisories(tx *gorm.DB, accountID int, groups map[string]string) *gorm.DB {
+	return Systems(tx, accountID, groups).
 		Joins("JOIN system_advisories sa on sa.system_id = sp.id AND sa.rh_account_id = ?", accountID)
 }
 
@@ -34,8 +32,8 @@ func SystemPackagesShort(tx *gorm.DB, accountID int) *gorm.DB {
 		Where("spkg.rh_account_id = ?", accountID)
 }
 
-func SystemPackages(tx *gorm.DB, accountID int) *gorm.DB {
-	return Systems(tx, accountID).
+func SystemPackages(tx *gorm.DB, accountID int, groups map[string]string) *gorm.DB {
+	return Systems(tx, accountID, groups).
 		Joins("JOIN system_package spkg on spkg.system_id = sp.id AND spkg.rh_account_id = ?", accountID).
 		Joins("JOIN package p on p.id = spkg.package_id").
 		Joins("JOIN package_name pn on pn.id = spkg.name_id")
@@ -53,11 +51,11 @@ func PackageByName(tx *gorm.DB, pkgName string) *gorm.DB {
 	return Packages(tx).Where("pn.name = ?", pkgName)
 }
 
-func SystemAdvisoriesByInventoryID(tx *gorm.DB, accountID int, inventoryID string) *gorm.DB {
-	return SystemAdvisories(tx, accountID).Where("sp.inventory_id = ?::uuid", inventoryID)
+func SystemAdvisoriesByInventoryID(tx *gorm.DB, accountID int, groups map[string]string, inventoryID string) *gorm.DB {
+	return SystemAdvisories(tx, accountID, groups).Where("sp.inventory_id = ?::uuid", inventoryID)
 }
 
-func SystemAdvisoriesBySystemID(tx *gorm.DB, accountID, systemID int) *gorm.DB {
+func SystemAdvisoriesBySystemID(tx *gorm.DB, accountID int, systemID int64) *gorm.DB {
 	query := systemAdvisoriesQuery(tx, accountID).Where("sp.id = ?", systemID)
 	return query
 }
@@ -135,13 +133,10 @@ func ExecFile(filename string) error {
 	return err
 }
 
-func IsPgErrorCode(err error, pgCode string) bool {
-	switch e := err.(type) {
-	case *pgconn.PgError:
-		return e.Code == pgCode
-	default:
-		return false
-	}
+// compare the dialect translated errors(like gorm.ErrDuplicatedKey)
+func IsPgErrorCode(db *gorm.DB, err error, expectedGormErr error) bool {
+	translatedErr := db.Dialector.(*postgres.Dialector).Translate(err)
+	return errors.Is(translatedErr, expectedGormErr)
 }
 
 func logAndWait(query string) {
@@ -223,3 +218,21 @@ func DBWait(waitForDB string) {
 func ReadReplicaConfigured() bool {
 	return len(utils.Cfg.DBReadReplicaHost) > 0 && utils.Cfg.DBReadReplicaPort != 0
 }
+
+func InventoryHostsJoin(tx *gorm.DB, groups map[string]string) *gorm.DB {
+	tx = tx.Joins("JOIN inventory.hosts ih ON ih.id = sp.inventory_id")
+	if _, ok := groups[rbac.KeyGrouped]; !ok {
+		if _, ok := groups[rbac.KeyUngrouped]; ok {
+			// show only systems with '[]' group
+			return tx.Where("ih.groups = '[]'")
+		}
+		// return query without WHERE if there are no groups
+		return tx
+	}
+
+	tx = tx.Where("ih.groups @> ANY (?::jsonb[])", groups[rbac.KeyGrouped])
+	if _, ok := groups[rbac.KeyUngrouped]; ok {
+		tx = tx.Or("ih.groups = '[]'")
+	}
+	return tx.Where(tx)
+}

base/database/utils_test.go

@@ -0,0 +1,49 @@
+package database
+
+import (
+	"app/base/rbac"
+	"app/base/utils"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	// counts of systems from system_platform JOIN inventory.hosts
+	nGroup1    int64 = 6
+	nGroup2    int64 = 2
+	nUngrouped int64 = 6
+	nAll       int64 = 16
+)
+
+// nolint: lll
+var testCases = []map[int64]map[string]string{
+	{nGroup1: {rbac.KeyGrouped: `{"[{\"id\":\"inventory-group-1\"}]"}`}},
+	{nGroup2: {rbac.KeyGrouped: `{"[{\"id\":\"inventory-group-2\"}]"}`}},
+	{nGroup1 + nGroup2: {rbac.KeyGrouped: `{"[{\"id\":\"inventory-group-1\"}]","[{\"id\":\"inventory-group-2\"}]"}`}},
+	{nGroup1 + nUngrouped: {
+		rbac.KeyGrouped:   `{"[{\"id\":\"inventory-group-1\"}]"}`,
+		rbac.KeyUngrouped: "[]",
+	}},
+	{nUngrouped: {
+		rbac.KeyGrouped:   `{"[{\"id\":\"non-existing-group\"}]"}`,
+		rbac.KeyUngrouped: "[]",
+	}},
+	{0: {rbac.KeyGrouped: `{"[{\"id\":\"non-existing-group\"}]"}`}},
+	{nUngrouped: {rbac.KeyUngrouped: "[]"}},
+	{nAll: {}},
+	{nAll: nil},
+}
+
+func TestInventoryHostsJoin(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	Configure()
+
+	for _, tc := range testCases {
+		for expectedCount, groups := range tc {
+			var count int64
+			InventoryHostsJoin(Db.Table("system_platform sp"), groups).Count(&count)
+			assert.Equal(t, expectedCount, count)
+		}
+	}
+}

base/inventory/inventory.go

@@ -8,6 +8,8 @@ type SystemProfile struct {
 	DnfModules        *[]DnfModule    `json:"dnf_modules,omitempty"`
 	OperatingSystem   OperatingSystem `json:"operating_system,omitempty"`
 	Rhsm              Rhsm            `json:"rhsm,omitempty"`
+	Releasever        *string         `json:"releasever,omitempty"`
+	SatelliteManaged  bool            `json:"satellite_managed,omitempty"`
 }
 
 func (t *SystemProfile) GetInstalledPackages() []string {