diff --git a/base/api/client.go b/base/api/client.go index 96f8d81a1..6bab2de53 100644 --- a/base/api/client.go +++ b/base/api/client.go @@ -44,9 +44,11 @@ func (o *Client) Request(ctx *context.Context, method, url string, return httpResp, errors.Wrap(err, "Response body reading failed") } - err = json.Unmarshal(bodyBytes, responseOutPtr) - if err != nil { - return httpResp, errors.Wrap(err, "Response json parsing failed") + if len(bodyBytes) > 0 { + err = json.Unmarshal(bodyBytes, responseOutPtr) + if err != nil { + return httpResp, errors.Wrap(err, "Response json parsing failed") + } } return httpResp, nil } diff --git a/base/candlepin/candlepin.go b/base/candlepin/candlepin.go new file mode 100644 index 000000000..39083114d --- /dev/null +++ b/base/candlepin/candlepin.go @@ -0,0 +1,13 @@ +package candlepin + +type ConsumersUpdateRequest struct { + Environments []ConsumersUpdateEnvironment +} + +type ConsumersUpdateEnvironment struct { + ID string +} + +type ConsumersUpdateResponse struct { + Message string `json:"displayMessage"` +} diff --git a/base/utils/config.go b/base/utils/config.go index da29bd38b..ba8d97bb6 100644 --- a/base/utils/config.go +++ b/base/utils/config.go @@ -70,6 +70,9 @@ type coreConfig struct { // services VmaasAddress string RbacAddress string + CandlepinAddress string + CandlepinCert string + CandlepinKey string ManagerPrivateAddress string ListenerPrivateAddress string EvaluatorUploadPrivateAddress string @@ -154,6 +157,9 @@ func initTopicsFromEnv() { func initServicesFromEnv() { CoreCfg.VmaasAddress = Getenv("VMAAS_ADDRESS", CoreCfg.VmaasAddress) CoreCfg.RbacAddress = Getenv("RBAC_ADDRESS", CoreCfg.RbacAddress) + CoreCfg.CandlepinAddress = Getenv("CANDLEPIN_ADDRESS", CoreCfg.CandlepinAddress) + CoreCfg.CandlepinCert = Getenv("CANDLEPIN_CERT", CoreCfg.CandlepinCert) + CoreCfg.CandlepinKey = Getenv("CANDLEPIN_KEY", CoreCfg.CandlepinKey) } func initDBFromClowder() { diff --git a/conf/common.env b/conf/common.env index 3f9177640..160626276 100644 --- a/conf/common.env +++ b/conf/common.env @@ -21,3 +21,5 @@ TEMPLATE_TOPIC=platform.content-sources.template # If vmaas is running locally, its available here #VMAAS_ADDRESS=http://vmaas_webapp:8080 ENABLE_PROFILER=true + +CANDLEPIN_ADDRESS=http://platform:9001/candlepin diff --git a/conf/local.env b/conf/local.env index 6d309e442..26768ccdb 100644 --- a/conf/local.env +++ b/conf/local.env @@ -14,6 +14,7 @@ DB_SSLMODE=verify-full DB_SSLROOTCERT=dev/database/secrets/pgca.crt VMAAS_ADDRESS=http://localhost:9001 +CANDLEPIN_ADDRESS=http://localhost:9001/candlepin #KAFKA_ADDRESS=localhost:29092 KAFKA_GROUP=patchman diff --git a/dev/test_data.sql b/dev/test_data.sql index 39bf59068..d9ce77f0a 100644 --- a/dev/test_data.sql +++ b/dev/test_data.sql @@ -194,19 +194,19 @@ INSERT INTO inventory.hosts_v1_0 (id, insights_id, account, display_name, tags, '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 1}, "rhsm": {"version": "8.0"}}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-1", "name": "group1"}]'), ('00000000000000000000000000000004', '00000000-0000-0000-0004-000000000001', '1', '00000000-0000-0000-0000-000000000004', '[{"key": "k3", "value": "val4", "namespace": "ns1"}]', - '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 2}, "rhsm": {"version": "8.3"}}', + '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 2}, "rhsm": {"version": "8.3"}, "owner_id": "cccccccc-0000-0000-0001-000000000004"}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-1", "name": "group1"}]'), ('00000000000000000000000000000005', '00000000-0000-0000-0005-000000000001', '1', '00000000-0000-0000-0000-000000000005', '[{"key": "k1", "value": "val1", "namespace": "ns1"}]', - '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 3}, "rhsm": {"version": "8.3"}}', + '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 3}, "rhsm": {"version": "8.3"}, "owner_id": "cccccccc-0000-0000-0001-000000000005"}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-1", "name": "group1"}]'), ('00000000000000000000000000000006', '00000000-0000-0000-0006-000000000001', '1', '00000000-0000-0000-0000-000000000006', '[{"key": "k1", "value": "val1", "namespace": "ns1"}]', '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 7, "minor": 3}, "rhsm": {"version": "7.3"}, "mssql": { "version": "15.3.0"}}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-1", "name": "group1"}]'), ('00000000000000000000000000000007', '00000000-0000-0000-0007-000000000001', '1', '00000000-0000-0000-0000-000000000007','[{"key": "k1", "value": "val1", "namespace": "ns1"}]', - '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": "x"}, "rhsm": {"version": "8.x"}, "ansible": {"controller_version": "1.0"}}', + '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": "x"}, "rhsm": {"version": "8.x"}, "ansible": {"controller_version": "1.0"}, "owner_id": "cccccccc-0000-0000-0001-000000000007"}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-2", "name": "group2"}]'), ('00000000000000000000000000000008', '00000000-0000-0000-0008-000000000001', '1', '00000000-0000-0000-0000-000000000008', '[{"key": "k1", "value": "val1", "namespace": "ns1"}]', - '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 3}, "rhsm": {"version": "8.3"}}', + '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 3}, "rhsm": {"version": "8.3"}, "owner_id": "cccccccc-0000-0000-0001-000000000008"}', 'puptoo', '{}', 'org_1', '[{"id": "inventory-group-2", "name": "group2"}]'), ('00000000000000000000000000000009', '00000000-0000-0000-0009-000000000001', '2', '00000000-0000-0000-0000-000000000009', '[{"key": "k1", "value": "val1", "namespace": "ns1"}]', '2018-09-22 12:00:00-04', '2018-08-26 12:00:00-04', '2018-08-26 12:00:00-04', '{"sap_system": true, "operating_system": {"name": "RHEL", "major": 8, "minor": 1}, "rhsm": {"version": "8.1"}}', diff --git a/docker-compose.test.yml b/docker-compose.test.yml index c34efae87..3bfa68f6f 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -43,6 +43,7 @@ services: target: buildimg image: patchman-engine-app env_file: + - ./conf/common.env - ./conf/platform.env command: ./dev/scripts/docker-compose-entrypoint.sh platform restart: unless-stopped diff --git a/evaluator/evaluate.go b/evaluator/evaluate.go index 8af76c84c..c2905389d 100644 --- a/evaluator/evaluate.go +++ b/evaluator/evaluate.go @@ -323,8 +323,8 @@ func getUpdatesData(ctx context.Context, system *models.SystemPlatform) (*vmaas. utils.LogWarn("Vmaas response error, continuing with yum updates only", vmaasErr.Error()) } - if system.SatelliteManaged { - // satellite managed systems has vmaas updates APPLICABLE instead of INSTALLABLE + if system.SatelliteManaged || system.TemplateID != nil { + // satellite managed systems and systems using template has vmaas updates APPLICABLE instead of INSTALLABLE mergedUpdateList := vmaasData.GetUpdateList() for nevra := range mergedUpdateList { (*mergedUpdateList[nevra]).SetUpdatesInstallability(APPLICABLE) diff --git a/manager/config/config.go b/manager/config/config.go index 4d2ba424d..fb1fe4e39 100644 --- a/manager/config/config.go +++ b/manager/config/config.go @@ -1,7 +1,11 @@ package config import ( + "app/base/api" "app/base/utils" + "crypto/tls" + "crypto/x509" + "net/http" log "github.com/sirupsen/logrus" ) @@ -25,6 +29,8 @@ var ( // Send recalc message for systems which have been assigned to a different baseline EnableBaselineChangeEval = utils.PodConfig.GetBool("baseline_change_eval", true) + // Send recalc message for systems which have been assigned to a different template + EnableTemplateChangeEval = utils.PodConfig.GetBool("template_change_eval", true) // Honor rbac permissions (can be disabled for tests) EnableRBACCHeck = utils.PodConfig.GetBool("rbac", true) @@ -33,5 +39,48 @@ var ( // Expose templates API (feature flag) EnableTemplates = utils.PodConfig.GetBool("templates_api", true) + // Toggle compression when calling Candlepi API + CandlepinCallCmp = utils.PodConfig.GetBool("candlepin_call_compression", true) + // Number of retries on Candlepin API + CandlepinRetries = utils.PodConfig.GetInt("candlepin_retries", 5) + // Toggle exponential retries on Candlepin API + CandlepinExpRetries = utils.PodConfig.GetBool("candlepin_exp_retries", true) + // Debug flag for API calls DebugRequest = log.IsLevelEnabled(log.TraceLevel) ) + +func CreateCandlepinClient() api.Client { + getTLSConfig := func() (*tls.Config, error) { + var tlsConfig *tls.Config + if utils.CoreCfg.CandlepinCert != "" && utils.CoreCfg.CandlepinKey != "" { + clientCert, err := tls.X509KeyPair([]byte(utils.CoreCfg.CandlepinCert), []byte(utils.CoreCfg.CandlepinKey)) + if err != nil { + return nil, err + } + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + } + utils.LogInfo("using cert to access candlepin") + } + return tlsConfig, nil + } + + tlsConfig, err := getTLSConfig() + if err != nil { + utils.LogError("err", err, "parsing candlepin cert") + } + + return api.Client{ + HTTPClient: &http.Client{Transport: &http.Transport{ + DisableCompression: !CandlepinCallCmp, + TLSClientConfig: tlsConfig, + }}, + Debug: DebugRequest, + } +} diff --git a/manager/controllers/template_systems.go b/manager/controllers/template_systems.go index a388a8971..557e24a6d 100644 --- a/manager/controllers/template_systems.go +++ b/manager/controllers/template_systems.go @@ -60,8 +60,7 @@ func getTemplate(c *gin.Context, tx *gorm.DB, account int, uuid string) (*models LogAndRespNotFound(c, err, err.Error()) return &template, err } - err := tx.Model(&models.Template{}). - Where("rh_account_id = ? AND uuid = ?::uuid ", account, uuid). + err := tx.Where("rh_account_id = ? AND uuid = ?::uuid ", account, uuid). // use Find() not First() otherwise it returns error "no rows found" if uuid is not present Find(&template).Error if err != nil { diff --git a/manager/controllers/template_systems_delete.go b/manager/controllers/template_systems_delete.go index 9dd0a4989..0770eef22 100644 --- a/manager/controllers/template_systems_delete.go +++ b/manager/controllers/template_systems_delete.go @@ -2,6 +2,8 @@ package controllers import ( "app/base/utils" + "app/manager/config" + "app/manager/kafka" "app/manager/middlewares" "net/http" @@ -38,9 +40,15 @@ func TemplateSystemsDeleteHandler(c *gin.Context) { return } - // TODO: re-evaluate systems removed from templates - // inventoryAIDs := kafka.InventoryIDs2InventoryAIDs(account, req.Systems) - // kafka.EvaluateBaselineSystems(inventoryAIDs) + err = assignCandlepinEnvironment(c, db, account, nil, req.Systems, groups) + if err != nil { + return + } + // re-evaluate systems removed from templates + if config.EnableTemplateChangeEval { + inventoryAIDs := kafka.InventoryIDs2InventoryAIDs(account, req.Systems) + kafka.EvaluateBaselineSystems(inventoryAIDs) + } c.Status(http.StatusOK) } diff --git a/manager/controllers/template_systems_update.go b/manager/controllers/template_systems_update.go index 0f5408924..bea2c3350 100644 --- a/manager/controllers/template_systems_update.go +++ b/manager/controllers/template_systems_update.go @@ -2,10 +2,14 @@ package controllers import ( "app/base" + "app/base/candlepin" "app/base/database" "app/base/models" "app/base/utils" + "app/manager/config" + "app/manager/kafka" "app/manager/middlewares" + "context" "fmt" "net/http" @@ -17,6 +21,9 @@ import ( "github.com/gin-gonic/gin" ) +var errCandlepin = errors.New("candlepin error") +var candlepinClient = config.CreateCandlepinClient() + type TemplateSystemsUpdateRequest struct { // List of inventory IDs to have templates removed Systems []string `json:"systems" example:"system1-uuid, system2-uuid, ..."` @@ -58,10 +65,16 @@ func TemplateSystemsUpdateHandler(c *gin.Context) { return } - // TODO: re-evaluate systems added/removed from templates - // inventoryAIDs := kafka.InventoryIDs2InventoryAIDs(account, req.InventoryIDs) - // kafka.EvaluateBaselineSystems(inventoryAIDs) + err = assignCandlepinEnvironment(c, db, account, &template.EnvironmentID, req.Systems, groups) + if err != nil { + return + } + // re-evaluate systems added/removed from templates + if config.EnableTemplateChangeEval { + inventoryAIDs := kafka.InventoryIDs2InventoryAIDs(account, req.Systems) + kafka.EvaluateBaselineSystems(inventoryAIDs) + } c.Status(http.StatusOK) } @@ -155,3 +168,63 @@ func templateArchVersionMatch( } return err } + +func callCandlepin(ctx context.Context, consumer string, request *candlepin.ConsumersUpdateRequest) ( + *candlepin.ConsumersUpdateResponse, error) { + candlepinEnvConsumersURL := utils.CoreCfg.CandlepinAddress + "/consumers/" + consumer + candlepinFunc := func() (interface{}, *http.Response, error) { + utils.LogTrace("request", *request, "candlepin /consumers request") + candlepinResp := candlepin.ConsumersUpdateResponse{} + resp, err := candlepinClient.Request(&ctx, http.MethodPut, candlepinEnvConsumersURL, request, &candlepinResp) + statusCode := utils.TryGetStatusCode(resp) + utils.LogDebug("status_code", statusCode, "candlepin /consumers call") + utils.LogTrace("response", resp, "candlepin /consumers response") + if err != nil && statusCode == 400 { + err = errors.Wrap(errCandlepin, err.Error()) + } + return &candlepinResp, resp, err + } + + candlepinRespPtr, err := utils.HTTPCallRetry(base.Context, candlepinFunc, config.CandlepinExpRetries, + config.CandlepinRetries, http.StatusServiceUnavailable) + if err != nil { + return nil, errors.Wrap(err, "candlepin /consumers call failed") + } + return candlepinRespPtr.(*candlepin.ConsumersUpdateResponse), nil +} + +func assignCandlepinEnvironment(c context.Context, db *gorm.DB, accountID int, env *string, inventoryIDs []string, + groups map[string]string) error { + var consumers = []struct { + InventoryID string + Consumer *string + }{} + + err := database.Systems(db, accountID, groups). + Select("ih.id as inventory_id, ih.system_profile->>'owner_id' as consumer"). + Where("ih.id in (?)", inventoryIDs).Find(&consumers).Error + if err != nil { + return err + } + + environments := []candlepin.ConsumersUpdateEnvironment{} + if env != nil { + environments = []candlepin.ConsumersUpdateEnvironment{{ID: *env}} + } + updateReq := candlepin.ConsumersUpdateRequest{ + Environments: environments, + } + for _, consumer := range consumers { + if consumer.Consumer == nil { + err = errors2.Join(err, errors.Errorf("Missing owner_id for '%s'", consumer.InventoryID)) + continue + } + resp, apiErr := callCandlepin(c, *consumer.Consumer, &updateReq) + // check response + if apiErr != nil { + err = errors2.Join(err, apiErr, errors.New(resp.Message)) + } + } + + return err +} diff --git a/manager/middlewares/rbac.go b/manager/middlewares/rbac.go index a7bde75aa..697b0bab5 100644 --- a/manager/middlewares/rbac.go +++ b/manager/middlewares/rbac.go @@ -32,6 +32,8 @@ var granularPerms = map[string]string{ "BaselineUpdateHandler": "patch:template:write", "BaselineDeleteHandler": "patch:template:write", "BaselineSystemsRemoveHandler": "patch:template:write", + "TemplateSystemsUpdateHandler": "content-sources:templates:write", + "TemplateSystemsDeleteHandler": "content-sources:templates:write", "SystemDeleteHandler": "patch:system:write", } @@ -44,7 +46,7 @@ func makeClient(identity string) *api.Client { } if rbacURL == "" { rbacURL = utils.FailIfEmpty(utils.CoreCfg.RbacAddress, "RBAC_ADDRESS") + base.RBACApiPrefix + - "/access/?application=patch,inventory" + "/access/?application=patch,inventory,content-sources" } return &client } diff --git a/platform/candlepin.go b/platform/candlepin.go new file mode 100644 index 000000000..b941dd911 --- /dev/null +++ b/platform/candlepin.go @@ -0,0 +1,39 @@ +package platform + +import ( + "app/base/utils" + "fmt" + "io" + "net/http" + + "github.com/gin-gonic/gin" +) + +func candlepinEnvHandler(c *gin.Context) { + envID := c.Param("envid") + /* + jsonData, _ := io.ReadAll(c.Request.Body) + json.Unmarshal(jsonData, &body) // nolint:errcheck + if body.ReturnStatus > 200 { + c.AbortWithStatus(body.ReturnStatus) + return + } + */ + data := fmt.Sprintf(`{ + "environment": "%s" + }`, envID) + utils.LogInfo(data) + c.Data(http.StatusOK, gin.MIMEJSON, []byte(data)) +} + +func candlepinConsumersHandler(c *gin.Context) { + consumer := c.Param("consumer") + jsonData, _ := io.ReadAll(c.Request.Body) + utils.LogInfo("consumer", consumer, "body", string(jsonData)) + c.Data(http.StatusOK, gin.MIMEJSON, []byte{}) +} + +func initCandlepin(app *gin.Engine) { + app.POST("/candlepin/environments/:envid/consumers", candlepinEnvHandler) + app.PUT("/candlepin/consumers/:consumer", candlepinConsumersHandler) +} diff --git a/platform/platform.go b/platform/platform.go index b351e8fb1..9cb57a695 100644 --- a/platform/platform.go +++ b/platform/platform.go @@ -184,6 +184,7 @@ func platformMock() { app.Use(middlewares.RequestResponseLogger()) initVMaaS(app) initRbac(app) + initCandlepin(app) // Control endpoint handler app.POST("/control/upload", mockUploadHandler)