Issue 3715 - agbot update node-level pattern secrets

agreementbot/governance.go

@@ -67,11 +67,35 @@ func (w *AgreementBotWorker) GovernAgreements() int {
 				getBusinessPolicies := exchange.GetHTTPBusinessPoliciesHandler(w)
 				if exchPolsMetadata, err = getBusinessPolicies(org, ""); err != nil {
 					glog.Errorf("unable to get business polices for org %v, error %v", org, err)
+					continue
 				}
 			}
-			err = w.secretUpdateManager.UpdateNodeSecrets(org, exchPolsMetadata, w.secretProvider, w.db, agp)
+			err = w.secretUpdateManager.UpdateNodePolicySecrets(org, exchPolsMetadata, w.secretProvider, w.db, agp)
 			if err != nil {
-				glog.Errorf("error updating node secrets %v", err)
+				glog.Errorf("error updating node policy secrets %v", err)
+			}
+		}
+
+		// Iterate over each org in the PatternManager and process all the patterns in that org
+		for _, org := range patternManager.GetAllPatternOrgs() {
+			var exchangePatternMetadata map[string]exchange.Pattern
+			var err error
+
+			// check if the org exists on the exchange or not
+			if _, err = exchange.GetOrganization(w.Config.Collaborators.HTTPClientFactory, org, w.GetExchangeURL(), w.GetExchangeId(), w.GetExchangeToken()); err != nil {
+				// org does not exist is returned as an error
+				glog.V(5).Infof(AWlogString(fmt.Sprintf("unable to get organization %v: %v", org, err)))
+				exchangePatternMetadata = make(map[string]exchange.Pattern)
+			} else {
+				// Query exchange for all patterns in the org
+				if exchangePatternMetadata, err = exchange.GetPatterns(w.Config.Collaborators.HTTPClientFactory, org, "", w.GetExchangeURL(), w.GetExchangeId(), w.GetExchangeToken()); err != nil {
+					glog.Errorf("unable to get patterns for org %v, error %v", org, err)
+					continue
+				}
+			}
+			err = w.secretUpdateManager.UpdateNodePatternSecrets(org, exchangePatternMetadata, w.secretProvider, w.db, agp)
+			if err != nil {
+				glog.Errorf("error updating node pattern secrets %v", err)
 			}
 		}
 
@@ -143,7 +167,7 @@ func (w *AgreementBotWorker) GovernAgreements() int {
 					var updatedSecrets []string
 					var newestUpdateTime uint64
 					if ag.Pattern != "" {
-						newestUpdateTime, updatedSecrets = secretUpdates.GetUpdatedSecretsForPattern(ag.Pattern, ag.LastSecretUpdateTime)
+						newestUpdateTime, updatedSecrets = secretUpdates.GetUpdatedSecretsForPattern(ag.Pattern, ag.DeviceId, ag.LastSecretUpdateTime)
 					} else {
 						newestUpdateTime, updatedSecrets = secretUpdates.GetUpdatedSecretsForPolicy(ag.PolicyName, ag.DeviceId, ag.LastSecretUpdateTime)
 					}

agreementbot/persistence/agreement.go

@@ -533,6 +533,10 @@ func PolAFilter(policyName string) AFilter {
         return func(a Agreement) bool { return a.PolicyName == policyName }
 }
 
+func PatAFilter(patternName string) AFilter {
+        return func(a Agreement) bool { return a.Pattern == patternName }
+}
+
 func RunFilters(ag *Agreement, filters []AFilter) *Agreement {
 	for _, filterFn := range filters {
 		if !filterFn(*ag) {

agreementbot/secret_updater.go

@@ -133,7 +133,7 @@ func (sm *SecretUpdateManager) CheckForUpdates(secretProvider secrets.AgbotSecre
 }
 
 // after each nodescan update the list of node secret managed by the agbot
-func (sm *SecretUpdateManager) UpdateNodeSecrets(org string, exchPolsMetadata map[string]exchange.ExchangeBusinessPolicy, secretProvider secrets.AgbotSecrets, db persistence.AgbotDatabase, agProtocol string) error {
+func (sm *SecretUpdateManager) UpdateNodePolicySecrets(org string, exchPolsMetadata map[string]exchange.ExchangeBusinessPolicy, secretProvider secrets.AgbotSecrets, db persistence.AgbotDatabase, agProtocol string) error {
 
        for policyName, dpol := range exchPolsMetadata {
 
@@ -218,6 +218,93 @@ func (sm *SecretUpdateManager) UpdateNodeSecrets(org string, exchPolsMetadata ma
 	return nil
 }
 
+// periodically called to update the list of node secret managed by the agbot
+func (sm *SecretUpdateManager) UpdateNodePatternSecrets(org string, exchPatsMetadata map[string]exchange.Pattern, secretProvider secrets.AgbotSecrets, db persistence.AgbotDatabase, agProtocol string) error {
+
+       for patternName, dpol := range exchPatsMetadata {
+
+                // Get the list of managed secrets for this pattern from the DB.
+                secretNames, err := db.GetManagedPatternSecretNames(org, exchange.GetId(patternName))
+                if err != nil {
+                        glog.Errorf(smlogString(fmt.Sprintf("Error retrieving managed secret list for %s, error: %v", patternName, err)))
+                        continue
+                }
+
+                // Keep track of which secrets are being referenced so that unused secrets can be removed at the end.
+                referencedSecrets := make(map[string]bool)
+
+		// Iterate the list of secret bindings in the policy to get the secret manager secret names.
+                for _, sb := range dpol.SecretBinding {
+                        // Iterate each bound secret
+                        for _, bs := range sb.Secrets {
+                                // Extract the secret manager secret name
+                                _, secretFullName := bs.GetBinding()
+                                referencedSecrets[fmt.Sprintf("%s/%s", org, secretFullName)] = true
+
+				if !sb.EnableNodeLevelSecrets {
+                                	continue
+                        	}
+
+                                secretUser, _, secretName, err := compcheck.ParseVaultSecretName(secretFullName, nil)
+                                if err != nil {
+                                        glog.Errorf(smlogString(fmt.Sprintf("unable to parse secret name %s, error: %v", secretFullName, err)))
+                                        continue
+                                }
+
+                                pName := exchange.GetId(patternName)
+
+                                // Use now as the last update time for secrets that dont exist yet.
+                                secretLastUpdateTime := time.Now().Unix()
+
+				agList, err := db.FindAgreements([]persistence.AFilter{persistence.PatAFilter(patternName), persistence.UnarchivedAFilter()}, agProtocol)
+				for _, ag := range agList {
+					secretNode := ag.DeviceId
+
+					if secretUser != "" {
+						referencedSecrets[fmt.Sprintf("%s/user/%s/node/%s/%s", org, secretUser, secretNode, secretName)] = true
+					} else {
+						referencedSecrets[fmt.Sprintf("%s/node/%s/%s", org, secretNode, secretName)] = true
+					}
+
+                                	// Get the secret's metadata, if it exists
+                                	sm, err := secretProvider.GetSecretMetadata(org, secretUser, secretNode, secretName)
+                                	if err != nil {
+                                        	// The secret should be stored in the table even if it doesnt exist, so that if it is created later
+                                        	// changes to it will be recognized.
+                                        	glog.Warningf(smlogString(fmt.Sprintf("unable to retrieve metadata for %s %s, error: %v", org, secretFullName, err)))
+                                        	//continue
+                                	} else {
+                                        	secretLastUpdateTime = sm.UpdateTime
+                                	}
+
+                                	glog.V(5).Infof(smlogString(fmt.Sprintf("storing managed secret %v %v from %v/%v", org, secretFullName, org, pName)))
+
+                                	// Only secrets that have never been referenced before are added to the DB. DB rows that already exist will not be updated.
+                                	err = db.AddManagedPatternSecret(org, secretFullName, org, pName, secretLastUpdateTime)
+                                	if err != nil {
+                                        	glog.Errorf(smlogString(fmt.Sprintf("unable to persist secret %v %v from %v/%v, error: %v", org, secretFullName, org, pName, err)))
+                                	}
+				}
+                        }
+                }
+
+                // Look for unreferenced secrets and remove them.
+                for _, secretName := range secretNames {
+                        if _, ok := referencedSecrets[secretName]; !ok {
+
+                                glog.V(5).Infof(smlogString(fmt.Sprintf("deleting managed secret %s from %s because it is no longer used", secretName, patternName)))
+                                err = db.DeletePatternSecret(exchange.GetOrg(secretName), exchange.GetId(secretName), org, exchange.GetId(patternName))
+                                if err != nil {
+                                        glog.Errorf(smlogString(fmt.Sprintf("unable to delete %s from secrets DB, error: %v", secretName, err)))
+                                }
+                        }
+                }
+
+	}
+	return nil
+}
+
+
 // When policies are added, changed or deleted, the list of managed secrets in the DB might need to be updated.
 func (sm *SecretUpdateManager) UpdatePolicies(org string, exchPolsMetadata map[string]exchange.ExchangeBusinessPolicy, secretProvider secrets.AgbotSecrets, db persistence.AgbotDatabase) error {
 

events/secret_events.go

@@ -79,7 +79,7 @@ func (sus *SecretUpdates) GetUpdatedSecretsForPolicy(policyName string, nodeName
 
 }
 
-func (sus *SecretUpdates) GetUpdatedSecretsForPattern(patternName string, lastUpdateTime uint64) (uint64, []string) {
+func (sus *SecretUpdates) GetUpdatedSecretsForPattern(patternName string, nodeName string, lastUpdateTime uint64) (uint64, []string) {
 
 	res := make([]string, 0)
 	newestUpdate := uint64(0)
@@ -89,7 +89,7 @@ func (sus *SecretUpdates) GetUpdatedSecretsForPattern(patternName string, lastUp
 	}
 
 	for _, su := range sus.Updates {
-		if cutil.SliceContains(su.PatternNames, patternName) && uint64(su.SecretUpdateTime) > lastUpdateTime {
+		if cutil.SliceContains(su.PatternNames, patternName) && uint64(su.SecretUpdateTime) > lastUpdateTime && (su.NodeName == "" || nodeName == su.NodeName) {
 			if uint64(su.SecretUpdateTime) > newestUpdate {
 				newestUpdate = uint64(su.SecretUpdateTime)
 			}