From 9ae53df929fa434d40e9905f9ff9a30b7feceb19 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 19 Nov 2024 19:03:52 +0100 Subject: [PATCH 01/11] simple attribute anonymizer --- .../common/RoleMiningExportUtils.java | 29 ++++- .../mining/ExportMiningConsumerWorker.java | 106 +++++++++++++++++- 2 files changed, 128 insertions(+), 7 deletions(-) diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java index 6b3750fd6e8..206b146db01 100644 --- a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java +++ b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java @@ -11,9 +11,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; -import java.util.Base64; -import java.util.List; -import java.util.UUID; +import java.util.*; import javax.crypto.Cipher; import javax.crypto.spec.SecretKeySpec; @@ -62,6 +60,23 @@ public String getDisplayString() { } } + public static class SequentialAnonymizer { + private final Map anonymizedValues = new HashMap<>(); + private final String baseName; + private int index = 0; + + public SequentialAnonymizer(String baseName) { + this.baseName = baseName; + } + + public String anonymize(String value) { + if (!anonymizedValues.containsKey(value)) { + anonymizedValues.put(value, baseName + index++); + } + return anonymizedValues.get(value); + } + } + private static PolyStringType encryptName(String name, int iterator, String prefix, @NotNull NameMode nameMode, String key) { if (nameMode.equals(NameMode.ENCRYPTED)) { return PolyStringType.fromOrig(encrypt(name, key) + EXPORT_SUFFIX); @@ -85,6 +100,12 @@ public static PolyStringType encryptRoleName(String name, int iterator, NameMode return encryptName(name, iterator, "Role", nameMode, key); } + public static ObjectReferenceType encryptObjectReference(ObjectReferenceType targetRef, SecurityMode securityMode, String key) { + ObjectReferenceType encryptedTargetRef = targetRef.clone(); + encryptedTargetRef.setOid(encryptedUUID(encryptedTargetRef.getOid(), securityMode, key)); + return encryptedTargetRef; + } + public static AssignmentType encryptObjectReference(@NotNull AssignmentType assignmentObject, SecurityMode securityMode, String key) { ObjectReferenceType encryptedTargetRef = assignmentObject.getTargetRef(); @@ -131,7 +152,7 @@ else if (key == null) { } } - private static String encrypt(String value, String key) { + public static String encrypt(String value, String key) { if (value == null) { return null; diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index cf6494221c3..89990bdb0e7 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -11,10 +11,17 @@ import java.io.IOException; import java.io.Writer; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.BlockingQueue; import javax.xml.namespace.QName; +import com.evolveum.midpoint.common.mining.utils.RoleAnalysisAttributeDefUtils; +import com.evolveum.midpoint.prism.*; +import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; @@ -28,9 +35,6 @@ import com.evolveum.midpoint.ninja.util.FileReference; import com.evolveum.midpoint.ninja.util.NinjaUtils; import com.evolveum.midpoint.ninja.util.OperationStatus; -import com.evolveum.midpoint.prism.PrismContainerValue; -import com.evolveum.midpoint.prism.PrismSerializer; -import com.evolveum.midpoint.prism.SerializationOptions; import com.evolveum.midpoint.prism.query.ObjectFilter; import com.evolveum.midpoint.prism.query.ObjectQuery; import com.evolveum.midpoint.schema.result.OperationResult; @@ -49,6 +53,8 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker businessRolePrefix; private List businessRoleSuffix; + + private static final Set ATTR_PATHS_USER = extractDefaultAttributePaths(UserType.COMPLEX_TYPE); + private static final Set ATTR_PATHS_ROLE = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE); + private static final Set ATTR_PATHS_ORG = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE); + public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue queue, OperationStatus operation) { super(context, options, queue, operation); @@ -142,6 +153,9 @@ private void write(Writer writer, PrismContainerValue prismContainerValue) th @NotNull private OrgType getPreparedOrgObject(@NotNull FocusType object) { OrgType org = new OrgType(); + + fillAttributes(object, org, ATTR_PATHS_ORG); + org.setName(encryptOrgName(object.getName().toString(), processedOrgIterator++, nameMode, encryptKey)); org.setOid(encryptedUUID(object.getOid(), securityMode, encryptKey)); @@ -172,6 +186,7 @@ && filterAllowedOrg(oid)) { @NotNull private UserType getPreparedUserObject(@NotNull FocusType object) { UserType user = new UserType(); + fillAttributes(object, user, ATTR_PATHS_USER); List assignment = object.getAssignment(); if (assignment == null || assignment.isEmpty()) { @@ -212,6 +227,8 @@ && filterAllowedRole(oid)) { @NotNull private RoleType getPreparedRoleObject(@NotNull FocusType object) { RoleType role = new RoleType(); + fillAttributes(object, role, ATTR_PATHS_ROLE); + String roleName = object.getName().toString(); PolyStringType encryptedName = encryptRoleName(roleName, processedRoleIterator++, nameMode, encryptKey); role.setName(encryptedName); @@ -342,4 +359,87 @@ private void loadRoleCategoryIdentifiers() { return null; } + + private static Set extractDefaultAttributePaths(QName type) { + var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(OrgType.COMPLEX_TYPE); + return extractAttributePaths(def); + } + + + private static Set extractAttributePaths(PrismContainerDefinition containerDef) { + Set paths = new HashSet<>(); + for (ItemDefinition def : containerDef.getDefinitions()) { + if (def.isOperational() || !def.isSingleValue()) { + // unsupported types + continue; + } + if (def instanceof PrismReferenceDefinition refDef) { + paths.add(refDef.getItemName()); + } + if (def instanceof PrismPropertyDefinition propertyDef && RoleAnalysisAttributeDefUtils.isSupportedPropertyType(propertyDef.getTypeClass())) { + paths.add(propertyDef.getItemName()); + } + } + return paths; + } + + private Object anonymizeAttributeValue(Item item) { + var def = item.getDefinition(); + var type = def.getTypeClass(); + var realValue = Objects.requireNonNull(item.getRealValue()); + + if (def instanceof PrismReferenceDefinition) { + var referenceValue = (ObjectReferenceType) realValue; + return encryptObjectReference(referenceValue, securityMode, encryptKey); + } + + // NOTE: do not encrypt ordinal values and booleans + var shouldAnonymizeValue = !List.of(Integer.class, Double.class, Boolean.class).contains(type); + + if (shouldAnonymizeValue) { + return encrypt(realValue.toString(), encryptKey); + } + return realValue; + } + + private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, ItemPath path, SequentialAnonymizer attributeNameAnonymizer) { + Item item = itemContainer.findItem(path); + if (item == null || item.getRealValue() == null) { + return; + } + try { + String originalAtributeName = item.getDefinition().getItemName().toString(); + String anonymizedAttributeName = attributeNameAnonymizer.anonymize(originalAtributeName); + Object anonymizedAttributeValue = anonymizeAttributeValue(item); + + QName propertyName = new QName(item.getDefinition().getItemName().getNamespaceURI(), anonymizedAttributeName); + PrismPropertyDefinition propertyDefinition = context + .getPrismContext() + .definitionFactory() + .newPropertyDefinition(propertyName, DOMUtil.XSD_STRING); + PrismProperty anonymizedProperty = propertyDefinition.instantiate(); + anonymizedProperty.setRealValue(anonymizedAttributeValue); + newObject.asPrismObject().addExtensionItem(anonymizedProperty); + } catch (Exception e) { + LOGGER.error("Failed to clone item for {} object {}. ", itemContainer.getClass(), item, e); + } + } + + private void fillAttributes(@NotNull FocusType origObject, @NotNull FocusType newObject, @NotNull Set defaultAttributePaths) { + var origContainer = origObject.asPrismObject(); + newObject.extension(new ExtensionType()); + + for (var path: defaultAttributePaths) { + anonymizeAttribute(newObject, origContainer, path, defaultAttributeNameAnonymizer); + } + + if (origContainer.getExtension() != null) { + PrismContainerDefinition definition = origContainer.getExtensionContainerValue().getDefinition(); + var extensionAttributePaths = extractAttributePaths(definition); + for (var path: extensionAttributePaths) { + anonymizeAttribute(newObject, origContainer.getExtension(), path, extensionAttributeNameAnonymizer); + } + } + } + } From 68ddf606164180a7100fefa7d2a30abaca96eefc Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Wed, 20 Nov 2024 12:54:05 +0100 Subject: [PATCH 02/11] activation export --- .../action/mining/ExportMiningConsumerWorker.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 89990bdb0e7..f3039c9810f 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -155,6 +155,7 @@ private OrgType getPreparedOrgObject(@NotNull FocusType object) { OrgType org = new OrgType(); fillAttributes(object, org, ATTR_PATHS_ORG); + fillActivation(object, org); org.setName(encryptOrgName(object.getName().toString(), processedOrgIterator++, nameMode, encryptKey)); org.setOid(encryptedUUID(object.getOid(), securityMode, encryptKey)); @@ -187,6 +188,7 @@ && filterAllowedOrg(oid)) { private UserType getPreparedUserObject(@NotNull FocusType object) { UserType user = new UserType(); fillAttributes(object, user, ATTR_PATHS_USER); + fillActivation(object, user); List assignment = object.getAssignment(); if (assignment == null || assignment.isEmpty()) { @@ -228,6 +230,7 @@ && filterAllowedRole(oid)) { private RoleType getPreparedRoleObject(@NotNull FocusType object) { RoleType role = new RoleType(); fillAttributes(object, role, ATTR_PATHS_ROLE); + fillActivation(object, role); String roleName = object.getName().toString(); PolyStringType encryptedName = encryptRoleName(roleName, processedRoleIterator++, nameMode, encryptKey); @@ -442,4 +445,12 @@ private void fillAttributes(@NotNull FocusType origObject, @NotNull FocusType ne } } + private void fillActivation(@NotNull FocusType origObject, @NotNull FocusType newObject) { + if (origObject.getActivation() == null) { + return; + } + var activation = new ActivationType().effectiveStatus(origObject.getActivation().getEffectiveStatus()); + newObject.setActivation(activation); + } + } From d6deed6f6f73dd93d20d41d7c9e8aa86865e6f0b Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Wed, 20 Nov 2024 15:54:52 +0100 Subject: [PATCH 03/11] sequential anonymization support --- .../common/RoleMiningExportUtils.java | 38 ++++++++++++++++++- .../mining/ExportMiningConsumerWorker.java | 8 +++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java index 206b146db01..afeb1c26cf7 100644 --- a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java +++ b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java @@ -77,6 +77,42 @@ public String anonymize(String value) { } } + private static class ScopedSequentialAnonymizer { + private final Map scopedAnonymizers = new HashMap<>(); + private final String baseName; + + public ScopedSequentialAnonymizer(String baseName) { + this.baseName = baseName; + } + + public String anonymize(String scope, String value) { + if (!scopedAnonymizers.containsKey(scope)) { + scopedAnonymizers.put(scope, new SequentialAnonymizer(baseName)); + } + return scopedAnonymizers.get(scope).anonymize(value); + } + } + + public static class AttributeValueAnonymizer { + private final ScopedSequentialAnonymizer sequentialAnonymizer = new ScopedSequentialAnonymizer("att"); + private final NameMode nameMode; + private final String encryptKey; + + public AttributeValueAnonymizer(NameMode nameMode, String encryptKey) { + this.nameMode = nameMode; + this.encryptKey = encryptKey; + } + + public String anonymize(String attributeName, String attributeValue) { + var anonymized = switch(nameMode) { + case ENCRYPTED -> encrypt(attributeValue, encryptKey); + case SEQUENTIAL -> sequentialAnonymizer.anonymize(attributeName, attributeValue); + case ORIGINAL -> attributeValue; + }; + return anonymized + EXPORT_SUFFIX; + } + } + private static PolyStringType encryptName(String name, int iterator, String prefix, @NotNull NameMode nameMode, String key) { if (nameMode.equals(NameMode.ENCRYPTED)) { return PolyStringType.fromOrig(encrypt(name, key) + EXPORT_SUFFIX); @@ -152,7 +188,7 @@ else if (key == null) { } } - public static String encrypt(String value, String key) { + private static String encrypt(String value, String key) { if (value == null) { return null; diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index f3039c9810f..de90bb2c7a2 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -55,6 +55,7 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker extractDefaultAttributePaths(QName type) { - var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(OrgType.COMPLEX_TYPE); + var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type); return extractAttributePaths(def); } @@ -400,7 +403,8 @@ private Object anonymizeAttributeValue(Item item) { var shouldAnonymizeValue = !List.of(Integer.class, Double.class, Boolean.class).contains(type); if (shouldAnonymizeValue) { - return encrypt(realValue.toString(), encryptKey); + var attributeName = item.getDefinition().getItemName().toString(); + return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString()); } return realValue; } From 5aef17c959d1c8a1c49205b30d022b789241aabf Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Wed, 20 Nov 2024 17:40:22 +0100 Subject: [PATCH 04/11] disable attributes option --- .../ninja/action/mining/ExportMiningConsumerWorker.java | 5 +++++ .../ninja/action/mining/ExportMiningOptions.java | 9 +++++++++ tools/ninja/src/main/resources/messages.properties | 1 + 3 files changed, 15 insertions(+) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index de90bb2c7a2..72e75d18a82 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -58,6 +58,7 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker itemConta } private void fillAttributes(@NotNull FocusType origObject, @NotNull FocusType newObject, @NotNull Set defaultAttributePaths) { + if (!attributesAllowed) { + return; + } var origContainer = origObject.asPrismObject(); newObject.extension(new ExtensionType()); diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java index fb95847fb80..239904c8ca9 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java @@ -35,6 +35,8 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor public static final String P_SUFFIX_BUSINESS_LONG = "--business-role-suffix"; public static final String P_ORG = "-do"; public static final String P_ORG_LONG = "--disable-org"; + public static final String P_ATTRIBUTE = "-da"; + public static final String P_ATTRIBUTE_LONG = "--disable-attribute"; public static final String P_NAME_OPTIONS = "-nm"; public static final String P_NAME_OPTIONS_LONG = "--name-mode"; public static final String P_ARCHETYPE_OID_APPLICATION_LONG = "--application-role-archetype-oid"; @@ -66,6 +68,9 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor @Parameter(names = { P_ORG, P_ORG_LONG }, descriptionKey = "export.prevent.org") private boolean disableOrg = false; + @Parameter(names = { P_ATTRIBUTE, P_ATTRIBUTE_LONG }, descriptionKey = "export.prevent.attribute") + private boolean disableAttribute = false; + @Parameter(names = { P_NAME_OPTIONS, P_NAME_OPTIONS_LONG }, descriptionKey = "export.name.options") private RoleMiningExportUtils.NameMode nameMode = RoleMiningExportUtils.NameMode.SEQUENTIAL; @@ -85,6 +90,10 @@ public boolean isIncludeOrg() { return !disableOrg; } + public boolean isIncludeAttributes() { + return !disableAttribute; + } + public String getApplicationRoleArchetypeOid() { return applicationRoleArchetypeOid; } diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties index 738016ca337..11e8b9a7f06 100644 --- a/tools/ninja/src/main/resources/messages.properties +++ b/tools/ninja/src/main/resources/messages.properties @@ -49,6 +49,7 @@ export.application.role.suffix=Suffix for identifying exported application roles export.business.role.prefix=Prefix for identifying exported business roles. Multiple prefixes can be specified using a comma "," as a delimiter. export.business.role.suffix=Suffix for identifying exported business roles. Multiple suffixes can be specified using a comma "," as a delimiter. export.prevent.org=Prevent the export of organizational structures. +export.prevent.attribute=Prevent the export of attributes export.name.options=Defines the format of the name parameter in the export. export.business.role.archetype.oid=Detects a business role based on a specific archetype, provided by its OID. export.application.role.archetype.oid=Detects an application role based on a specific archetype, provided by its OID. From 10295ffd5f1ea9bf167f0ca1f9ccae82d3d488a5 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Wed, 20 Nov 2024 17:45:52 +0100 Subject: [PATCH 05/11] anonymize archetype ref --- .../ninja/action/mining/ExportMiningConsumerWorker.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 72e75d18a82..4e81ad89699 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -80,6 +80,7 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker ATTR_PATHS_USER = extractDefaultAttributePaths(UserType.COMPLEX_TYPE); private static final Set ATTR_PATHS_ROLE = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE); private static final Set ATTR_PATHS_ORG = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE); + private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef"; public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue queue, OperationStatus operation) { @@ -377,7 +378,8 @@ private static Set extractDefaultAttributePaths(QName type) { private static Set extractAttributePaths(PrismContainerDefinition containerDef) { Set paths = new HashSet<>(); for (ItemDefinition def : containerDef.getDefinitions()) { - if (def.isOperational() || !def.isSingleValue()) { + var isArchetypeRef = def.getItemName().toString().equals(ARCHETYPE_REF_ATTRIBUTE_NAME); + if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { // unsupported types continue; } From a49fcdb543947f3f392889f6ca52cea3ecf854a3 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Thu, 21 Nov 2024 14:11:01 +0100 Subject: [PATCH 06/11] exclude attributes option --- .../mining/ExportMiningConsumerWorker.java | 54 ++++++++++++++----- .../action/mining/ExportMiningOptions.java | 33 ++++++++++++ .../src/main/resources/messages.properties | 3 ++ 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 4e81ad89699..0c1ecb004f4 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -16,6 +16,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.stream.Stream; import javax.xml.namespace.QName; import com.evolveum.midpoint.common.mining.utils.RoleAnalysisAttributeDefUtils; @@ -77,10 +78,14 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker businessRoleSuffix; - private static final Set ATTR_PATHS_USER = extractDefaultAttributePaths(UserType.COMPLEX_TYPE); - private static final Set ATTR_PATHS_ROLE = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE); - private static final Set ATTR_PATHS_ORG = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE); + private Set attrPathsUser; + private Set attrPathsRole; + private Set attrPathsOrg; + private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef"; + private static final List DEFAULT_EXCLUDED_ATTRIBUTES = List.of("description", "documentation", "emailAddress", + "telephoneNumber", "fullName", "givenName", "familyName", "additionalName", "nickName", "personalNumber", "identifier", + "jpegPhoto"); public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue queue, OperationStatus operation) { @@ -99,6 +104,10 @@ protected void init() { nameMode = options.getNameMode(); + attrPathsUser = extractDefaultAttributePaths(UserType.COMPLEX_TYPE, options.getExcludedAttributesUser()); + attrPathsRole = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE, options.getExcludedAttributesUser()); + attrPathsOrg = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE, options.getExcludedAttributesOrg()); + attributeValuesAnonymizer = new AttributeValueAnonymizer(nameMode, encryptKey); SerializationOptions serializationOptions = SerializationOptions.createSerializeForExport() @@ -160,7 +169,7 @@ private void write(Writer writer, PrismContainerValue prismContainerValue) th private OrgType getPreparedOrgObject(@NotNull FocusType object) { OrgType org = new OrgType(); - fillAttributes(object, org, ATTR_PATHS_ORG); + fillAttributes(object, org, attrPathsOrg, options.getExcludedAttributesOrg()); fillActivation(object, org); org.setName(encryptOrgName(object.getName().toString(), processedOrgIterator++, nameMode, encryptKey)); @@ -193,7 +202,7 @@ && filterAllowedOrg(oid)) { @NotNull private UserType getPreparedUserObject(@NotNull FocusType object) { UserType user = new UserType(); - fillAttributes(object, user, ATTR_PATHS_USER); + fillAttributes(object, user, attrPathsUser, options.getExcludedAttributesUser()); fillActivation(object, user); List assignment = object.getAssignment(); @@ -235,7 +244,7 @@ && filterAllowedRole(oid)) { @NotNull private RoleType getPreparedRoleObject(@NotNull FocusType object) { RoleType role = new RoleType(); - fillAttributes(object, role, ATTR_PATHS_ROLE); + fillAttributes(object, role, attrPathsRole, options.getExcludedAttributesRole()); fillActivation(object, role); String roleName = object.getName().toString(); @@ -369,16 +378,21 @@ private void loadRoleCategoryIdentifiers() { return null; } - private static Set extractDefaultAttributePaths(QName type) { + private Set extractDefaultAttributePaths(QName type, List excludedDefaultAttributes) { var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type); - return extractAttributePaths(def); + var excludedAttributes = Stream.concat(excludedDefaultAttributes.stream(), DEFAULT_EXCLUDED_ATTRIBUTES.stream()).toList(); + return extractAttributePaths(def, excludedAttributes); } - private static Set extractAttributePaths(PrismContainerDefinition containerDef) { + private Set extractAttributePaths(PrismContainerDefinition containerDef, List excludedAttributeNames) { Set paths = new HashSet<>(); for (ItemDefinition def : containerDef.getDefinitions()) { - var isArchetypeRef = def.getItemName().toString().equals(ARCHETYPE_REF_ATTRIBUTE_NAME); + var attributeName = def.getItemName().toString(); + if (excludedAttributeNames.contains(attributeName)) { + continue; + } + var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME); if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { // unsupported types continue; @@ -413,14 +427,21 @@ private Object anonymizeAttributeValue(Item item) { return realValue; } + public String anonymizeAttributeName(Item item, SequentialAnonymizer attributeNameAnonymizer) { + String originalAtributeName = item.getDefinition().getItemName().toString(); + if (nameMode.equals(NameMode.ORIGINAL)) { + return originalAtributeName; + } + return attributeNameAnonymizer.anonymize(originalAtributeName); + } + private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, ItemPath path, SequentialAnonymizer attributeNameAnonymizer) { Item item = itemContainer.findItem(path); if (item == null || item.getRealValue() == null) { return; } try { - String originalAtributeName = item.getDefinition().getItemName().toString(); - String anonymizedAttributeName = attributeNameAnonymizer.anonymize(originalAtributeName); + String anonymizedAttributeName = anonymizeAttributeName(item, attributeNameAnonymizer); Object anonymizedAttributeValue = anonymizeAttributeValue(item); QName propertyName = new QName(item.getDefinition().getItemName().getNamespaceURI(), anonymizedAttributeName); @@ -436,7 +457,12 @@ private void anonymizeAttribute(FocusType newObject, PrismContainer itemConta } } - private void fillAttributes(@NotNull FocusType origObject, @NotNull FocusType newObject, @NotNull Set defaultAttributePaths) { + private void fillAttributes( + @NotNull FocusType origObject, + @NotNull FocusType newObject, + @NotNull Set defaultAttributePaths, + @NotNull List excludedAttributes + ) { if (!attributesAllowed) { return; } @@ -449,7 +475,7 @@ private void fillAttributes(@NotNull FocusType origObject, @NotNull FocusType ne if (origContainer.getExtension() != null) { PrismContainerDefinition definition = origContainer.getExtensionContainerValue().getDefinition(); - var extensionAttributePaths = extractAttributePaths(definition); + var extensionAttributePaths = extractAttributePaths(definition, excludedAttributes); for (var path: extensionAttributePaths) { anonymizeAttribute(newObject, origContainer.getExtension(), path, extensionAttributeNameAnonymizer); } diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java index 239904c8ca9..9c58fb221be 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java @@ -16,6 +16,8 @@ import com.evolveum.midpoint.common.RoleMiningExportUtils; import com.evolveum.midpoint.ninja.action.BasicExportOptions; +import com.evolveum.midpoint.ninja.util.ItemPathConverter; +import com.evolveum.midpoint.prism.path.ItemPath; @Parameters(resourceBundle = "messages", commandDescriptionKey = "exportMining") public class ExportMiningOptions extends BaseMiningOptions implements BasicExportOptions { @@ -37,6 +39,9 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor public static final String P_ORG_LONG = "--disable-org"; public static final String P_ATTRIBUTE = "-da"; public static final String P_ATTRIBUTE_LONG = "--disable-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_USER_LONG = "--exclude-user-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_ROLE_LONG = "--exclude-role-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_ORG_LONG = "--exclude-org-attribute"; public static final String P_NAME_OPTIONS = "-nm"; public static final String P_NAME_OPTIONS_LONG = "--name-mode"; public static final String P_ARCHETYPE_OID_APPLICATION_LONG = "--application-role-archetype-oid"; @@ -82,6 +87,18 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor descriptionKey = "export.business.role.archetype.oid") private String businessRoleArchetypeOid = "00000000-0000-0000-0000-000000000321"; + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_USER_LONG }, descriptionKey = "export.exclude.attributes.user", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesUser = new ArrayList<>(); + + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_ROLE_LONG }, descriptionKey = "export.exclude.attributes.role", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesRole = new ArrayList<>(); + + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_ORG_LONG }, descriptionKey = "export.exclude.attributes.org", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesOrg = new ArrayList<>(); + public RoleMiningExportUtils.SecurityMode getSecurityLevel() { return securityMode; } @@ -145,4 +162,20 @@ public List getBusinessRoleSuffix() { String[] separateSuffixes = businessRoleSuffix.split(DELIMITER); return new ArrayList<>(Arrays.asList(separateSuffixes)); } + + private List itemPathsToStrings(List itemPaths) { + return itemPaths.stream().map(ItemPath::toString).toList(); + } + + public List getExcludedAttributesUser() { + return itemPathsToStrings(excludedAttributesUser); + } + + public List getExcludedAttributesRole() { + return itemPathsToStrings(excludedAttributesRole); + } + + public List getExcludedAttributesOrg() { + return itemPathsToStrings(excludedAttributesOrg); + } } diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties index 11e8b9a7f06..9f2062df606 100644 --- a/tools/ninja/src/main/resources/messages.properties +++ b/tools/ninja/src/main/resources/messages.properties @@ -50,6 +50,9 @@ export.business.role.prefix=Prefix for identifying exported business roles. Mult export.business.role.suffix=Suffix for identifying exported business roles. Multiple suffixes can be specified using a comma "," as a delimiter. export.prevent.org=Prevent the export of organizational structures. export.prevent.attribute=Prevent the export of attributes +export.exclude.attributes.user=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.exclude.attributes.role=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.exclude.attributes.org=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. export.name.options=Defines the format of the name parameter in the export. export.business.role.archetype.oid=Detects a business role based on a specific archetype, provided by its OID. export.application.role.archetype.oid=Detects an application role based on a specific archetype, provided by its OID. From 2ba8a453fd2a8b71692acdfce71946a511994019 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 3 Dec 2024 09:35:03 +0100 Subject: [PATCH 07/11] incude attributes without definition --- .../mining/ExportMiningConsumerWorker.java | 122 +++++++++++------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 0c1ecb004f4..d9087afc3dd 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -11,22 +11,24 @@ import java.io.IOException; import java.io.Writer; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.xml.namespace.QName; import com.evolveum.midpoint.common.mining.utils.RoleAnalysisAttributeDefUtils; import com.evolveum.midpoint.prism.*; -import com.evolveum.midpoint.prism.path.ItemPath; +import com.evolveum.midpoint.prism.path.ItemName; import com.evolveum.midpoint.util.DOMUtil; import com.evolveum.midpoint.util.logging.Trace; import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.prism.xml.ns._public.types_3.RawType; + import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -78,9 +80,14 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker businessRoleSuffix; - private Set attrPathsUser; - private Set attrPathsRole; - private Set attrPathsOrg; + // NOTE: this class is used to mark attributes that we don't know the schema definition in Ninja - extension attributes defined via GUI + static class UnknownAttributeType {} + + record AttributeInfo(ItemName itemName, Class typeClass) { } + + private Set attrPathsUser; + private Set attrPathsRole; + private Set attrPathsOrg; private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef"; private static final List DEFAULT_EXCLUDED_ATTRIBUTES = List.of("description", "documentation", "emailAddress", @@ -105,7 +112,7 @@ protected void init() { nameMode = options.getNameMode(); attrPathsUser = extractDefaultAttributePaths(UserType.COMPLEX_TYPE, options.getExcludedAttributesUser()); - attrPathsRole = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE, options.getExcludedAttributesUser()); + attrPathsRole = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE, options.getExcludedAttributesRole()); attrPathsOrg = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE, options.getExcludedAttributesOrg()); attributeValuesAnonymizer = new AttributeValueAnonymizer(nameMode, encryptKey); @@ -378,73 +385,97 @@ private void loadRoleCategoryIdentifiers() { return null; } - private Set extractDefaultAttributePaths(QName type, List excludedDefaultAttributes) { - var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type); + private AttributeInfo makeAttributeInfo(ItemDefinition def, ItemName itemName, List excludedAttributeNames) { + var attributeName = itemName.toString(); + if (excludedAttributeNames.contains(attributeName)) { + return null; + } + if (def == null) { + // extension attributes from GUI schema does not contain definition in Ninja + return new AttributeInfo(itemName, UnknownAttributeType.class); + } + var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME); + if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { + // unsupported types + return null; + } + if (def instanceof PrismReferenceDefinition) { + return new AttributeInfo(itemName, PrismReferenceDefinition.class); + } + if (def instanceof PrismPropertyDefinition propertyDef && RoleAnalysisAttributeDefUtils.isSupportedPropertyType(propertyDef.getTypeClass())) { + return new AttributeInfo(itemName, propertyDef.getTypeClass()); + } + return null; + } + + private Set extractDefaultAttributePaths(QName type, List excludedDefaultAttributes) { + var containerDef = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type); var excludedAttributes = Stream.concat(excludedDefaultAttributes.stream(), DEFAULT_EXCLUDED_ATTRIBUTES.stream()).toList(); - return extractAttributePaths(def, excludedAttributes); + return containerDef.getDefinitions().stream() + .map(def -> makeAttributeInfo(def, def.getItemName(), excludedAttributes)) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); } - private Set extractAttributePaths(PrismContainerDefinition containerDef, List excludedAttributeNames) { - Set paths = new HashSet<>(); - for (ItemDefinition def : containerDef.getDefinitions()) { - var attributeName = def.getItemName().toString(); - if (excludedAttributeNames.contains(attributeName)) { - continue; - } - var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME); - if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { - // unsupported types - continue; - } - if (def instanceof PrismReferenceDefinition refDef) { - paths.add(refDef.getItemName()); - } - if (def instanceof PrismPropertyDefinition propertyDef && RoleAnalysisAttributeDefUtils.isSupportedPropertyType(propertyDef.getTypeClass())) { - paths.add(propertyDef.getItemName()); - } - } - return paths; + private Set extractExtensionAttributePaths(PrismContainerValue containerValue, List excludedAttributeNames) { + return containerValue.getItems().stream() + .map(item -> makeAttributeInfo(item.getDefinition(), item.getElementName(), excludedAttributeNames)) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); } - private Object anonymizeAttributeValue(Item item) { - var def = item.getDefinition(); - var type = def.getTypeClass(); + private Object anonymizeAttributeValue(Item item, AttributeInfo attributeInfo) { + var typeClass = attributeInfo.typeClass(); var realValue = Objects.requireNonNull(item.getRealValue()); + var attributeName = attributeInfo.itemName().toString(); - if (def instanceof PrismReferenceDefinition) { + if (PrismReferenceDefinition.class.equals(typeClass)) { + // anonymize references var referenceValue = (ObjectReferenceType) realValue; return encryptObjectReference(referenceValue, securityMode, encryptKey); } - // NOTE: do not encrypt ordinal values and booleans - var shouldAnonymizeValue = !List.of(Integer.class, Double.class, Boolean.class).contains(type); + if (typeClass.equals(UnknownAttributeType.class)) { + // anonymize attributes with unknown (extension schema from GUI) + RawType rawValue = item.getValues().get(0).getRealValue(); + try { + var value = Objects.requireNonNull(rawValue).getValue(); + return attributeValuesAnonymizer.anonymize(attributeName, value.toString()); + } catch (SchemaException e) { + throw new RuntimeException(e); + } + } + + // NOTE: do not encrypt ordinal values + var shouldAnonymizeValue = !List.of(Integer.class, Double.class).contains(typeClass); if (shouldAnonymizeValue) { - var attributeName = item.getDefinition().getItemName().toString(); + // anonymize attributes with known schema return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString()); } + return realValue; } public String anonymizeAttributeName(Item item, SequentialAnonymizer attributeNameAnonymizer) { - String originalAtributeName = item.getDefinition().getItemName().toString(); + String originalAttributeName = item.getElementName().toString(); if (nameMode.equals(NameMode.ORIGINAL)) { - return originalAtributeName; + return originalAttributeName; } - return attributeNameAnonymizer.anonymize(originalAtributeName); + return attributeNameAnonymizer.anonymize(originalAttributeName); } - private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, ItemPath path, SequentialAnonymizer attributeNameAnonymizer) { - Item item = itemContainer.findItem(path); + private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, AttributeInfo attributeInfo, SequentialAnonymizer attributeNameAnonymizer) { + Item item = itemContainer.findItem(attributeInfo.itemName()); if (item == null || item.getRealValue() == null) { return; } try { String anonymizedAttributeName = anonymizeAttributeName(item, attributeNameAnonymizer); - Object anonymizedAttributeValue = anonymizeAttributeValue(item); + Object anonymizedAttributeValue = anonymizeAttributeValue(item, attributeInfo); - QName propertyName = new QName(item.getDefinition().getItemName().getNamespaceURI(), anonymizedAttributeName); + QName propertyName = new QName(attributeInfo.itemName().getNamespaceURI(), anonymizedAttributeName); PrismPropertyDefinition propertyDefinition = context .getPrismContext() .definitionFactory() @@ -460,7 +491,7 @@ private void anonymizeAttribute(FocusType newObject, PrismContainer itemConta private void fillAttributes( @NotNull FocusType origObject, @NotNull FocusType newObject, - @NotNull Set defaultAttributePaths, + @NotNull Set defaultAttributePaths, @NotNull List excludedAttributes ) { if (!attributesAllowed) { @@ -474,8 +505,7 @@ private void fillAttributes( } if (origContainer.getExtension() != null) { - PrismContainerDefinition definition = origContainer.getExtensionContainerValue().getDefinition(); - var extensionAttributePaths = extractAttributePaths(definition, excludedAttributes); + var extensionAttributePaths = extractExtensionAttributePaths(origContainer.getExtensionContainerValue(), excludedAttributes); for (var path: extensionAttributePaths) { anonymizeAttribute(newObject, origContainer.getExtension(), path, extensionAttributeNameAnonymizer); } From c05cd1aa38b411642b68e4bd53f1d9f996713487 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 3 Dec 2024 10:02:47 +0100 Subject: [PATCH 08/11] fixed logging --- .../mining/ExportMiningConsumerWorker.java | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index d9087afc3dd..d9a413d19f3 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -23,9 +23,6 @@ import com.evolveum.midpoint.prism.*; import com.evolveum.midpoint.prism.path.ItemName; import com.evolveum.midpoint.util.DOMUtil; -import com.evolveum.midpoint.util.logging.Trace; - -import com.evolveum.midpoint.util.logging.TraceManager; import com.evolveum.prism.xml.ns._public.types_3.RawType; @@ -42,14 +39,11 @@ import com.evolveum.midpoint.prism.query.ObjectQuery; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.util.exception.SchemaException; -import com.evolveum.midpoint.util.logging.LoggingUtils; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker { - private static final Trace LOGGER = TraceManager.getTrace(ExportMiningConsumerWorker.class); - OperationResult operationResult = new OperationResult(DOT_CLASS + "searchObjectByCondition"); private PrismSerializer serializer; @@ -335,7 +329,7 @@ private boolean filterAllowedOrg(String oid) { return !context.getRepository().searchObjects(OrgType.class, objectQuery, null, operationResult).isEmpty(); } catch (SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to search organization object. ", e); + context.getLog().error("Failed to search organization object. ", e); } return false; } @@ -351,7 +345,7 @@ private boolean filterAllowedRole(String oid) { return !context.getRepository().searchObjects(RoleType.class, objectQuery, null, operationResult).isEmpty(); } catch (SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to search role object. ", e); + context.getLog().error("Failed to search role object. ", e); } return false; } @@ -361,7 +355,7 @@ private void loadFilters(FileReference roleFileReference, FileReference orgFileR this.filterRole = NinjaUtils.createObjectFilter(roleFileReference, context, RoleType.class); this.filterOrg = NinjaUtils.createObjectFilter(orgFileReference, context, OrgType.class); } catch (IOException | SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to crate object filter. ", e); + context.getLog().error("Failed to crate object filter. ", e); } } @@ -425,7 +419,7 @@ private Set extractExtensionAttributePaths(PrismContainerValue .collect(Collectors.toUnmodifiableSet()); } - private Object anonymizeAttributeValue(Item item, AttributeInfo attributeInfo) { + private Object anonymizeAttributeValue(Item item, AttributeInfo attributeInfo) throws SchemaException { var typeClass = attributeInfo.typeClass(); var realValue = Objects.requireNonNull(item.getRealValue()); var attributeName = attributeInfo.itemName().toString(); @@ -439,12 +433,8 @@ private Object anonymizeAttributeValue(Item item, AttributeInfo attributeI if (typeClass.equals(UnknownAttributeType.class)) { // anonymize attributes with unknown (extension schema from GUI) RawType rawValue = item.getValues().get(0).getRealValue(); - try { - var value = Objects.requireNonNull(rawValue).getValue(); - return attributeValuesAnonymizer.anonymize(attributeName, value.toString()); - } catch (SchemaException e) { - throw new RuntimeException(e); - } + var value = Objects.requireNonNull(rawValue).getValue(); + return attributeValuesAnonymizer.anonymize(attributeName, value.toString()); } // NOTE: do not encrypt ordinal values @@ -484,7 +474,7 @@ private void anonymizeAttribute(FocusType newObject, PrismContainer itemConta anonymizedProperty.setRealValue(anonymizedAttributeValue); newObject.asPrismObject().addExtensionItem(anonymizedProperty); } catch (Exception e) { - LOGGER.error("Failed to clone item for {} object {}. ", itemContainer.getClass(), item, e); + context.getLog().warn("Failed to anonymize attribute:\n{}\n{}. ", attributeInfo, item, e); } } From 1c4880f7220212a0c8e0644594dc7be6177d3abc Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 3 Dec 2024 10:22:52 +0100 Subject: [PATCH 09/11] note on anonymizer concurrency --- .../ninja/action/mining/ExportMiningConsumerWorker.java | 4 ++++ .../ninja/action/mining/ExportMiningRepositoryAction.java | 1 + 2 files changed, 5 insertions(+) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index d9a413d19f3..163f5df31a9 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -42,6 +42,10 @@ import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; +/** + * Anonymize and write midpoint's objects. + * - it is currently assumed to run in a single thread, therefore the state does not need to be shared and thread safe + */ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker { OperationResult operationResult = new OperationResult(DOT_CLASS + "searchObjectByCondition"); diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java index 95724c956a9..1173ac612ed 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java @@ -69,6 +69,7 @@ public Void execute() throws Exception { executor.execute(new ProgressReporterWorker<>(context, options, queue, operation)); + // NOTE: the consumer is designed to be executed in a single thread Runnable consumer = createConsumer(queue, operation); executor.execute(consumer); From f7bd80fb9097d936efad4a2351b59196aaad3757 Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 3 Dec 2024 10:46:57 +0100 Subject: [PATCH 10/11] more attribute export options --- .../midpoint/common/RoleMiningExportUtils.java | 2 +- .../mining/ExportMiningConsumerWorker.java | 17 ++++++++--------- .../action/mining/ExportMiningOptions.java | 16 ++++++++++++++++ .../src/main/resources/messages.properties | 2 ++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java index afeb1c26cf7..ccfaecca0a6 100644 --- a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java +++ b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java @@ -63,7 +63,7 @@ public String getDisplayString() { public static class SequentialAnonymizer { private final Map anonymizedValues = new HashMap<>(); private final String baseName; - private int index = 0; + private long index = 0; public SequentialAnonymizer(String baseName) { this.baseName = baseName; diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 163f5df31a9..30b04197c8b 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -441,15 +441,12 @@ private Object anonymizeAttributeValue(Item item, AttributeInfo attributeI return attributeValuesAnonymizer.anonymize(attributeName, value.toString()); } - // NOTE: do not encrypt ordinal values - var shouldAnonymizeValue = !List.of(Integer.class, Double.class).contains(typeClass); - - if (shouldAnonymizeValue) { - // anonymize attributes with known schema - return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString()); + if (!options.isAnonymizeOrdinalAttributeValues() && List.of(Integer.class, Long.class, Double.class).contains(typeClass)) { + // do not anonymize ordinal values + return realValue; } - return realValue; + return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString()); } public String anonymizeAttributeName(Item item, SequentialAnonymizer attributeNameAnonymizer) { @@ -466,10 +463,12 @@ private void anonymizeAttribute(FocusType newObject, PrismContainer itemConta return; } try { - String anonymizedAttributeName = anonymizeAttributeName(item, attributeNameAnonymizer); + String attributeName = options.isAnonymizeAttributeNames() + ? anonymizeAttributeName(item, attributeNameAnonymizer) + : item.getElementName().toString(); Object anonymizedAttributeValue = anonymizeAttributeValue(item, attributeInfo); - QName propertyName = new QName(attributeInfo.itemName().getNamespaceURI(), anonymizedAttributeName); + QName propertyName = new QName(attributeInfo.itemName().getNamespaceURI(), attributeName); PrismPropertyDefinition propertyDefinition = context .getPrismContext() .definitionFactory() diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java index 9c58fb221be..57e17deda24 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java @@ -42,6 +42,8 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor public static final String P_EXCLUDE_ATTRIBUTES_USER_LONG = "--exclude-user-attribute"; public static final String P_EXCLUDE_ATTRIBUTES_ROLE_LONG = "--exclude-role-attribute"; public static final String P_EXCLUDE_ATTRIBUTES_ORG_LONG = "--exclude-org-attribute"; + public static final String P_ANONYMIZE_ATTRIBUTE_NAMES = "--anonymize-attribute-names"; + public static final String P_ANONYMIZE_ORDINAL_ATTRIBUTE_VALUES = "--anonymize-ordinal-attribute-values"; public static final String P_NAME_OPTIONS = "-nm"; public static final String P_NAME_OPTIONS_LONG = "--name-mode"; public static final String P_ARCHETYPE_OID_APPLICATION_LONG = "--application-role-archetype-oid"; @@ -99,6 +101,12 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) private List excludedAttributesOrg = new ArrayList<>(); + @Parameter(names = { P_ANONYMIZE_ATTRIBUTE_NAMES }, descriptionKey = "export.anonymizeAttributeNames") + private Boolean anonymizeAttributeNames = false; + + @Parameter(names = { P_ANONYMIZE_ORDINAL_ATTRIBUTE_VALUES }, descriptionKey = "export.anonymizeOrdinalAttributeValues") + private Boolean anonymizeOrdinalAttributeValues = false; + public RoleMiningExportUtils.SecurityMode getSecurityLevel() { return securityMode; } @@ -178,4 +186,12 @@ public List getExcludedAttributesRole() { public List getExcludedAttributesOrg() { return itemPathsToStrings(excludedAttributesOrg); } + + public Boolean isAnonymizeAttributeNames() { + return anonymizeAttributeNames; + } + + public Boolean isAnonymizeOrdinalAttributeValues() { + return anonymizeOrdinalAttributeValues; + } } diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties index 9f2062df606..40a7615caf4 100644 --- a/tools/ninja/src/main/resources/messages.properties +++ b/tools/ninja/src/main/resources/messages.properties @@ -53,6 +53,8 @@ export.prevent.attribute=Prevent the export of attributes export.exclude.attributes.user=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. export.exclude.attributes.role=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. export.exclude.attributes.org=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.anonymizeAttributeNames=Anonymize attribute names +export.anonymizeOrdinalAttributeValues=Anonymize values of ordinal attributes (Integer, Long, Double) export.name.options=Defines the format of the name parameter in the export. export.business.role.archetype.oid=Detects a business role based on a specific archetype, provided by its OID. export.application.role.archetype.oid=Detects an application role based on a specific archetype, provided by its OID. From ed6a19375c5d4bf03a6b51ef6e655c8e0dda978b Mon Sep 17 00:00:00 2001 From: Martin Bielik Date: Tue, 3 Dec 2024 12:42:26 +0100 Subject: [PATCH 11/11] anonymize polystring attribute without definition --- .../mining/ExportMiningConsumerWorker.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index 30b04197c8b..61e700ce8d2 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -78,9 +78,6 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker businessRoleSuffix; - // NOTE: this class is used to mark attributes that we don't know the schema definition in Ninja - extension attributes defined via GUI - static class UnknownAttributeType {} - record AttributeInfo(ItemName itemName, Class typeClass) { } private Set attrPathsUser; @@ -89,8 +86,8 @@ record AttributeInfo(ItemName itemName, Class typeClass) { } private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef"; private static final List DEFAULT_EXCLUDED_ATTRIBUTES = List.of("description", "documentation", "emailAddress", - "telephoneNumber", "fullName", "givenName", "familyName", "additionalName", "nickName", "personalNumber", "identifier", - "jpegPhoto"); + "telephoneNumber", "name", "fullName", "givenName", "familyName", "additionalName", "nickName", "personalNumber", + "identifier", "jpegPhoto"); public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue queue, OperationStatus operation) { @@ -390,7 +387,7 @@ private AttributeInfo makeAttributeInfo(ItemDefinition def, ItemName itemName } if (def == null) { // extension attributes from GUI schema does not contain definition in Ninja - return new AttributeInfo(itemName, UnknownAttributeType.class); + return new AttributeInfo(itemName, String.class); } var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME); if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { @@ -423,9 +420,19 @@ private Set extractExtensionAttributePaths(PrismContainerValue .collect(Collectors.toUnmodifiableSet()); } + private Object parseRealValue(Item item) throws SchemaException { + if (item.hasCompleteDefinition()) { + return Objects.requireNonNull(item.getRealValue()); + } + // it is unknown if item without definition is multivalued, therefore take any value + RawType rawValue = item.getAnyValue().getRealValue(); + // WORKAROUND: parsing as PolyStringType works for PolyString and all other primitive types + return Objects.requireNonNull(rawValue).getParsedRealValue(PolyStringType.class); + } + private Object anonymizeAttributeValue(Item item, AttributeInfo attributeInfo) throws SchemaException { var typeClass = attributeInfo.typeClass(); - var realValue = Objects.requireNonNull(item.getRealValue()); + var realValue = parseRealValue(item); var attributeName = attributeInfo.itemName().toString(); if (PrismReferenceDefinition.class.equals(typeClass)) { @@ -434,14 +441,8 @@ private Object anonymizeAttributeValue(Item item, AttributeInfo attributeI return encryptObjectReference(referenceValue, securityMode, encryptKey); } - if (typeClass.equals(UnknownAttributeType.class)) { - // anonymize attributes with unknown (extension schema from GUI) - RawType rawValue = item.getValues().get(0).getRealValue(); - var value = Objects.requireNonNull(rawValue).getValue(); - return attributeValuesAnonymizer.anonymize(attributeName, value.toString()); - } - - if (!options.isAnonymizeOrdinalAttributeValues() && List.of(Integer.class, Long.class, Double.class).contains(typeClass)) { + var isOrdinalValue = List.of(Integer.class, Long.class, Double.class).contains(typeClass); + if (!options.isAnonymizeOrdinalAttributeValues() && isOrdinalValue) { // do not anonymize ordinal values return realValue; } @@ -459,10 +460,10 @@ public String anonymizeAttributeName(Item item, SequentialAnonymizer attri private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, AttributeInfo attributeInfo, SequentialAnonymizer attributeNameAnonymizer) { Item item = itemContainer.findItem(attributeInfo.itemName()); - if (item == null || item.getRealValue() == null) { - return; - } try { + if (item == null || item.hasNoValues()) { + return; + } String attributeName = options.isAnonymizeAttributeNames() ? anonymizeAttributeName(item, attributeNameAnonymizer) : item.getElementName().toString(); @@ -477,7 +478,7 @@ private void anonymizeAttribute(FocusType newObject, PrismContainer itemConta anonymizedProperty.setRealValue(anonymizedAttributeValue); newObject.asPrismObject().addExtensionItem(anonymizedProperty); } catch (Exception e) { - context.getLog().warn("Failed to anonymize attribute:\n{}\n{}. ", attributeInfo, item, e); + context.getLog().warn("Failed to anonymize attribute: \n{}\n{}\n{}", e, attributeInfo, item); } }