diff --git a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java index 6b3750fd6e8..ccfaecca0a6 100644 --- a/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java +++ b/infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java @@ -11,9 +11,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; -import java.util.Base64; -import java.util.List; -import java.util.UUID; +import java.util.*; import javax.crypto.Cipher; import javax.crypto.spec.SecretKeySpec; @@ -62,6 +60,59 @@ public String getDisplayString() { } } + public static class SequentialAnonymizer { + private final Map anonymizedValues = new HashMap<>(); + private final String baseName; + private long index = 0; + + public SequentialAnonymizer(String baseName) { + this.baseName = baseName; + } + + public String anonymize(String value) { + if (!anonymizedValues.containsKey(value)) { + anonymizedValues.put(value, baseName + index++); + } + return anonymizedValues.get(value); + } + } + + private static class ScopedSequentialAnonymizer { + private final Map scopedAnonymizers = new HashMap<>(); + private final String baseName; + + public ScopedSequentialAnonymizer(String baseName) { + this.baseName = baseName; + } + + public String anonymize(String scope, String value) { + if (!scopedAnonymizers.containsKey(scope)) { + scopedAnonymizers.put(scope, new SequentialAnonymizer(baseName)); + } + return scopedAnonymizers.get(scope).anonymize(value); + } + } + + public static class AttributeValueAnonymizer { + private final ScopedSequentialAnonymizer sequentialAnonymizer = new ScopedSequentialAnonymizer("att"); + private final NameMode nameMode; + private final String encryptKey; + + public AttributeValueAnonymizer(NameMode nameMode, String encryptKey) { + this.nameMode = nameMode; + this.encryptKey = encryptKey; + } + + public String anonymize(String attributeName, String attributeValue) { + var anonymized = switch(nameMode) { + case ENCRYPTED -> encrypt(attributeValue, encryptKey); + case SEQUENTIAL -> sequentialAnonymizer.anonymize(attributeName, attributeValue); + case ORIGINAL -> attributeValue; + }; + return anonymized + EXPORT_SUFFIX; + } + } + private static PolyStringType encryptName(String name, int iterator, String prefix, @NotNull NameMode nameMode, String key) { if (nameMode.equals(NameMode.ENCRYPTED)) { return PolyStringType.fromOrig(encrypt(name, key) + EXPORT_SUFFIX); @@ -85,6 +136,12 @@ public static PolyStringType encryptRoleName(String name, int iterator, NameMode return encryptName(name, iterator, "Role", nameMode, key); } + public static ObjectReferenceType encryptObjectReference(ObjectReferenceType targetRef, SecurityMode securityMode, String key) { + ObjectReferenceType encryptedTargetRef = targetRef.clone(); + encryptedTargetRef.setOid(encryptedUUID(encryptedTargetRef.getOid(), securityMode, key)); + return encryptedTargetRef; + } + public static AssignmentType encryptObjectReference(@NotNull AssignmentType assignmentObject, SecurityMode securityMode, String key) { ObjectReferenceType encryptedTargetRef = assignmentObject.getTargetRef(); diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java index cf6494221c3..61e700ce8d2 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java @@ -12,12 +12,19 @@ import java.io.IOException; import java.io.Writer; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.xml.namespace.QName; -import com.evolveum.midpoint.util.logging.Trace; +import com.evolveum.midpoint.common.mining.utils.RoleAnalysisAttributeDefUtils; +import com.evolveum.midpoint.prism.*; +import com.evolveum.midpoint.prism.path.ItemName; +import com.evolveum.midpoint.util.DOMUtil; -import com.evolveum.midpoint.util.logging.TraceManager; +import com.evolveum.prism.xml.ns._public.types_3.RawType; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -28,29 +35,31 @@ import com.evolveum.midpoint.ninja.util.FileReference; import com.evolveum.midpoint.ninja.util.NinjaUtils; import com.evolveum.midpoint.ninja.util.OperationStatus; -import com.evolveum.midpoint.prism.PrismContainerValue; -import com.evolveum.midpoint.prism.PrismSerializer; -import com.evolveum.midpoint.prism.SerializationOptions; import com.evolveum.midpoint.prism.query.ObjectFilter; import com.evolveum.midpoint.prism.query.ObjectQuery; import com.evolveum.midpoint.schema.result.OperationResult; import com.evolveum.midpoint.util.exception.SchemaException; -import com.evolveum.midpoint.util.logging.LoggingUtils; import com.evolveum.midpoint.xml.ns._public.common.common_3.*; import com.evolveum.prism.xml.ns._public.types_3.PolyStringType; +/** + * Anonymize and write midpoint's objects. + * - it is currently assumed to run in a single thread, therefore the state does not need to be shared and thread safe + */ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker { - private static final Trace LOGGER = TraceManager.getTrace(ExportMiningConsumerWorker.class); - OperationResult operationResult = new OperationResult(DOT_CLASS + "searchObjectByCondition"); private PrismSerializer serializer; private int processedRoleIterator = 0; private int processedUserIterator = 0; private int processedOrgIterator = 0; + private final SequentialAnonymizer defaultAttributeNameAnonymizer = new SequentialAnonymizer("default_attr"); + private final SequentialAnonymizer extensionAttributeNameAnonymizer = new SequentialAnonymizer("extension_attr"); + private AttributeValueAnonymizer attributeValuesAnonymizer; private boolean orgAllowed; + private boolean attributesAllowed; private boolean firstObject = true; private boolean jsonFormat = false; @@ -68,6 +77,18 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker businessRolePrefix; private List businessRoleSuffix; + + record AttributeInfo(ItemName itemName, Class typeClass) { } + + private Set attrPathsUser; + private Set attrPathsRole; + private Set attrPathsOrg; + + private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef"; + private static final List DEFAULT_EXCLUDED_ATTRIBUTES = List.of("description", "documentation", "emailAddress", + "telephoneNumber", "name", "fullName", "givenName", "familyName", "additionalName", "nickName", "personalNumber", + "identifier", "jpegPhoto"); + public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue queue, OperationStatus operation) { super(context, options, queue, operation); @@ -81,9 +102,16 @@ protected void init() { securityMode = options.getSecurityLevel(); encryptKey = RoleMiningExportUtils.updateEncryptKey(securityMode); orgAllowed = options.isIncludeOrg(); + attributesAllowed = options.isIncludeAttributes(); nameMode = options.getNameMode(); + attrPathsUser = extractDefaultAttributePaths(UserType.COMPLEX_TYPE, options.getExcludedAttributesUser()); + attrPathsRole = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE, options.getExcludedAttributesRole()); + attrPathsOrg = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE, options.getExcludedAttributesOrg()); + + attributeValuesAnonymizer = new AttributeValueAnonymizer(nameMode, encryptKey); + SerializationOptions serializationOptions = SerializationOptions.createSerializeForExport() .serializeReferenceNames(true) .serializeForExport(true) @@ -142,6 +170,10 @@ private void write(Writer writer, PrismContainerValue prismContainerValue) th @NotNull private OrgType getPreparedOrgObject(@NotNull FocusType object) { OrgType org = new OrgType(); + + fillAttributes(object, org, attrPathsOrg, options.getExcludedAttributesOrg()); + fillActivation(object, org); + org.setName(encryptOrgName(object.getName().toString(), processedOrgIterator++, nameMode, encryptKey)); org.setOid(encryptedUUID(object.getOid(), securityMode, encryptKey)); @@ -172,6 +204,8 @@ && filterAllowedOrg(oid)) { @NotNull private UserType getPreparedUserObject(@NotNull FocusType object) { UserType user = new UserType(); + fillAttributes(object, user, attrPathsUser, options.getExcludedAttributesUser()); + fillActivation(object, user); List assignment = object.getAssignment(); if (assignment == null || assignment.isEmpty()) { @@ -212,6 +246,9 @@ && filterAllowedRole(oid)) { @NotNull private RoleType getPreparedRoleObject(@NotNull FocusType object) { RoleType role = new RoleType(); + fillAttributes(object, role, attrPathsRole, options.getExcludedAttributesRole()); + fillActivation(object, role); + String roleName = object.getName().toString(); PolyStringType encryptedName = encryptRoleName(roleName, processedRoleIterator++, nameMode, encryptKey); role.setName(encryptedName); @@ -293,7 +330,7 @@ private boolean filterAllowedOrg(String oid) { return !context.getRepository().searchObjects(OrgType.class, objectQuery, null, operationResult).isEmpty(); } catch (SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to search organization object. ", e); + context.getLog().error("Failed to search organization object. ", e); } return false; } @@ -309,7 +346,7 @@ private boolean filterAllowedRole(String oid) { return !context.getRepository().searchObjects(RoleType.class, objectQuery, null, operationResult).isEmpty(); } catch (SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to search role object. ", e); + context.getLog().error("Failed to search role object. ", e); } return false; } @@ -319,7 +356,7 @@ private void loadFilters(FileReference roleFileReference, FileReference orgFileR this.filterRole = NinjaUtils.createObjectFilter(roleFileReference, context, RoleType.class); this.filterOrg = NinjaUtils.createObjectFilter(orgFileReference, context, OrgType.class); } catch (IOException | SchemaException e) { - LoggingUtils.logException(LOGGER, "Failed to crate object filter. ", e); + context.getLog().error("Failed to crate object filter. ", e); } } @@ -342,4 +379,139 @@ private void loadRoleCategoryIdentifiers() { return null; } + + private AttributeInfo makeAttributeInfo(ItemDefinition def, ItemName itemName, List excludedAttributeNames) { + var attributeName = itemName.toString(); + if (excludedAttributeNames.contains(attributeName)) { + return null; + } + if (def == null) { + // extension attributes from GUI schema does not contain definition in Ninja + return new AttributeInfo(itemName, String.class); + } + var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME); + if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) { + // unsupported types + return null; + } + if (def instanceof PrismReferenceDefinition) { + return new AttributeInfo(itemName, PrismReferenceDefinition.class); + } + if (def instanceof PrismPropertyDefinition propertyDef && RoleAnalysisAttributeDefUtils.isSupportedPropertyType(propertyDef.getTypeClass())) { + return new AttributeInfo(itemName, propertyDef.getTypeClass()); + } + return null; + } + + private Set extractDefaultAttributePaths(QName type, List excludedDefaultAttributes) { + var containerDef = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type); + var excludedAttributes = Stream.concat(excludedDefaultAttributes.stream(), DEFAULT_EXCLUDED_ATTRIBUTES.stream()).toList(); + return containerDef.getDefinitions().stream() + .map(def -> makeAttributeInfo(def, def.getItemName(), excludedAttributes)) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + + private Set extractExtensionAttributePaths(PrismContainerValue containerValue, List excludedAttributeNames) { + return containerValue.getItems().stream() + .map(item -> makeAttributeInfo(item.getDefinition(), item.getElementName(), excludedAttributeNames)) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + private Object parseRealValue(Item item) throws SchemaException { + if (item.hasCompleteDefinition()) { + return Objects.requireNonNull(item.getRealValue()); + } + // it is unknown if item without definition is multivalued, therefore take any value + RawType rawValue = item.getAnyValue().getRealValue(); + // WORKAROUND: parsing as PolyStringType works for PolyString and all other primitive types + return Objects.requireNonNull(rawValue).getParsedRealValue(PolyStringType.class); + } + + private Object anonymizeAttributeValue(Item item, AttributeInfo attributeInfo) throws SchemaException { + var typeClass = attributeInfo.typeClass(); + var realValue = parseRealValue(item); + var attributeName = attributeInfo.itemName().toString(); + + if (PrismReferenceDefinition.class.equals(typeClass)) { + // anonymize references + var referenceValue = (ObjectReferenceType) realValue; + return encryptObjectReference(referenceValue, securityMode, encryptKey); + } + + var isOrdinalValue = List.of(Integer.class, Long.class, Double.class).contains(typeClass); + if (!options.isAnonymizeOrdinalAttributeValues() && isOrdinalValue) { + // do not anonymize ordinal values + return realValue; + } + + return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString()); + } + + public String anonymizeAttributeName(Item item, SequentialAnonymizer attributeNameAnonymizer) { + String originalAttributeName = item.getElementName().toString(); + if (nameMode.equals(NameMode.ORIGINAL)) { + return originalAttributeName; + } + return attributeNameAnonymizer.anonymize(originalAttributeName); + } + + private void anonymizeAttribute(FocusType newObject, PrismContainer itemContainer, AttributeInfo attributeInfo, SequentialAnonymizer attributeNameAnonymizer) { + Item item = itemContainer.findItem(attributeInfo.itemName()); + try { + if (item == null || item.hasNoValues()) { + return; + } + String attributeName = options.isAnonymizeAttributeNames() + ? anonymizeAttributeName(item, attributeNameAnonymizer) + : item.getElementName().toString(); + Object anonymizedAttributeValue = anonymizeAttributeValue(item, attributeInfo); + + QName propertyName = new QName(attributeInfo.itemName().getNamespaceURI(), attributeName); + PrismPropertyDefinition propertyDefinition = context + .getPrismContext() + .definitionFactory() + .newPropertyDefinition(propertyName, DOMUtil.XSD_STRING); + PrismProperty anonymizedProperty = propertyDefinition.instantiate(); + anonymizedProperty.setRealValue(anonymizedAttributeValue); + newObject.asPrismObject().addExtensionItem(anonymizedProperty); + } catch (Exception e) { + context.getLog().warn("Failed to anonymize attribute: \n{}\n{}\n{}", e, attributeInfo, item); + } + } + + private void fillAttributes( + @NotNull FocusType origObject, + @NotNull FocusType newObject, + @NotNull Set defaultAttributePaths, + @NotNull List excludedAttributes + ) { + if (!attributesAllowed) { + return; + } + var origContainer = origObject.asPrismObject(); + newObject.extension(new ExtensionType()); + + for (var path: defaultAttributePaths) { + anonymizeAttribute(newObject, origContainer, path, defaultAttributeNameAnonymizer); + } + + if (origContainer.getExtension() != null) { + var extensionAttributePaths = extractExtensionAttributePaths(origContainer.getExtensionContainerValue(), excludedAttributes); + for (var path: extensionAttributePaths) { + anonymizeAttribute(newObject, origContainer.getExtension(), path, extensionAttributeNameAnonymizer); + } + } + } + + private void fillActivation(@NotNull FocusType origObject, @NotNull FocusType newObject) { + if (origObject.getActivation() == null) { + return; + } + var activation = new ActivationType().effectiveStatus(origObject.getActivation().getEffectiveStatus()); + newObject.setActivation(activation); + } + } diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java index fb95847fb80..57e17deda24 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java @@ -16,6 +16,8 @@ import com.evolveum.midpoint.common.RoleMiningExportUtils; import com.evolveum.midpoint.ninja.action.BasicExportOptions; +import com.evolveum.midpoint.ninja.util.ItemPathConverter; +import com.evolveum.midpoint.prism.path.ItemPath; @Parameters(resourceBundle = "messages", commandDescriptionKey = "exportMining") public class ExportMiningOptions extends BaseMiningOptions implements BasicExportOptions { @@ -35,6 +37,13 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor public static final String P_SUFFIX_BUSINESS_LONG = "--business-role-suffix"; public static final String P_ORG = "-do"; public static final String P_ORG_LONG = "--disable-org"; + public static final String P_ATTRIBUTE = "-da"; + public static final String P_ATTRIBUTE_LONG = "--disable-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_USER_LONG = "--exclude-user-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_ROLE_LONG = "--exclude-role-attribute"; + public static final String P_EXCLUDE_ATTRIBUTES_ORG_LONG = "--exclude-org-attribute"; + public static final String P_ANONYMIZE_ATTRIBUTE_NAMES = "--anonymize-attribute-names"; + public static final String P_ANONYMIZE_ORDINAL_ATTRIBUTE_VALUES = "--anonymize-ordinal-attribute-values"; public static final String P_NAME_OPTIONS = "-nm"; public static final String P_NAME_OPTIONS_LONG = "--name-mode"; public static final String P_ARCHETYPE_OID_APPLICATION_LONG = "--application-role-archetype-oid"; @@ -66,6 +75,9 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor @Parameter(names = { P_ORG, P_ORG_LONG }, descriptionKey = "export.prevent.org") private boolean disableOrg = false; + @Parameter(names = { P_ATTRIBUTE, P_ATTRIBUTE_LONG }, descriptionKey = "export.prevent.attribute") + private boolean disableAttribute = false; + @Parameter(names = { P_NAME_OPTIONS, P_NAME_OPTIONS_LONG }, descriptionKey = "export.name.options") private RoleMiningExportUtils.NameMode nameMode = RoleMiningExportUtils.NameMode.SEQUENTIAL; @@ -77,6 +89,24 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor descriptionKey = "export.business.role.archetype.oid") private String businessRoleArchetypeOid = "00000000-0000-0000-0000-000000000321"; + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_USER_LONG }, descriptionKey = "export.exclude.attributes.user", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesUser = new ArrayList<>(); + + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_ROLE_LONG }, descriptionKey = "export.exclude.attributes.role", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesRole = new ArrayList<>(); + + @Parameter(names = { P_EXCLUDE_ATTRIBUTES_ORG_LONG }, descriptionKey = "export.exclude.attributes.org", + validateWith = ItemPathConverter.class, converter = ItemPathConverter.class) + private List excludedAttributesOrg = new ArrayList<>(); + + @Parameter(names = { P_ANONYMIZE_ATTRIBUTE_NAMES }, descriptionKey = "export.anonymizeAttributeNames") + private Boolean anonymizeAttributeNames = false; + + @Parameter(names = { P_ANONYMIZE_ORDINAL_ATTRIBUTE_VALUES }, descriptionKey = "export.anonymizeOrdinalAttributeValues") + private Boolean anonymizeOrdinalAttributeValues = false; + public RoleMiningExportUtils.SecurityMode getSecurityLevel() { return securityMode; } @@ -85,6 +115,10 @@ public boolean isIncludeOrg() { return !disableOrg; } + public boolean isIncludeAttributes() { + return !disableAttribute; + } + public String getApplicationRoleArchetypeOid() { return applicationRoleArchetypeOid; } @@ -136,4 +170,28 @@ public List getBusinessRoleSuffix() { String[] separateSuffixes = businessRoleSuffix.split(DELIMITER); return new ArrayList<>(Arrays.asList(separateSuffixes)); } + + private List itemPathsToStrings(List itemPaths) { + return itemPaths.stream().map(ItemPath::toString).toList(); + } + + public List getExcludedAttributesUser() { + return itemPathsToStrings(excludedAttributesUser); + } + + public List getExcludedAttributesRole() { + return itemPathsToStrings(excludedAttributesRole); + } + + public List getExcludedAttributesOrg() { + return itemPathsToStrings(excludedAttributesOrg); + } + + public Boolean isAnonymizeAttributeNames() { + return anonymizeAttributeNames; + } + + public Boolean isAnonymizeOrdinalAttributeValues() { + return anonymizeOrdinalAttributeValues; + } } diff --git a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java index 95724c956a9..1173ac612ed 100644 --- a/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java +++ b/tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningRepositoryAction.java @@ -69,6 +69,7 @@ public Void execute() throws Exception { executor.execute(new ProgressReporterWorker<>(context, options, queue, operation)); + // NOTE: the consumer is designed to be executed in a single thread Runnable consumer = createConsumer(queue, operation); executor.execute(consumer); diff --git a/tools/ninja/src/main/resources/messages.properties b/tools/ninja/src/main/resources/messages.properties index 738016ca337..40a7615caf4 100644 --- a/tools/ninja/src/main/resources/messages.properties +++ b/tools/ninja/src/main/resources/messages.properties @@ -49,6 +49,12 @@ export.application.role.suffix=Suffix for identifying exported application roles export.business.role.prefix=Prefix for identifying exported business roles. Multiple prefixes can be specified using a comma "," as a delimiter. export.business.role.suffix=Suffix for identifying exported business roles. Multiple suffixes can be specified using a comma "," as a delimiter. export.prevent.org=Prevent the export of organizational structures. +export.prevent.attribute=Prevent the export of attributes +export.exclude.attributes.user=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.exclude.attributes.role=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.exclude.attributes.org=Exclude attributes by attribute name. Option can be used multiple times, or values can be separated by comma. +export.anonymizeAttributeNames=Anonymize attribute names +export.anonymizeOrdinalAttributeValues=Anonymize values of ordinal attributes (Integer, Long, Double) export.name.options=Defines the format of the name parameter in the export. export.business.role.archetype.oid=Detects a business role based on a specific archetype, provided by its OID. export.application.role.archetype.oid=Detects an application role based on a specific archetype, provided by its OID.