Merge remote-tracking branch 'origin/master'

repo/repo-cache/src/main/java/com/evolveum/midpoint/repo/cache/RepositoryCache.java

@@ -15,6 +15,7 @@
 import javax.annotation.PreDestroy;
 
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -188,6 +189,20 @@ public <T extends ObjectType> ModifyObjectResult<T> modifyObject(
         return modificationOpHandler.modifyObject(type, oid, modifications, precondition, options, parentResult);
     }
 
+    @Override
+    public @NotNull <T extends ObjectType> ModifyObjectResult<T> modifyObjectDynamically(
+            @NotNull Class<T> type,
+            @NotNull String oid,
+            @Nullable Collection<SelectorOptions<GetOperationOptions>> getOptions,
+            @NotNull ModificationsSupplier<T> modificationsSupplier,
+            @Nullable RepoModifyOptions modifyOptions,
+            @NotNull OperationResult parentResult)
+            throws ObjectNotFoundException, SchemaException, ObjectAlreadyExistsException {
+        // TODO implement properly, currently only to support tests, probably not used via cache in normal code
+        return repositoryService.modifyObjectDynamically(
+                type, oid, getOptions, modificationsSupplier, modifyOptions, parentResult);
+    }
+
     @NotNull
     @Override
     public <T extends ObjectType> DeleteObjectResult deleteObject(Class<T> type, String oid, OperationResult parentResult)

repo/repo-common/pom.xml

@@ -220,4 +220,23 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>dbtest</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-failsafe-plugin</artifactId>
+                        <configuration>
+                            <suiteXmlFiles>
+                                <suiteXmlFile>testng-integration.xml</suiteXmlFile>
+                                <suiteXmlFile>testng-db-specific.xml</suiteXmlFile>
+                            </suiteXmlFiles>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>

repo/repo-common/src/test/resources/concurrency/user.xml

@@ -0,0 +1,25 @@
+<!--
+  ~ Copyright (C) 2010-2021 Evolveum and contributors
+  ~
+  ~ This work is dual-licensed under the Apache License 2.0
+  ~ and European Union Public License. See LICENSE file for details.
+  -->
+
+<user version="1"
+        xmlns="http://midpoint.evolveum.com/xml/ns/public/common/common-3"
+        xmlns:t="http://prism.evolveum.com/xml/ns/public/types-3">
+    <name>concurrentModifyUser002</name>
+    <fullName>
+        <t:orig>will be replaced</t:orig>
+        <t:norm>will be replaced</t:norm>
+    </fullName>
+    <givenName>d</givenName>
+    <familyName>d</familyName>
+    <emailAddress>[email protected]</emailAddress>
+    <assignment id="1">
+        <description>d</description>
+        <construction>
+            <description>d</description>
+        </construction>
+    </assignment>
+</user>

repo/repo-common/testng-db-specific.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (C) 2010-2021 Evolveum and contributors
+  ~
+  ~ This work is dual-licensed under the Apache License 2.0
+  ~ and European Union Public License. See LICENSE file for details.
+  -->
+
+<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
+<suite name="integration" parallel="false" verbose="1">
+    <listeners>
+        <listener class-name="com.evolveum.midpoint.tools.testng.AlphabeticalMethodInterceptor"/>
+    </listeners>
+    <test name="Repo common DB specific" parallel="false" verbose="10">
+        <classes>
+            <class name="com.evolveum.midpoint.repo.common.RepoConcurrencyTest"/>
+        </classes>
+    </test>
+</suite>

repo/repo-common/testng-unit.xml

@@ -1,11 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  ~ Copyright (C) 2010-2020 Evolveum and contributors
+  ~ Copyright (C) 2010-2021 Evolveum and contributors
   ~
   ~ This work is dual-licensed under the Apache License 2.0
   ~ and European Union Public License. See LICENSE file for details.
   -->
+
 <!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
 <suite name="unit" parallel="false">
-    <!-- Tests are mostly in model-common and model-impl -->
+    <!-- No unit tests, see integration test suite XML. -->
 </suite>

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/SqaleRepoBaseTest.java

@@ -59,7 +59,7 @@ public class SqaleRepoBaseTest extends AbstractSpringTest
 
     @BeforeClass
     public void clearDatabase() {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             // object delete cascades to sub-rows of the "object aggregate"
 
             jdbcSession.executeStatement("TRUNCATE m_object CASCADE;");
@@ -78,7 +78,7 @@ public void clearDatabase() {
 
         // this is "suite" scope code, but @BeforeSuite can't use injected fields
         if (!cacheTablesCleared) {
-            try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+            try (JdbcSession jdbcSession = startTransaction()) {
                 // We could skip default relation ID with .where(u.id.gt(0)), but it must work.
                 jdbcSession.newDelete(QUri.DEFAULT).execute();
                 jdbcSession.newDelete(QExtItem.DEFAULT).execute();
@@ -97,7 +97,7 @@ public void clearDatabase() {
 
     // Called on demand
     public void clearAudit() {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.executeStatement("TRUNCATE ma_audit_event CASCADE;");
             jdbcSession.commit();
             display("AUDIT tables cleared");
@@ -141,7 +141,7 @@ protected <R, Q extends FlexibleRelationalPathBase<R>> long count(
 
     protected <R, Q extends FlexibleRelationalPathBase<R>> long count(
             Q path, Predicate... conditions) {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             SQLQuery<?> query = jdbcSession.newQuery()
                     .from(path)
                     .where(conditions);
@@ -156,7 +156,7 @@ protected <R, Q extends FlexibleRelationalPathBase<R>> List<R> select(
 
     protected <R, Q extends FlexibleRelationalPathBase<R>> List<R> select(
             Q path, Predicate... conditions) {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             return jdbcSession.newQuery()
                     .from(path)
                     .where(conditions)
@@ -174,7 +174,7 @@ protected <R, Q extends FlexibleRelationalPathBase<R>> List<R> select(
 
     protected <R, Q extends FlexibleRelationalPathBase<R>> @Nullable R selectOneNullable(
             Q path, Predicate... conditions) {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             return jdbcSession.newQuery()
                     .from(path)
                     .where(conditions)
@@ -335,7 +335,7 @@ private String extKey(Containerable extContainer, String itemName, MExtItemHolde
         PrismContainerValue<?> pcv = extContainer.asPrismContainerValue();
         ItemDefinition<?> def = pcv.getDefinition().findItemDefinition(new ItemName(itemName));
         MExtItem.Key key = MExtItem.keyFrom(def, holder);
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             QExtItem ei = QExtItem.DEFAULT;
             return jdbcSession.newQuery()
                     .from(ei)
@@ -418,4 +418,12 @@ public ItemDefinition<?> getDefinition(ItemName attributeName) {
         }
     }
     // endregion
+
+    protected JdbcSession startTransaction() {
+        return sqlRepoContext.newJdbcSession().startTransaction();
+    }
+
+    protected JdbcSession startReadOnlyTransaction() {
+        return sqlRepoContext.newJdbcSession().startReadOnlyTransaction();
+    }
 }

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/AuditCleanupTest.java

@@ -116,7 +116,7 @@ public void test200CleanupByAge() throws SchemaException {
     }
 
     private long selectMinMaxId(QAuditEventRecord qae, NumberExpression<Long> minMaxPath) {
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             return jdbcSession.newQuery()
                     .select(minMaxPath)
                     .from(qae)

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSearchIterativeTest.java

@@ -228,7 +228,7 @@ public void test125SearchIterativeWithCustomOrdering() throws Exception {
 
         and("all objects were processed in proper order");
         QUser u = aliasFor(QUser.class);
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startReadOnlyTransaction()) {
+        try (JdbcSession jdbcSession = startReadOnlyTransaction()) {
             List<String> result = jdbcSession.newQuery()
                     .from(u)
                     .orderBy(u.costCenter.asc(), u.oid.asc())

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSearchTest.java

@@ -15,13 +15,18 @@
 import static com.evolveum.midpoint.prism.xml.XmlTypeConverter.createXMLGregorianCalendar;
 import static com.evolveum.midpoint.schema.constants.SchemaConstants.ORG_DEFAULT;
 import static com.evolveum.midpoint.util.MiscUtil.asXMLGregorianCalendar;
+import static com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationType.F_VALID_FROM;
+import static com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationType.F_VALID_TO;
+import static com.evolveum.midpoint.xml.ns._public.common.common_3.AssignmentHolderType.F_ASSIGNMENT;
+import static com.evolveum.midpoint.xml.ns._public.common.common_3.FocusType.F_ACTIVATION;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.time.Instant;
 import java.util.List;
 import java.util.UUID;
 import java.util.function.Function;
+import javax.xml.datatype.XMLGregorianCalendar;
 import javax.xml.namespace.QName;
 
 import org.jetbrains.annotations.NotNull;
@@ -897,25 +902,42 @@ public void test322AndFilterVersusExistsAndFilter() throws SchemaException {
 
     @Test
     public void test323ActivationOrAssignmentActivationBeforeDate() throws SchemaException {
-        // this is close to real-life validity query
+        XMLGregorianCalendar scanDate = createXMLGregorianCalendar("2022-01-01T00:00:00Z");
         searchUsersTest("having activation/valid* or assignment/activation/valid* before",
-                f -> f.item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_FROM)
-                        .le(createXMLGregorianCalendar("2022-01-01T00:00:00Z"))
+                f -> f.item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_FROM).le(scanDate)
                         .or()
-                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_TO)
-                        .le(createXMLGregorianCalendar("2022-01-01T00:00:00Z"))
+                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_TO).le(scanDate)
                         .or()
                         .exists(UserType.F_ASSIGNMENT)
                         .block() // block necessary, otherwise the second item goes from User
-                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_FROM)
-                        .le(createXMLGregorianCalendar("2022-01-01T00:00:00Z"))
+                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_FROM).le(scanDate)
                         .or()
-                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_TO)
-                        .le(createXMLGregorianCalendar("2022-01-01T00:00:00Z"))
+                        .item(AssignmentType.F_ACTIVATION, ActivationType.F_VALID_TO).le(scanDate)
                         .endBlock(),
                 user1Oid, user2Oid, user3Oid);
     }
 
+    @Test
+    public void test324ActivationOrAssignmentActivationBetweenDates() throws SchemaException {
+        // taken from FocusValidityScanPartialExecutionSpecifics.createStandardFilter
+        XMLGregorianCalendar lastScanTimestamp = createXMLGregorianCalendar("2021-01-01T00:00:00Z");
+        XMLGregorianCalendar thisScanTimestamp = createXMLGregorianCalendar("2021-06-01T00:00:00Z");
+        searchUsersTest("having activation/valid* or assignment/activation/valid* between dates",
+                f -> f.item(F_ACTIVATION, F_VALID_FROM).gt(lastScanTimestamp)
+                        .and().item(F_ACTIVATION, F_VALID_FROM).le(thisScanTimestamp)
+                        .or().item(F_ACTIVATION, F_VALID_TO).gt(lastScanTimestamp)
+                        .and().item(F_ACTIVATION, F_VALID_TO).le(thisScanTimestamp)
+                        .or().exists(F_ASSIGNMENT)
+                        .block()
+                        .item(AssignmentType.F_ACTIVATION, F_VALID_FROM).gt(lastScanTimestamp)
+                        .and().item(AssignmentType.F_ACTIVATION, F_VALID_FROM).le(thisScanTimestamp)
+                        .or().item(AssignmentType.F_ACTIVATION, F_VALID_TO).gt(lastScanTimestamp)
+                        .and().item(AssignmentType.F_ACTIVATION, F_VALID_TO).le(thisScanTimestamp)
+                        .endBlock(),
+                user1Oid, user2Oid);
+        // user3 barely misses it, it has validFrom = lastSCanTimestamp, but the condition > is exclusive
+    }
+
     @Test
     public void test325ExistsFilterWithSizeColumn() throws SchemaException {
         searchObjectTest("having pending operations", ShadowType.class,

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/func/SqaleRepoSmokeTest.java

@@ -89,7 +89,7 @@ public void test020TestOrgClosureConsistency() throws Exception {
         OperationResult result = createOperationResult();
 
         given("reset closure");
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.executeStatement("CALL m_refresh_org_closure(true)");
             jdbcSession.commit();
         }
@@ -126,7 +126,7 @@ public void test021OrgClosureIsRefreshedBeforeOrgFilterQuery() throws Exception
         OperationResult result = createOperationResult();
 
         given("reset closure");
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.executeStatement("CALL m_refresh_org_closure(true)");
             jdbcSession.commit();
         }
@@ -365,7 +365,7 @@ public void test900WorkingWithPgArraysJsonbAndBytea() {
         user.subtypes = new String[] { "subtype1", "subtype2" };
         user.ext = new Jsonb("{\"key\" : \"value\",\n\"number\": 47} "); // more whitespaces/lines
         user.photo = new byte[] { 0, 1, 0, 1 };
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.newInsert(u).populate(user).execute();
             jdbcSession.commit();
         }
@@ -378,7 +378,7 @@ public void test900WorkingWithPgArraysJsonbAndBytea() {
         assertThat(row.photo).hasSize(4);
 
         // setting NULLs
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.newUpdate(u)
                     .setNull(u.policySituations)
                     .set(u.subtypes, (String[]) null) // this should do the same

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/perf/OrgHierarchyPerfTest.java

@@ -45,7 +45,7 @@ public class OrgHierarchyPerfTest extends SqaleRepoBaseTest {
     public void initObjects() throws Exception {
         OperationResult result = createOperationResult();
 
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             jdbcSession.executeStatement("CALL m_refresh_org_closure(true)");
             jdbcSession.commit();
         }

repo/repo-sqale/src/test/java/com/evolveum/midpoint/repo/sqale/slow/SequenceTestConcurrency.java

@@ -103,7 +103,7 @@ public void test210TenThreadsReturning() throws Exception {
     private void concurrencyUniversal(
             long duration, WorkerThread[] workerThreads, boolean alwaysOrder) throws Exception {
 
-        try (JdbcSession jdbcSession = sqlRepoContext.newJdbcSession().startTransaction()) {
+        try (JdbcSession jdbcSession = startTransaction()) {
             System.out.println(">>>>" + jdbcSession.connection().getTransactionIsolation());
         }