diff --git a/.github/workflows/deploy-instance.yml b/.github/workflows/deploy-instance.yml index 905f6a8f96b4..9dc69037123b 100644 --- a/.github/workflows/deploy-instance.yml +++ b/.github/workflows/deploy-instance.yml @@ -21,9 +21,6 @@ jobs: INSTANCE_HOST: 'https://dev.im.dhis2.org' INSTANCE_NAME: pr-${{ github.event.pull_request.number }} steps: - - name: Dump context - uses: crazy-max/ghaction-dump-context@v2 - - name: Wait for API tests # Using this fork of the upstream https://github.com/lewagon/wait-on-check-action, # as it will filter out and check only the latest run of a workflow when checking for the allowed conclusions, diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateService.java index 963c07af7909..7c115460649b 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateService.java @@ -29,7 +29,6 @@ import java.util.List; import org.hisp.dhis.dataset.DataSet; -import org.hisp.dhis.program.notification.NotificationTrigger; /** Created by zubair@dhis2.org on 20.07.17. */ public interface DataSetNotificationTemplateService { @@ -39,7 +38,7 @@ public interface DataSetNotificationTemplateService { List getCompleteNotifications(DataSet dataSet); - List getScheduledNotifications(NotificationTrigger trigger); + List getScheduledNotifications(DataSetNotificationTrigger trigger); List getAll(); diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStore.java index fce43cd27122..e0667bbe0290 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStore.java @@ -30,7 +30,6 @@ import java.util.List; import org.hisp.dhis.common.IdentifiableObjectStore; import org.hisp.dhis.dataset.DataSet; -import org.hisp.dhis.program.notification.NotificationTrigger; /** Created by zubair@dhis2.org on 13.07.17. */ public interface DataSetNotificationTemplateStore @@ -38,5 +37,5 @@ public interface DataSetNotificationTemplateStore List getNotificationsByTriggerType( DataSet dataSet, DataSetNotificationTrigger trigger); - List getScheduledNotifications(NotificationTrigger trigger); + List getScheduledNotifications(DataSetNotificationTrigger trigger); } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/MonitoringService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/dxf2/monitoring/MonitoringService.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/MonitoringService.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/dxf2/monitoring/MonitoringService.java diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ParseType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ParseType.java index 35b4c4e1f38f..e24d1f2cfe65 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ParseType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/expression/ParseType.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.expression; -import static org.hisp.dhis.analytics.DataType.*; +import static org.hisp.dhis.analytics.DataType.BOOLEAN; +import static org.hisp.dhis.analytics.DataType.NUMERIC; import org.hisp.dhis.analytics.DataType; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java index c14f451b9423..b0f9e992a6f1 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/feedback/ErrorCode.java @@ -146,6 +146,9 @@ public enum ErrorCode { E2206("Max results exceeds the allowed max limit: `{0}`"), E2207("Data start date must be before data end date"), E2208("Non-numeric data values encountered during outlier value detection"), + E2209("Data start date not allowed"), + E2210("Data end date not allowed"), + E2211("Algorithm min-max values not allowed"), /* Followup analysis */ E2300("At least one data element or data set must be specified"), @@ -336,6 +339,8 @@ public enum ErrorCode { /* File resource */ E6100("Filename not present"), E6101("File type not allowed"), + E6102("File content could not be stored"), + E6103("File resource appears to have no content"), /* Users */ E6200("Feedback message recipients user group not defined"), diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java index b512e9aaacf6..51b21212497d 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceContentStore.java @@ -34,6 +34,8 @@ import java.net.URI; import java.util.Map; import java.util.NoSuchElementException; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; /** * @author Halvdan Hoem Grelland @@ -64,7 +66,8 @@ public interface FileResourceContentStore { * @param bytes the byte array. * @return the key on success or null if saving failed. */ - String saveFileResourceContent(FileResource fileResource, byte[] bytes); + @CheckForNull + String saveFileResourceContent(@Nonnull FileResource fileResource, @Nonnull byte[] bytes); /** * Save the contents of the File to the file store. @@ -73,7 +76,8 @@ public interface FileResourceContentStore { * @param file the File. Will be consumed upon deletion. * @return the key on success or null if saving failed. */ - String saveFileResourceContent(FileResource fileResource, File file); + @CheckForNull + String saveFileResourceContent(@Nonnull FileResource fileResource, @Nonnull File file); /** * Save the content of image files. @@ -82,8 +86,9 @@ public interface FileResourceContentStore { * @param imageFile will map image dimension to its associated file. * @return the key on success or null if saving failed. */ + @CheckForNull String saveFileResourceContent( - FileResource fileResource, Map imageFile); + @Nonnull FileResource fileResource, @Nonnull Map imageFile); /** * Delete the content bytes of a file resource. diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java index 92e43b63b40c..90f7915a2b2c 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceService.java @@ -32,7 +32,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; -import java.time.Duration; import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; @@ -90,9 +89,31 @@ public interface FileResourceService { */ List findOwnersByStorageKey(@CheckForNull String storageKey); - void saveFileResource(FileResource fileResource, File file); + /** + * Creates the provided file resource and stores the file content asynchronously. + * + * @param fileResource the resource to create + * @param file the content stored asynchronously + */ + void asyncSaveFileResource(FileResource fileResource, File file); + + /** + * Creates the provided file resource and stores the content asynchronously. + * + * @param fileResource the resource to create + * @param bytes the content stored asynchronously + * @return the UID of the created file resource + */ + String asyncSaveFileResource(FileResource fileResource, byte[] bytes); - String saveFileResource(FileResource fileResource, byte[] bytes); + /** + * Creates the provided file resource and stores the content synchronously. + * + * @param fileResource the resource to create + * @param bytes the content stored asynchronously + * @return the UID of the created file resource + */ + String syncSaveFileResource(FileResource fileResource, byte[] bytes) throws ConflictException; void deleteFileResource(String uid); @@ -101,10 +122,6 @@ public interface FileResourceService { @Nonnull InputStream getFileResourceContent(FileResource fileResource) throws ConflictException; - @Nonnull - InputStream getFileResourceContent(FileResource fileResource, Duration timeout) - throws ConflictException; - /** Copy fileResource content to outputStream and Return File content length */ void copyFileResourceContent(FileResource fileResource, OutputStream outputStream) throws IOException, NoSuchElementException; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/BinaryFileSavedEvent.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/BinaryFileSavedEvent.java index c6f469ddece7..74dd97b15743 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/BinaryFileSavedEvent.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/BinaryFileSavedEvent.java @@ -27,24 +27,16 @@ */ package org.hisp.dhis.fileresource.events; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + /** * @Author Zubair Asghar. */ +@Getter +@RequiredArgsConstructor public class BinaryFileSavedEvent { - private String fileResource; - - private byte[] bytes; - - public BinaryFileSavedEvent(String fileResource, byte[] bytes) { - this.fileResource = fileResource; - this.bytes = bytes; - } - - public String getFileResource() { - return fileResource; - } - public byte[] getBytes() { - return bytes; - } + private final String fileResource; + private final byte[] bytes; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileDeletedEvent.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileDeletedEvent.java index 97ab63ec05db..f97536705aa2 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileDeletedEvent.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileDeletedEvent.java @@ -27,33 +27,18 @@ */ package org.hisp.dhis.fileresource.events; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.fileresource.FileResourceDomain; /** * @Author Zubair Asghar. */ +@Getter +@RequiredArgsConstructor public class FileDeletedEvent { - private String storageKey; - private String contentType; - - private FileResourceDomain domain; - - public FileDeletedEvent(String storageKey, String contentType, FileResourceDomain domain) { - this.storageKey = storageKey; - this.contentType = contentType; - this.domain = domain; - } - - public String getStorageKey() { - return storageKey; - } - - public String getContentType() { - return contentType; - } - - public FileResourceDomain getDomain() { - return domain; - } + private final String storageKey; + private final String contentType; + private final FileResourceDomain domain; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileSavedEvent.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileSavedEvent.java index a80548bec28f..c28fea5a0899 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileSavedEvent.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/FileSavedEvent.java @@ -28,25 +28,16 @@ package org.hisp.dhis.fileresource.events; import java.io.File; +import lombok.Getter; +import lombok.RequiredArgsConstructor; /** * @Author Zubair Asghar. */ +@Getter +@RequiredArgsConstructor public class FileSavedEvent { - private String fileResource; - private File file; - - public FileSavedEvent(String fileResource, File file) { - this.fileResource = fileResource; - this.file = file; - } - - public String getFileResource() { - return fileResource; - } - - public File getFile() { - return file; - } + private final String fileResource; + private final File file; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/ImageFileSavedEvent.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/ImageFileSavedEvent.java index 6fe2998c952f..44498af1c22c 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/ImageFileSavedEvent.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/events/ImageFileSavedEvent.java @@ -29,26 +29,17 @@ import java.io.File; import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.fileresource.ImageFileDimension; /** * @Author Zubair Asghar. */ +@Getter +@RequiredArgsConstructor public class ImageFileSavedEvent { - private String fileResource; - private Map imageFiles; - - public ImageFileSavedEvent(String fileResource, Map imageFiles) { - this.fileResource = fileResource; - this.imageFiles = imageFiles; - } - - public String getFileResource() { - return fileResource; - } - - public Map getImageFiles() { - return imageFiles; - } + private final String fileResource; + private final Map imageFiles; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java index 5ae65e907996..9d675f8a6554 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnit.java @@ -564,7 +564,21 @@ public boolean isDescendant(Collection ancestors) { return ancestors.stream() .filter(Objects::nonNull) .map(OrganisationUnit::getUid) - .anyMatch(uid -> StringUtils.contains(path, uid)); + .anyMatch(uid -> StringUtils.contains(this.getPath(), uid)); + } + + /** + * Indicates whether this org unit is a descendant of the given ancestor org unit. + * + * @param ancestor the ancestor org unit. + * @return true if this org unit is a descendant of the ancestor. + */ + public boolean isDescendant(OrganisationUnit ancestor) { + if (ancestor == null) { + return false; + } + + return StringUtils.contains(this.getPath(), ancestor.getUid()); } public Set getChildrenThisIfEmpty() { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java index 9aa51ab2f5c9..e3843e18b974 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitGroup.java @@ -34,7 +34,11 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.util.HashSet; import java.util.Set; -import org.hisp.dhis.common.*; +import org.hisp.dhis.common.BaseDimensionalItemObject; +import org.hisp.dhis.common.BaseIdentifiableObject; +import org.hisp.dhis.common.DimensionItemType; +import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.common.MetadataObject; import org.hisp.dhis.common.coordinate.CoordinateObject; import org.hisp.dhis.common.coordinate.CoordinateUtils; import org.locationtech.jts.geom.Geometry; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java index f8e43ccc47f5..0425844a8120 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/organisationunit/OrganisationUnitService.java @@ -387,10 +387,6 @@ List getOrganisationUnitByCoordinate( */ boolean isInUserHierarchy(String uid, Set organisationUnits); - boolean isDescendant(OrganisationUnit organisationUnit, Set ancestors); - - boolean isDescendant(OrganisationUnit organisationUnit, OrganisationUnit ancestor); - /** * Indicates whether the given organisation unit is part of the hierarchy of the data view * organisation units of the current user. diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java index 19fbcda300fc..c1fe2e7ab52f 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/period/RelativePeriods.java @@ -34,7 +34,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.IntStream; import java.util.stream.Stream; import lombok.EqualsAndHashCode; diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEntityMapperService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEntityMapperService.java index 08796dc64a7f..bbb9ca2547c6 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEntityMapperService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEntityMapperService.java @@ -35,7 +35,10 @@ import org.hisp.dhis.programrule.ProgramRule; import org.hisp.dhis.programrule.ProgramRuleVariable; import org.hisp.dhis.rules.DataItem; -import org.hisp.dhis.rules.models.*; +import org.hisp.dhis.rules.models.Rule; +import org.hisp.dhis.rules.models.RuleEnrollment; +import org.hisp.dhis.rules.models.RuleEvent; +import org.hisp.dhis.rules.models.RuleVariable; import org.hisp.dhis.trackedentityattributevalue.TrackedEntityAttributeValue; /** diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobProgress.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobProgress.java index ee0f14d50f0f..cd513d30acae 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobProgress.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobProgress.java @@ -34,9 +34,21 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; -import java.util.*; +import java.util.Collection; +import java.util.Date; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.*; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -44,9 +56,14 @@ import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; -import lombok.*; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.experimental.Accessors; import org.hisp.dhis.feedback.ErrorCode; +import org.hisp.dhis.tracker.imports.validation.ValidationCode; /** * @@ -147,6 +164,14 @@ default void addError( addError(code, uid, type, List.of(args)); } + default void addError( + @Nonnull ValidationCode code, + @CheckForNull String uid, + @Nonnull String type, + String... args) { + addError(code, uid, type, List.of(args)); + } + default void addError( @Nonnull ErrorCode code, @CheckForNull String uid, @@ -156,6 +181,15 @@ default void addError( // default is to not collect errors } + default void addError( + @Nonnull ValidationCode code, + @CheckForNull String uid, + @Nonnull String type, + @Nonnull List args) { + // is overridden by a tracker that collects errors + // default is to not collect errors + } + /* * Tracking API: */ @@ -580,7 +614,7 @@ final class Progress { @Nonnull @JsonProperty @JsonInclude(JsonInclude.Include.NON_EMPTY) - private final Map>> errors; + private final Map>> errors; public Progress() { this.sequence = new ConcurrentLinkedDeque<>(); @@ -590,7 +624,7 @@ public Progress() { @JsonCreator public Progress( @Nonnull @JsonProperty("sequence") Deque sequence, - @CheckForNull @JsonProperty("errors") Map>> errors) { + @CheckForNull @JsonProperty("errors") Map>> errors) { this.sequence = sequence; this.errors = errors == null ? Map.of() : errors; } @@ -609,7 +643,7 @@ public boolean hasErrors() { return !errors.isEmpty(); } - public Set getErrorCodes() { + public Set getErrorCodes() { return errors.values().stream() .flatMap(e -> e.keySet().stream()) .collect(toUnmodifiableSet()); @@ -620,7 +654,7 @@ public Set getErrorCodes() { @Accessors(chain = true) final class Error { - @Nonnull @JsonProperty private final ErrorCode code; + @Nonnull @JsonProperty private final String code; /** The object that has the error */ @Nonnull @JsonProperty private final String id; @@ -641,7 +675,7 @@ final class Error { @JsonCreator public Error( - @Nonnull @JsonProperty("code") ErrorCode code, + @Nonnull @JsonProperty("code") String code, @Nonnull @JsonProperty("id") String id, @Nonnull @JsonProperty("type") String type, @Nonnull @JsonProperty("args") List args) { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobRunErrorsParams.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobRunErrorsParams.java index f7d0f28c74e1..7c26002f7b84 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobRunErrorsParams.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobRunErrorsParams.java @@ -72,4 +72,6 @@ public class JobRunErrorsParams { /** The {@link JobType} with errors to select, any match combined */ @CheckForNull private List type; + + boolean includeInput = false; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/AnalyticsJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/AnalyticsJobParameters.java index dde5d496f5e5..e8291d0f0482 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/AnalyticsJobParameters.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/AnalyticsJobParameters.java @@ -55,15 +55,18 @@ public class AnalyticsJobParameters implements JobParameters { private Set skipPrograms = new HashSet<>(); @JsonProperty private boolean skipResourceTables = false; + @JsonProperty private boolean skipOutliers = false; public AnalyticsJobParameters( Integer lastYears, Set skipTableTypes, Set skipPrograms, - boolean skipResourceTables) { + boolean skipResourceTables, + boolean skipOutliers) { this.lastYears = lastYears; this.skipTableTypes = skipTableTypes; this.skipPrograms = skipPrograms; this.skipResourceTables = skipResourceTables; + this.skipOutliers = skipOutliers; } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/ContinuousAnalyticsJobParameters.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/ContinuousAnalyticsJobParameters.java index 98acdb263804..136378626e26 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/ContinuousAnalyticsJobParameters.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/parameters/ContinuousAnalyticsJobParameters.java @@ -52,10 +52,17 @@ public class ContinuousAnalyticsJobParameters implements JobParameters { /** The types of analytics tables for which to skip update. */ @JsonProperty private Set skipTableTypes = new HashSet<>(); + /** Outliers statistics columns of Analytics tables will be skipped. */ + @JsonProperty private Boolean skipOutliers = false; + public ContinuousAnalyticsJobParameters( - Integer fullUpdateHourOfDay, Integer lastYears, Set skipTableTypes) { + Integer fullUpdateHourOfDay, + Integer lastYears, + Set skipTableTypes, + Boolean skipOutliers) { this.fullUpdateHourOfDay = fullUpdateHourOfDay; this.lastYears = lastYears; this.skipTableTypes = skipTableTypes; + this.skipOutliers = skipOutliers; } } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemInfo.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemInfo.java new file mode 100644 index 000000000000..dcd04c3e234d --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemInfo.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Date; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.hisp.dhis.system.database.DatabaseInfo; + +/** + * @author Lars Helge Overland + */ +@Getter +@Setter +@Builder(toBuilder = true) +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class SystemInfo { + // ------------------------------------------------------------------------- + // Transient properties + // ------------------------------------------------------------------------- + + @JsonProperty private final String contextPath; + @JsonProperty private final String userAgent; + + // ------------------------------------------------------------------------- + // Volatile properties + // ------------------------------------------------------------------------- + + @JsonProperty private final String calendar; + @JsonProperty private final String dateFormat; + @JsonProperty private final Date serverDate; + @JsonProperty private final String serverTimeZoneId; + @JsonProperty private final String serverTimeZoneDisplayName; + @JsonProperty private final Date lastAnalyticsTableSuccess; + @JsonProperty private final String intervalSinceLastAnalyticsTableSuccess; + @JsonProperty private final String lastAnalyticsTableRuntime; + @JsonProperty private final Date lastSystemMonitoringSuccess; + @JsonProperty private final Date lastAnalyticsTablePartitionSuccess; + @JsonProperty private final String intervalSinceLastAnalyticsTablePartitionSuccess; + @JsonProperty private final String lastAnalyticsTablePartitionRuntime; + @JsonProperty private final DatabaseInfo databaseInfo; + + // ------------------------------------------------------------------------- + // Stable properties + // ------------------------------------------------------------------------- + + @JsonProperty private final String version; + @JsonProperty private final String revision; + @JsonProperty private final Date buildTime; + @JsonProperty private final String jasperReportsVersion; + @JsonProperty private final String environmentVariable; + @JsonProperty private final String fileStoreProvider; + @JsonProperty private final String readOnlyMode; + @JsonProperty private final String nodeId; + @JsonProperty private final String javaVersion; + @JsonProperty private final String javaVendor; + @JsonProperty private final String javaOpts; + @JsonProperty private final String osName; + @JsonProperty private final String osArchitecture; + @JsonProperty private final String osVersion; + @JsonProperty private final String externalDirectory; + @JsonProperty private final Integer readReplicaCount; + @JsonProperty private final String memoryInfo; + @JsonProperty private final Integer cpuCores; + @JsonProperty private final boolean encryption; + @JsonProperty private final boolean emailConfigured; + @JsonProperty private final boolean redisEnabled; + @JsonProperty private final String redisHostname; + @JsonProperty private final String systemId; + @JsonProperty private final String systemName; + @JsonProperty private final String systemMetadataVersion; + @JsonProperty private final String instanceBaseUrl; + @JsonProperty private final String systemMonitoringUrl; + @JsonProperty private final String clusterHostname; + @JsonProperty private final Boolean isMetadataVersionEnabled; + @JsonProperty private final Date lastMetadataVersionSyncAttempt; + @JsonProperty private final Boolean isMetadataSyncEnabled; + + /** + * Clears sensitive system info properties. + * + *

Note that {@code systemId} must be present for {@link + * org.hisp.dhis.dxf2.monitoring.MonitoringService} to function. + */ + public SystemInfo withoutSensitiveInfo() { + return toBuilder() + .jasperReportsVersion(null) + .environmentVariable(null) + .fileStoreProvider(null) + .readOnlyMode(null) + .nodeId(null) + .javaVersion(null) + .javaVendor(null) + .javaOpts(null) + .osName(null) + .osArchitecture(null) + .osVersion(null) + .externalDirectory(null) + .readReplicaCount(null) + .memoryInfo(null) + .cpuCores(null) + .systemMonitoringUrl(null) + .encryption(false) + .redisEnabled(false) + .redisHostname(null) + .clusterHostname(null) + .databaseInfo(databaseInfo.withoutSensitiveInfo()) + .build(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemService.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemService.java similarity index 95% rename from dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemService.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemService.java index 4264adcf065f..d890b818e684 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemService.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/SystemService.java @@ -30,6 +30,11 @@ /** * @author Lars Helge Overland */ +@FunctionalInterface public interface SystemService { + + /** + * @return The system info summary for right now + */ SystemInfo getSystemInfo(); } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoFactoryBean.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java similarity index 67% rename from dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoFactoryBean.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java index 20003514ed37..c6805aa90cdf 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoFactoryBean.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java @@ -27,36 +27,33 @@ */ package org.hisp.dhis.system.database; -import static com.google.common.base.Preconditions.checkNotNull; - -import org.springframework.beans.factory.FactoryBean; -import org.springframework.stereotype.Component; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Date; +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * @author Lars Helge Overland */ -@Component -public class DatabaseInfoFactoryBean implements FactoryBean { - private final DatabaseInfoProvider databaseInfoProvider; - - public DatabaseInfoFactoryBean(DatabaseInfoProvider databaseInfoProvider) { - checkNotNull(databaseInfoProvider); - - this.databaseInfoProvider = databaseInfoProvider; - } - - @Override - public DatabaseInfo getObject() { - return databaseInfoProvider.getDatabaseInfo(); - } - - @Override - public Class getObjectType() { - return DatabaseInfo.class; - } - - @Override - public boolean isSingleton() { - return true; +@Getter +@Setter +@Builder(toBuilder = true) +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +@ToString +public final class DatabaseInfo { + + @JsonProperty private final String name; + @JsonProperty private final String user; + @JsonProperty private final String url; + @JsonProperty private final String databaseVersion; + @JsonProperty private final boolean spatialSupport; + @JsonProperty private final Date time; + + public DatabaseInfo withoutSensitiveInfo() { + return toBuilder().name(null).user(null).url(null).databaseVersion(null).build(); } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java similarity index 94% rename from dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java index 3e0b14f5f2cf..75f68a97fb15 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/system/database/DatabaseInfoProvider.java @@ -29,12 +29,13 @@ /** * @author Lars Helge Overland - * @version $Id$ */ +@FunctionalInterface public interface DatabaseInfoProvider { - String ID = DatabaseInfoProvider.class.getName(); DatabaseInfo getDatabaseInfo(); - boolean isInMemory(); + default boolean isInMemory() { + return true; + } } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/ValidationCode.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/tracker/imports/validation/ValidationCode.java similarity index 100% rename from dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/ValidationCode.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/tracker/imports/validation/ValidationCode.java diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/SqlExceptionUtils.java similarity index 50% rename from dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/SqlExceptionUtils.java index 4816c73aea89..5a25c2b4f48d 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/DatabaseInfo.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/SqlExceptionUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2023, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,65 +25,36 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.system.database; +package org.hisp.dhis.util; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.lang.reflect.InvocationTargetException; -import javax.annotation.Nonnull; -import lombok.Getter; +import java.sql.SQLException; +import java.util.Optional; +import lombok.AccessLevel; import lombok.NoArgsConstructor; -import lombok.Setter; -import org.apache.commons.beanutils.BeanUtils; -/** - * @author Lars Helge Overland - */ -@Getter -@Setter -@NoArgsConstructor -public class DatabaseInfo { - @JsonProperty private String name; - - @JsonProperty private String user; - - @JsonProperty private String url; +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class SqlExceptionUtils { + public static final String ERR_MSG_TABLE_NOT_EXISTING = + "Query failed, likely because the requested analytics table does not exist: "; - @JsonProperty private String databaseVersion; + public static final String ERR_MSG_SQL_SYNTAX_ERROR = + "An error occurred during the execution of an analytics query: "; - @JsonProperty private boolean spatialSupport; - - // ------------------------------------------------------------------------- - // Logic - // ------------------------------------------------------------------------- - - public void clearSensitiveInfo() { - this.name = null; - this.user = null; - this.url = null; - this.databaseVersion = null; - } + public static final String ERR_MSG_SILENT_FALLBACK = + "An exception occurred - silently fallback since it's multiple analytics query: "; /** - * @return a cloned instance of this object. + * Utility method to detect if the {@link SQLException} refers to a missing relation in the + * database. + * + * @param ex a {@link SQLException} to analyze + * @return true if the error is a missing relation error, false otherwise */ - @Nonnull - public DatabaseInfo instance() { - final DatabaseInfo cloned = new DatabaseInfo(); - try { - BeanUtils.copyProperties(cloned, this); - } catch (IllegalAccessException | InvocationTargetException ex) { - throw new IllegalStateException(ex); + public static boolean relationDoesNotExist(SQLException ex) { + if (ex != null) { + return Optional.of(ex).map(SQLException::getSQLState).filter("42P01"::equals).isPresent(); } - return cloned; - } - - // ------------------------------------------------------------------------- - // toString - // ------------------------------------------------------------------------- - - @Override - public String toString() { - return "[Name: " + name + ", User: " + user + ", URL: " + url + "]"; + return false; } } diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/ApiTokenAuthTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/ApiTokenAuthTest.java index 49f00de5dde9..b679b37a2d59 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/ApiTokenAuthTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/ApiTokenAuthTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.eventhook.targets.auth; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.hisp.dhis.common.auth.ApiTokenAuth; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/HttpBasicAuthTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/HttpBasicAuthTest.java index 2ecb8fbeefac..b0a7e770e2c1 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/HttpBasicAuthTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/eventhook/targets/auth/HttpBasicAuthTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.eventhook.targets.auth; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.hisp.dhis.common.auth.HttpBasicAuth; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitTest.java b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitTest.java index 66fdcd6a0bea..4ace6f72f0e3 100644 --- a/dhis-2/dhis-api/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitTest.java +++ b/dhis-2/dhis-api/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitTest.java @@ -171,6 +171,7 @@ void testIsDescendant() { assertTrue(unitC.isDescendant(Set.of(unitA, unitD))); assertTrue(unitB.isDescendant(Set.of(unitA))); assertTrue(unitB.isDescendant(Set.of(unitA, unitD))); + assertTrue(unitD.isDescendant(Set.of(unitA, unitB, unitC))); assertFalse(unitC.isDescendant(Set.of(unitD))); assertFalse(unitB.isDescendant(Set.of(unitD))); diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/merge/orgunit/OrgUnitMergeValidator.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/merge/orgunit/OrgUnitMergeValidator.java index d483749443a2..7c557963ee36 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/merge/orgunit/OrgUnitMergeValidator.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/merge/orgunit/OrgUnitMergeValidator.java @@ -74,7 +74,7 @@ public ErrorMessage validateForErrorMessage(OrgUnitMergeRequest request) { if (request.getSources().contains(request.getTarget())) { return new ErrorMessage(ErrorCode.E1502); } - if (organisationUnitService.isDescendant(request.getTarget(), request.getSources())) { + if (request.getTarget().isDescendant(request.getSources())) { return new ErrorMessage(ErrorCode.E1504); } diff --git a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/split/orgunit/OrgUnitSplitValidator.java b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/split/orgunit/OrgUnitSplitValidator.java index c89250939dca..278c75dd3219 100644 --- a/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/split/orgunit/OrgUnitSplitValidator.java +++ b/dhis-2/dhis-services/dhis-service-administration/src/main/java/org/hisp/dhis/split/orgunit/OrgUnitSplitValidator.java @@ -83,7 +83,7 @@ public ErrorMessage validateForErrorMessage(OrgUnitSplitRequest request) { return new ErrorMessage(ErrorCode.E1514); } for (OrganisationUnit target : request.getTargets()) { - if (organisationUnitService.isDescendant(target, request.getSource())) { + if (target.isDescendant(request.getSource())) { return new ErrorMessage(ErrorCode.E1516, target.getUid()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableUpdateParams.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableUpdateParams.java index abf220d86778..b5b20eb7b731 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableUpdateParams.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableUpdateParams.java @@ -55,7 +55,10 @@ public class AnalyticsTableUpdateParams { private Integer lastYears; /** Indicates whether to skip update of resource tables. */ - boolean skipResourceTables; + private boolean skipResourceTables; + + /** Indicates whether to skip update of analytics tables, outliers stats columns. */ + private boolean skipOutliers; /** Analytics table types to skip. */ private Set skipTableTypes = new HashSet<>(); @@ -101,6 +104,10 @@ public boolean isSkipResourceTables() { return skipResourceTables; } + public boolean isSkipOutliers() { + return skipOutliers; + } + public Set getSkipTableTypes() { return skipTableTypes; } @@ -149,6 +156,7 @@ public String toString() { .add("skip resource tables", skipResourceTables) .add("skip table types", skipTableTypes) .add("skip programs", skipPrograms) + .add("skip outliers statistics", skipOutliers) .add("start time", DateUtils.getLongDateString(startTime)) .toString(); } @@ -186,6 +194,7 @@ public AnalyticsTableUpdateParams instance() { params.lastYears = this.lastYears; params.skipResourceTables = this.skipResourceTables; + params.skipOutliers = this.skipOutliers; params.skipTableTypes = new HashSet<>(this.skipTableTypes); params.skipPrograms = new HashSet<>(this.skipPrograms); params.jobId = this.jobId; @@ -230,6 +239,11 @@ public Builder withSkipResourceTables(boolean skipResourceTables) { return this; } + public Builder withSkipOutliers(boolean skipOutliers) { + this.params.skipOutliers = skipOutliers; + return this; + } + public Builder withSkipTableTypes(Set skipTableTypes) { this.params.skipTableTypes = skipTableTypes; return this; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/NotLikeConditionRenderer.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/NotLikeConditionRenderer.java index 169c233d47d6..bb4d49c30407 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/NotLikeConditionRenderer.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/common/query/NotLikeConditionRenderer.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.analytics.common.query; -import static org.hisp.dhis.common.QueryOperator.*; +import static org.hisp.dhis.common.QueryOperator.NLIKE; public class NotLikeConditionRenderer extends AbstractLikeConditionRenderer { private NotLikeConditionRenderer(Renderable field, Renderable value) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java index bd3c2a0f9b57..1a41a7e6eb23 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java @@ -47,15 +47,15 @@ import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteAliasCommaSeparate; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quoteWithFunction; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quotedListOf; -import static org.hisp.dhis.analytics.util.AnalyticsUtils.ERR_MSG_SILENT_FALLBACK; import static org.hisp.dhis.analytics.util.AnalyticsUtils.throwIllegalQueryEx; import static org.hisp.dhis.analytics.util.AnalyticsUtils.withExceptionHandling; import static org.hisp.dhis.common.DimensionalObject.DIMENSION_SEP; import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids; import static org.hisp.dhis.commons.collection.CollectionUtils.concat; import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString; -import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist; import static org.hisp.dhis.util.DateUtils.getMediumDateString; +import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_SILENT_FALLBACK; +import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/NestedIndicatorCyclicDependencyInspector.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/NestedIndicatorCyclicDependencyInspector.java index 74b4e69750b2..838de2c32387 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/NestedIndicatorCyclicDependencyInspector.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/NestedIndicatorCyclicDependencyInspector.java @@ -28,7 +28,7 @@ package org.hisp.dhis.analytics.data; import static java.lang.String.format; -import static java.util.Collections.*; +import static java.util.Collections.emptyList; import static org.hisp.dhis.common.DimensionItemType.INDICATOR; import static org.hisp.dhis.common.DimensionalObjectUtils.asTypedList; import static org.hisp.dhis.expression.ParseType.INDICATOR_EXPRESSION; @@ -40,7 +40,10 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; -import org.hisp.dhis.common.*; +import org.hisp.dhis.common.CyclicReferenceException; +import org.hisp.dhis.common.DimensionService; +import org.hisp.dhis.common.DimensionalItemId; +import org.hisp.dhis.common.DimensionalItemObject; import org.hisp.dhis.expression.ExpressionService; import org.hisp.dhis.indicator.Indicator; import org.springframework.stereotype.Component; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java index 071e76fc4b69..17254fe66451 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/AbstractAnalyticsService.java @@ -29,7 +29,9 @@ import static java.util.Collections.emptyList; import static java.util.Optional.empty; -import static java.util.stream.Collectors.*; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.joinWith; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java index 9743044c660b..6be654aaedcf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java @@ -93,7 +93,7 @@ import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.legend.Legend; import org.hisp.dhis.option.Option; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; import org.hisp.dhis.user.CurrentUserService; @@ -183,7 +183,7 @@ public class DefaultEventAnalyticsService extends AbstractAnalyticsService private final EventQueryPlanner queryPlanner; - private final DatabaseInfo databaseInfo; + private final boolean spatialSupport; private final AnalyticsCache analyticsCache; @@ -195,7 +195,7 @@ public DefaultEventAnalyticsService( AnalyticsSecurityManager securityManager, EventQueryPlanner queryPlanner, EventQueryValidator queryValidator, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, AnalyticsCache analyticsCache, EnrollmentAnalyticsManager enrollmentAnalyticsManager, SchemeIdResponseMapper schemeIdResponseMapper, @@ -207,7 +207,7 @@ public DefaultEventAnalyticsService( checkNotNull(eventAnalyticsManager); checkNotNull(eventDataQueryService); checkNotNull(queryPlanner); - checkNotNull(databaseInfo); + checkNotNull(databaseInfoProvider); checkNotNull(analyticsCache); checkNotNull(schemeIdResponseMapper); @@ -216,7 +216,7 @@ public DefaultEventAnalyticsService( this.eventAnalyticsManager = eventAnalyticsManager; this.eventDataQueryService = eventDataQueryService; this.queryPlanner = queryPlanner; - this.databaseInfo = databaseInfo; + this.spatialSupport = databaseInfoProvider.getDatabaseInfo().isSpatialSupport(); this.analyticsCache = analyticsCache; this.enrollmentAnalyticsManager = enrollmentAnalyticsManager; } @@ -641,7 +641,7 @@ public Grid getEvents(EventQueryParams params) { @Override public Grid getEventClusters(EventQueryParams params) { - if (!databaseInfo.isSpatialSupport()) { + if (!spatialSupport) { throwIllegalQueryEx(ErrorCode.E7218); } @@ -679,7 +679,7 @@ public Grid getEventClusters(EventQueryParams params) { @Override public Rectangle getRectangle(EventQueryParams params) { - if (!databaseInfo.isSpatialSupport()) { + if (!spatialSupport) { throwIllegalQueryEx(ErrorCode.E7218); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultQueryItemLocator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultQueryItemLocator.java index 1346eba19c57..a5ec9ea522af 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultQueryItemLocator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultQueryItemLocator.java @@ -34,7 +34,11 @@ import static org.hisp.dhis.common.DimensionalObject.ITEM_SEP; import static org.hisp.dhis.feedback.ErrorCode.E7224; -import java.util.*; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java index 2faf1cd85cdb..8f8a918050df 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractEventJdbcTableManager.java @@ -52,7 +52,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.springframework.jdbc.core.JdbcTemplate; @@ -70,7 +70,7 @@ public AbstractEventJdbcTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -84,7 +84,7 @@ public AbstractEventJdbcTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); @@ -120,7 +120,7 @@ protected String getSelectClause(ValueType valueType, String columnName) { + " = 'false' then 0 else null end"; } else if (valueType.isDate()) { return "cast(" + columnName + " as timestamp)"; - } else if (valueType.isGeo() && databaseInfo.isSpatialSupport()) { + } else if (valueType.isGeo() && isSpatialSupport()) { return "ST_GeomFromGeoJSON('{\"type\":\"Point\", \"coordinates\":' || (" + columnName + ") || ', \"crs\":{\"type\":\"name\", \"properties\":{\"name\":\"EPSG:4326\"}}}')"; @@ -188,8 +188,7 @@ protected List addTrackedEntityAttributes(Program program) List columns = new ArrayList<>(); for (TrackedEntityAttribute attribute : program.getNonConfidentialTrackedEntityAttributes()) { - ColumnDataType dataType = - getColumnType(attribute.getValueType(), databaseInfo.isSpatialSupport()); + ColumnDataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String dataClause = attribute.isNumericType() ? getNumericClause() diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 22279bacd757..b515cf862f6f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -75,7 +75,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.util.DateUtils; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.BadSqlGrammarException; @@ -129,7 +129,9 @@ public abstract class AbstractJdbcTableManager implements AnalyticsTableManager protected final PartitionManager partitionManager; - protected final DatabaseInfo databaseInfo; + private final DatabaseInfoProvider databaseInfoProvider; + + protected Boolean spatialSupport; protected final JdbcTemplate jdbcTemplate; @@ -139,6 +141,12 @@ public abstract class AbstractJdbcTableManager implements AnalyticsTableManager private static final String WITH_AUTOVACUUM_ENABLED_FALSE = "with(autovacuum_enabled = false)"; + protected boolean isSpatialSupport() { + if (spatialSupport == null) + spatialSupport = databaseInfoProvider.getDatabaseInfo().isSpatialSupport(); + return spatialSupport; + } + // ------------------------------------------------------------------------- // Implementation // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 777c67382b6d..4d0d1dbf4f4f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -149,8 +149,6 @@ public void generateResourceTables(JobProgress progress) { // ------------------------------------------------------------------------- private void generateResourceTablesInternal(JobProgress progress) { - final Date startTime = new Date(); - resourceTableService.dropAllSqlViews(progress); Map generators = new LinkedHashMap<>(); @@ -184,6 +182,6 @@ private void generateResourceTablesInternal(JobProgress progress) { resourceTableService.createAllSqlViews(progress); systemSettingManager.saveSystemSetting( - SettingKey.LAST_SUCCESSFUL_RESOURCE_TABLES_UPDATE, startTime); + SettingKey.LAST_SUCCESSFUL_RESOURCE_TABLES_UPDATE, new Date()); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 40752f7591db..bb20119e464d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -32,6 +32,7 @@ import static org.hisp.dhis.analytics.ColumnDataType.INTEGER; import static org.hisp.dhis.analytics.ColumnDataType.TEXT; import static org.hisp.dhis.analytics.ColumnDataType.TIMESTAMP; +import static org.hisp.dhis.analytics.ColumnDataType.VARCHAR_255; import static org.hisp.dhis.analytics.ColumnNotNullConstraint.NOT_NULL; import static org.hisp.dhis.analytics.table.PartitionUtils.getLatestTablePartition; import static org.hisp.dhis.analytics.util.AnalyticsSqlUtils.quote; @@ -76,7 +77,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.system.util.MathUtils; import org.hisp.dhis.util.DateUtils; import org.hisp.dhis.util.ObjectUtils; @@ -128,7 +129,7 @@ public JdbcAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -142,7 +143,7 @@ public JdbcAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); @@ -330,7 +331,7 @@ private void populateTable( String sql = "insert into " + partition.getTempTableName() + " ("; - List columns = getDimensionColumns(partition.getYear()); + List columns = getDimensionColumns(partition.getYear(), params); List values = partition.getMasterTable().getValueColumns(); validateDimensionColumns(columns); @@ -356,6 +357,7 @@ private void populateTable( + "from datavalue dv " + "inner join period pe on dv.periodid=pe.periodid " + "inner join _periodstructure ps on dv.periodid=ps.periodid " + + "left join periodtype pt on pe.periodtypeid = pt.periodtypeid " + "inner join dataelement de on dv.dataelementid=de.dataelementid " + "inner join _dataelementstructure des on dv.dataelementid = des.dataelementid " + "inner join _dataelementgroupsetstructure degs on dv.dataelementid=degs.dataelementid " @@ -368,8 +370,14 @@ private void populateTable( + "inner join _categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid " + "inner join _categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid " + "inner join _categoryoptioncomboname aon on dv.attributeoptioncomboid=aon.categoryoptioncomboid " - + "inner join _categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid " - + approvalClause + + "inner join _categoryoptioncomboname con on dv.categoryoptioncomboid=con.categoryoptioncomboid "; + + if (!skipOutliers(params)) { + sql += getOutliersJoinStatement(); + } + + sql += + approvalClause + "where de.valuetype in (" + valTypes + ") " @@ -425,10 +433,11 @@ private String getApprovalJoinClause(Integer year) { } private List getDimensionColumns() { - return getDimensionColumns(null); + return getDimensionColumns(null, null); } - private List getDimensionColumns(Integer year) { + private List getDimensionColumns( + Integer year, AnalyticsTableUpdateParams params) { List columns = new ArrayList<>(); String idColAlias = @@ -514,6 +523,9 @@ private List getDimensionColumns(Integer year) { columns.add(new AnalyticsTableColumn(quote("approvallevel"), INTEGER, approvalCol)); columns.addAll(getFixedColumns()); + if (!skipOutliers(params)) { + columns.addAll(getOutlierStatsColumns()); + } return filterDimensionColumns(columns); } @@ -531,6 +543,53 @@ private List getValueColumns() { new AnalyticsTableColumn(quote("textvalue"), TEXT, "textvalue")); } + /** + * Statistical outlier detection involves applying statistical tests or procedures to identify + * extreme values. The extreme data are converted into z scores that tell us how many standard + * deviations away they are from the mean. If a value has a high enough or low enough z score, it + * can be considered an outlier. Z scores can be affected by unusually large or small data values, + * which is why a more robust way to detect outliers can be used (a modified z-score). + * + *

Z-Score (xi – μ) / σ where: xi: A single data value μ: The mean of the dataset σ: The + * standard deviation of the dataset + * + *

Modified z-score = 0.6745(xi – x̃) / MAD where: xi: A single data value x̃: The median of + * the dataset MAD: The median absolute deviation of the dataset 0.6745: conversion factor (0.75 + * percentiles) + * + * @return collection of analytics table columns dedicated to outlier identification. + */ + private List getOutlierStatsColumns() { + return List.of( + new AnalyticsTableColumn(quote("de_uid"), CHARACTER_11, NOT_NULL, "de.uid"), + new AnalyticsTableColumn(quote("coc_uid"), CHARACTER_11, NOT_NULL, "co.uid"), + new AnalyticsTableColumn(quote("aoc_uid"), CHARACTER_11, NOT_NULL, "ao.uid"), + new AnalyticsTableColumn(quote("ou_uid"), CHARACTER_11, NOT_NULL, "ou.uid"), + new AnalyticsTableColumn(quote("dataelementid"), INTEGER, NOT_NULL, "dv.dataelementid") + .withIndexColumns(List.of(quote("dataelementid"))), + new AnalyticsTableColumn(quote("sourceid"), INTEGER, NOT_NULL, "dv.sourceid"), + new AnalyticsTableColumn(quote("periodid"), INTEGER, NOT_NULL, "dv.periodid"), + new AnalyticsTableColumn( + quote("categoryoptioncomboid"), INTEGER, NOT_NULL, "dv.categoryoptioncomboid"), + new AnalyticsTableColumn( + quote("attributeoptioncomboid"), INTEGER, NOT_NULL, "dv.attributeoptioncomboid"), + new AnalyticsTableColumn(quote("de_name"), VARCHAR_255, "de.name"), + new AnalyticsTableColumn(quote("ou_name"), VARCHAR_255, "ou.name"), + new AnalyticsTableColumn(quote("coc_name"), VARCHAR_255, "co.name"), + new AnalyticsTableColumn(quote("aoc_name"), VARCHAR_255, "ao.name"), + new AnalyticsTableColumn(quote("pt_name"), VARCHAR_255, "pt.name"), + new AnalyticsTableColumn(quote("path"), VARCHAR_255, "ou.path"), + // mean + new AnalyticsTableColumn(quote("avg_middle_value"), DOUBLE, "stats.avg_middle_value"), + // median + new AnalyticsTableColumn( + quote("percentile_middle_value"), DOUBLE, "stats.percentile_middle_value"), + // median of absolute deviations "MAD" + new AnalyticsTableColumn(quote("mad"), DOUBLE, "stats.mad"), + // standard deviation + new AnalyticsTableColumn(quote("std_dev"), DOUBLE, "stats.std_dev")); + } + /** * Returns the distinct years which contain data values, relative to the from date in the given * parameters, if it exists. @@ -617,4 +676,88 @@ private boolean isApprovalEnabled(Integer year) { return setting && levels; } } + + /** + * The outlier identification is using z-score and modified z-score. The function is retrieving + * the sql statement for analytics table population (analytics and its partitions). + * + * @return sql statement fraction of statistic basic values for the outlier identification. + */ + private String getOutliersJoinStatement() { + return "left join (select t3.dataelementid, " + + " t3.sourceid, " + + " t3.categoryoptioncomboid, " + + " t3.attributeoptioncomboid, " + // median of absolute deviations "mad" (median(xi - median(xi))) + + " percentile_cont(0.5) " + + " within group (order by abs(t3.value::double precision - t3.percentile_middle_value)) as MAD, " + // mean + + " avg(t3.value::double precision) as avg_middle_value, " + // median of the samples (median(xi)) + + " percentile_cont(0.5) " + + " within group (order by t3.value::double precision) as percentile_middle_value, " + // standard deviation of the normal distribution + + " stddev_pop(t3.value::double precision) as std_dev " + // Table "t3" is the composition of the tables "t2" (median of xi) and "t3" (values xi). + // For Z-Score the mean (avg_middle_value) and standard deviation (std_dev) is used ((xi - + // mean(x))/std_dev). + // For modified Z-Score the median (percentile_middle_value) and the median of absolute + // deviations (mad) is used (0.6745*(xi - median(x)/mad)). + // The factor 0.6745 is the 0.75 quartile of the normal distribution, to which the "mad" + // converges to. + + " from (select t1.dataelementid, " + + " t1.sourceid, " + + " t1.categoryoptioncomboid, " + + " t1.attributeoptioncomboid, " + + " t1.percentile_middle_value, " + + " t2.value " + // Table "t1" retrieving the median of all data element (dataelementid) values belongs to + // the same organisation (sourceid) + // coc and aoc. + + " from (select dv1.dataelementid as dataelementid, " + + " dv1.sourceid as sourceid, " + + " dv1.categoryoptioncomboid as categoryoptioncomboid, " + + " dv1.attributeoptioncomboid as attributeoptioncomboid, " + // median + + " percentile_cont(0.5) " + + " within group (order by dv1.value::double precision) as percentile_middle_value " + + " from datavalue dv1 " + + " inner join period pe on dv1.periodid = pe.periodid " + + " inner join organisationunit ou on dv1.sourceid = ou.organisationunitid " + // Only numeric values (value is varchar or string) can be used for stats calculation. + + " where dv1.value ~ '^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?$' " + + " group by dv1.dataelementid, dv1.sourceid, dv1.categoryoptioncomboid, " + + " dv1.attributeoptioncomboid) t1 " + + " join " + // Table "t2" is the complement of the t1 table. It contains all values belong to the + // specific median (see t1). + // To "group by" criteria is added the time dimension (periodid). This part of the query has + // to be verified (maybe add tei to aggregation criteria). + + " (select dv1.dataelementid as dataelementid, " + + " dv1.sourceid as sourceid, " + + " dv1.categoryoptioncomboid as categoryoptioncomboid, " + + " dv1.attributeoptioncomboid as attributeoptioncomboid, " + + " dv1.value, " + + " dv1.periodid " + + " from datavalue dv1 " + + " inner join period pe on dv1.periodid = pe.periodid " + + " inner join organisationunit ou on dv1.sourceid = ou.organisationunitid " + // Only numeric values (varchars) can be used for stats calculation. + + " where dv1.value ~ '^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?$' " + + " group by dv1.dataelementid, dv1.sourceid, dv1.categoryoptioncomboid, " + + " dv1.attributeoptioncomboid, dv1.value, dv1.periodid) t2 " + + " on t1.sourceid = t2.sourceid " + + " and t1.categoryoptioncomboid = t2.categoryoptioncomboid " + + " and t1.attributeoptioncomboid = t2.attributeoptioncomboid " + + " and t1.dataelementid = t2.dataelementid) as t3 " + + " group by t3.dataelementid, t3.sourceid, t3.categoryoptioncomboid, " + + " t3.attributeoptioncomboid) as stats " + + " on dv.dataelementid = stats.dataelementid and dv.sourceid = stats.sourceid and " + + " dv.categoryoptioncomboid = stats.categoryoptioncomboid and " + + " dv.attributeoptioncomboid = stats.attributeoptioncomboid "; + } + + private boolean skipOutliers(AnalyticsTableUpdateParams params) { + return params != null && params.isSkipOutliers(); + } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index bd282659a920..a59e05038d51 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -65,7 +65,7 @@ import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.util.DateUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; @@ -87,7 +87,7 @@ public JdbcCompletenessTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -101,7 +101,7 @@ public JdbcCompletenessTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java index 0e73ebeb91cc..d32cec720ecd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java @@ -61,7 +61,7 @@ import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -91,7 +91,7 @@ public JdbcCompletenessTargetTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -105,7 +105,7 @@ public JdbcCompletenessTargetTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java index be49e8fb3cb8..9c93d7109460 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManager.java @@ -62,7 +62,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -83,7 +83,7 @@ public JdbcEnrollmentAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -97,7 +97,7 @@ public JdbcEnrollmentAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4e1d88945055..67b731a4ac25 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -82,7 +82,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.util.DateUtils; import org.springframework.beans.factory.annotation.Qualifier; @@ -113,7 +113,7 @@ public JdbcEventAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -127,7 +127,7 @@ public JdbcEventAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); @@ -242,7 +242,7 @@ public List getAnalyticsTables(AnalyticsTableUpdateParams params log.info( format( "Get tables using earliest: %s, spatial support: %b", - params.getFromDate(), databaseInfo.isSpatialSupport())); + params.getFromDate(), isSpatialSupport())); List availableDataYears = periodDataProvider.getAvailableYears( @@ -594,8 +594,7 @@ private List getColumnFromTrackedEntityAttribute( boolean withLegendSet) { List columns = new ArrayList<>(); - ColumnDataType dataType = - getColumnType(attribute.getValueType(), databaseInfo.isSpatialSupport()); + ColumnDataType dataType = getColumnType(attribute.getValueType(), isSpatialSupport()); String dataClause = attribute.isNumericType() ? numericClause : attribute.isDateType() ? dateClause : ""; String select = getSelectClause(attribute.getValueType(), "value"); @@ -651,8 +650,7 @@ private List getColumnFromDataElement( DataElement dataElement, boolean withLegendSet) { List columns = new ArrayList<>(); - ColumnDataType dataType = - getColumnType(dataElement.getValueType(), databaseInfo.isSpatialSupport()); + ColumnDataType dataType = getColumnType(dataElement.getValueType(), isSpatialSupport()); String dataClause = getDataClause(dataElement.getUid(), dataElement.getValueType()); String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; String select = getSelectClause(dataElement.getValueType(), columnName); @@ -676,7 +674,7 @@ private List getColumnsFromOrgUnitTrackedEntityAttribute( TrackedEntityAttribute attribute, String dataClause) { List columns = new ArrayList<>(); - if (databaseInfo.isSpatialSupport()) { + if (isSpatialSupport()) { String geoSql = selectForInsert( attribute, @@ -708,7 +706,7 @@ private List getColumnFromOrgUnitDataElement( String columnName = "eventdatavalues #>> '{" + dataElement.getUid() + ", value}'"; - if (databaseInfo.isSpatialSupport()) { + if (isSpatialSupport()) { String geoSql = selectForInsert( dataElement, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java index 7a876be2fa0d..eca734e83e5f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java @@ -56,7 +56,7 @@ import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -77,7 +77,7 @@ public JdbcOrgUnitTargetTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -91,7 +91,7 @@ public JdbcOrgUnitTargetTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java index 6c8c718a1ec4..dc42086605bd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManager.java @@ -64,7 +64,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.quick.JdbcConfiguration; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; @@ -97,7 +97,7 @@ public JdbcOwnershipAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, JdbcConfiguration jdbcConfiguration, AnalyticsExportSettings analyticsExportSettings, @@ -112,7 +112,7 @@ public JdbcOwnershipAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java index 10cd0dea7a88..66a50bd908b3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiAnalyticsTableManager.java @@ -77,7 +77,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; import org.hisp.dhis.trackedentity.TrackedEntityType; @@ -107,7 +107,7 @@ public JdbcTeiAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, TrackedEntityAttributeService trackedEntityAttributeService, @@ -123,7 +123,7 @@ public JdbcTeiAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, settings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java index 1a67795566b0..9c614f666b46 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEnrollmentsAnalyticsTableManager.java @@ -68,7 +68,7 @@ import org.hisp.dhis.period.PeriodDataProvider; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityTypeService; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; @@ -89,7 +89,7 @@ public JdbcTeiEnrollmentsAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, AnalyticsExportSettings settings, @@ -104,7 +104,7 @@ public JdbcTeiEnrollmentsAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, settings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java index 8790957e2988..724cd1c30302 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcTeiEventsAnalyticsTableManager.java @@ -78,7 +78,7 @@ import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityType; import org.hisp.dhis.trackedentity.TrackedEntityTypeService; import org.springframework.beans.factory.annotation.Qualifier; @@ -102,7 +102,7 @@ public JdbcTeiEventsAnalyticsTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, TrackedEntityTypeService trackedEntityTypeService, AnalyticsExportSettings settings, @@ -117,7 +117,7 @@ public JdbcTeiEventsAnalyticsTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, settings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index b7a3a1aeec12..c021db3d7ba3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -63,7 +63,7 @@ import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.util.DateUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; @@ -91,7 +91,7 @@ public JdbcValidationResultTableManager( AnalyticsTableHookService tableHookService, StatementBuilder statementBuilder, PartitionManager partitionManager, - DatabaseInfo databaseInfo, + DatabaseInfoProvider databaseInfoProvider, @Qualifier("analyticsJdbcTemplate") JdbcTemplate jdbcTemplate, AnalyticsExportSettings analyticsExportSettings, PeriodDataProvider periodDataProvider) { @@ -105,7 +105,7 @@ public JdbcValidationResultTableManager( tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/AnalyticsTableJob.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/AnalyticsTableJob.java index 2e57c255fd1e..fe4ff25fac0c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/AnalyticsTableJob.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/AnalyticsTableJob.java @@ -60,6 +60,7 @@ public void execute(JobConfiguration jobConfiguration, JobProgress progress) { AnalyticsTableUpdateParams.newBuilder() .withLastYears(parameters.getLastYears()) .withSkipResourceTables(parameters.isSkipResourceTables()) + .withSkipOutliers(parameters.isSkipOutliers()) .withSkipTableTypes(parameters.getSkipTableTypes()) .withSkipPrograms(parameters.getSkipPrograms()) .withJobId(jobConfiguration) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJob.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJob.java index 50f3e0bee6be..a5b7d6c146c5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJob.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/scheduling/ContinuousAnalyticsTableJob.java @@ -103,6 +103,7 @@ public void execute(JobConfiguration jobConfiguration, JobProgress progress) { AnalyticsTableUpdateParams.newBuilder() .withLastYears(parameters.getLastYears()) .withSkipResourceTables(false) + .withSkipOutliers(parameters.getSkipOutliers()) .withSkipTableTypes(parameters.getSkipTableTypes()) .withJobId(jobConfiguration) .withStartTime(now) @@ -122,6 +123,7 @@ public void execute(JobConfiguration jobConfiguration, JobProgress progress) { AnalyticsTableUpdateParams.newBuilder() .withLatestPartition() .withSkipResourceTables(true) + .withSkipOutliers(parameters.getSkipOutliers()) .withSkipTableTypes(parameters.getSkipTableTypes()) .withJobId(jobConfiguration) .withStartTime(now) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java index 7a02d97d90b9..58e52f063a4e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/util/AnalyticsUtils.java @@ -35,12 +35,15 @@ import static org.hisp.dhis.common.DimensionalObject.ORGUNIT_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.PERIOD_DIM_ID; import static org.hisp.dhis.common.DimensionalObject.QUERY_MODS_ID_SEPARATOR; -import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.relationDoesNotExist; import static org.hisp.dhis.expression.ExpressionService.SYMBOL_WILDCARD; import static org.hisp.dhis.feedback.ErrorCode.E7131; import static org.hisp.dhis.feedback.ErrorCode.E7132; import static org.hisp.dhis.system.util.MathUtils.getRounded; import static org.hisp.dhis.util.DateUtils.getMediumDateString; +import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_SILENT_FALLBACK; +import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_SQL_SYNTAX_ERROR; +import static org.hisp.dhis.util.SqlExceptionUtils.ERR_MSG_TABLE_NOT_EXISTING; +import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist; import static org.springframework.util.Assert.isTrue; import com.google.common.collect.Lists; @@ -124,15 +127,6 @@ public class AnalyticsUtils { private static final Pattern OU_LEVEL_PATTERN = Pattern.compile(DataQueryParams.PREFIX_ORG_UNIT_LEVEL + "(\\d+)"); - public static final String ERR_MSG_TABLE_NOT_EXISTING = - "Query failed, likely because the requested analytics table does not exist: "; - - public static final String ERR_MSG_SQL_SYNTAX_ERROR = - "An error occurred during the execution of an analytics query: "; - - public static final String ERR_MSG_SILENT_FALLBACK = - "An exception occurred - silently fallback since it's multiple analytics query: "; - /** * Returns an SQL statement for retrieving raw data values for an aggregate query. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/BinaryConditionRendererTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/BinaryConditionRendererTest.java index fbe7ebea928c..f789b0282231 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/BinaryConditionRendererTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/common/query/BinaryConditionRendererTest.java @@ -28,9 +28,20 @@ package org.hisp.dhis.analytics.common.query; import static org.hisp.dhis.analytics.common.query.Field.of; -import static org.hisp.dhis.common.QueryOperator.*; +import static org.hisp.dhis.common.QueryOperator.EQ; +import static org.hisp.dhis.common.QueryOperator.GE; +import static org.hisp.dhis.common.QueryOperator.GT; +import static org.hisp.dhis.common.QueryOperator.ILIKE; +import static org.hisp.dhis.common.QueryOperator.IN; import static org.hisp.dhis.common.QueryOperator.LE; -import static org.junit.jupiter.api.Assertions.*; +import static org.hisp.dhis.common.QueryOperator.LIKE; +import static org.hisp.dhis.common.QueryOperator.LT; +import static org.hisp.dhis.common.QueryOperator.NEQ; +import static org.hisp.dhis.common.QueryOperator.NILIKE; +import static org.hisp.dhis.common.QueryOperator.NLIKE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java index 6425a61f8689..7f427d2648dc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/data/DimensionalObjectProducerTest.java @@ -38,13 +38,21 @@ import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnitGroup; import static org.hisp.dhis.analytics.AnalyticsFinancialYearStartKey.FINANCIAL_YEAR_APRIL; import static org.hisp.dhis.analytics.DataQueryParams.DYNAMIC_DIM_CLASSES; -import static org.hisp.dhis.common.DimensionType.*; +import static org.hisp.dhis.common.DimensionType.CATEGORY; +import static org.hisp.dhis.common.DimensionType.DATA_X; +import static org.hisp.dhis.common.DimensionType.ORGANISATION_UNIT; +import static org.hisp.dhis.common.DimensionType.ORGANISATION_UNIT_GROUP; +import static org.hisp.dhis.common.DimensionType.PERIOD; import static org.hisp.dhis.common.DisplayProperty.SHORTNAME; import static org.hisp.dhis.common.IdScheme.NAME; import static org.hisp.dhis.common.IdScheme.UID; import static org.hisp.dhis.feedback.ErrorCode.E7124; import static org.hisp.dhis.setting.SettingKey.ANALYTICS_FINANCIAL_YEAR_START; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.when; @@ -59,8 +67,15 @@ import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.category.Category; import org.hisp.dhis.category.CategoryOption; -import org.hisp.dhis.common.*; +import org.hisp.dhis.common.BaseDimensionalItemObject; +import org.hisp.dhis.common.BaseDimensionalObject; +import org.hisp.dhis.common.DimensionItemKeywords; import org.hisp.dhis.common.DimensionItemKeywords.Keyword; +import org.hisp.dhis.common.DimensionService; +import org.hisp.dhis.common.DimensionalItemObject; +import org.hisp.dhis.common.DisplayProperty; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementGroup; import org.hisp.dhis.i18n.I18n; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java index cb6d6f0c2ae5..09588bbab0a8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/AbstractJdbcEventAnalyticsManagerTest.java @@ -50,7 +50,9 @@ import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.AGGREGATE; import static org.hisp.dhis.common.RequestTypeAware.EndpointAction.QUERY; import static org.hisp.dhis.common.RequestTypeAware.EndpointItem.ENROLLMENT; -import static org.hisp.dhis.common.ValueType.*; +import static org.hisp.dhis.common.ValueType.BOOLEAN; +import static org.hisp.dhis.common.ValueType.NUMBER; +import static org.hisp.dhis.common.ValueType.TEXT; import static org.hisp.dhis.period.RelativePeriodEnum.THIS_YEAR; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java index 520e4a0acf1e..7ff07fdcdc2b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsServiceTest.java @@ -54,6 +54,7 @@ import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.program.Program; import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; import org.hisp.dhis.user.CurrentUserService; import org.junit.jupiter.api.BeforeEach; @@ -87,7 +88,7 @@ class DefaultEventAnalyticsServiceTest { @Mock private EventQueryPlanner queryPlanner; - @Mock private DatabaseInfo databaseInfo; + @Mock private DatabaseInfoProvider databaseInfoProvider; @Mock private AnalyticsCache analyticsCache; @@ -97,6 +98,7 @@ class DefaultEventAnalyticsServiceTest { @BeforeEach public void setUp() { + when(databaseInfoProvider.getDatabaseInfo()).thenReturn(DatabaseInfo.builder().build()); defaultEventAnalyticsService = new DefaultEventAnalyticsService( dataElementService, @@ -106,7 +108,7 @@ public void setUp() { securityManager, queryPlanner, eventQueryValidator, - databaseInfo, + databaseInfoProvider, analyticsCache, enrollmentAnalyticsManager, schemeIdResponseMapper, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java index a84a695178fe..bebcb0917224 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java @@ -54,7 +54,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -96,7 +96,7 @@ public void setUp() { mock(AnalyticsTableHookService.class), mock(StatementBuilder.class), mock(PartitionManager.class), - mock(DatabaseInfo.class), + mock(DatabaseInfoProvider.class), jdbcTemplate, analyticsExportSettings, periodDataProvider); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java index 2f8887802184..b6db92b35d3c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEnrollmentAnalyticsTableManagerTest.java @@ -54,6 +54,7 @@ import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; @@ -71,7 +72,7 @@ class JdbcEnrollmentAnalyticsTableManagerTest { @Mock private IdentifiableObjectManager idObjectManager; - @Mock private DatabaseInfo databaseInfo; + @Mock private DatabaseInfoProvider databaseInfoProvider; @Mock private JdbcTemplate jdbcTemplate; @@ -85,6 +86,7 @@ class JdbcEnrollmentAnalyticsTableManagerTest { @BeforeEach public void setUp() { + when(databaseInfoProvider.getDatabaseInfo()).thenReturn(DatabaseInfo.builder().build()); subject = new JdbcEnrollmentAnalyticsTableManager( idObjectManager, @@ -96,7 +98,7 @@ public void setUp() { mock(AnalyticsTableHookService.class), new PostgreSQLStatementBuilder(), mock(PartitionManager.class), - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); @@ -105,7 +107,8 @@ public void setUp() { @Test void verifyTeiTypeOrgUnitFetchesOuUidWhenPopulatingEventAnalyticsTable() { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program p1 = createProgram('A'); TrackedEntityAttribute tea = createTrackedEntityAttribute('a', ValueType.ORGANISATION_UNIT); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 01eb45e0c725..62e6f4be9cd6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -101,6 +101,7 @@ import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; @@ -129,7 +130,7 @@ class JdbcEventAnalyticsTableManagerTest { @Mock private SystemSettingManager systemSettingManager; - @Mock private DatabaseInfo databaseInfo; + @Mock private DatabaseInfoProvider databaseInfoProvider; @Mock private JdbcTemplate jdbcTemplate; @@ -168,6 +169,7 @@ public void setUp() { today = Date.from(LocalDate.of(2019, 7, 6).atStartOfDay(ZoneId.systemDefault()).toInstant()); + when(databaseInfoProvider.getDatabaseInfo()).thenReturn(DatabaseInfo.builder().build()); subject = new JdbcEventAnalyticsTableManager( idObjectManager, @@ -179,14 +181,10 @@ public void setUp() { mock(AnalyticsTableHookService.class), statementBuilder, mock(PartitionManager.class), - databaseInfo, + databaseInfoProvider, jdbcTemplate, analyticsExportSettings, periodDataProvider); - } - - @Test - void verifyTableType() { assertThat(subject.getAnalyticsTableType(), is(AnalyticsTableType.EVENT)); } @@ -365,7 +363,8 @@ private AnalyticsTableColumn getColumn(String column, AnalyticsTable analyticsTa @Test void verifyGetTableWithDataElements() { - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program program = createProgram('A'); DataElement d1 = createDataElement('Z', ValueType.TEXT, AggregationType.SUM); @@ -480,7 +479,8 @@ void verifyGetTableWithDataElements() { @Test void verifyGetTableWithTrackedEntityAttribute() { - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program program = createProgram('A'); TrackedEntityAttribute tea1 = rnd.nextObject(TrackedEntityAttribute.class); @@ -550,7 +550,8 @@ void verifyGetTableWithTrackedEntityAttribute() { @Test void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program programA = createProgram('A'); DataElement d5 = createDataElement('G', ValueType.ORGANISATION_UNIT, AggregationType.NONE); @@ -598,7 +599,8 @@ void verifyDataElementTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable( @Test void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program programA = createProgram('A'); TrackedEntityAttribute tea = createTrackedEntityAttribute('a', ValueType.ORGANISATION_UNIT); @@ -646,7 +648,8 @@ void verifyTeiTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { void verifyOrgUnitOwnershipJoinsWhenPopulatingEventAnalyticsTable() { // Given fixtures/expectations ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program programA = createProgram('A'); TrackedEntityAttribute tea = createTrackedEntityAttribute('a', ValueType.ORGANISATION_UNIT); @@ -871,7 +874,8 @@ private String quote(String string) { @Test void verifyTeaTypeOrgUnitFetchesOuNameWhenPopulatingEventAnalyticsTable() { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - when(databaseInfo.isSpatialSupport()).thenReturn(true); + when(databaseInfoProvider.getDatabaseInfo()) + .thenReturn(DatabaseInfo.builder().spatialSupport(true).build()); Program programA = createProgram('A'); TrackedEntityAttribute tea = createTrackedEntityAttribute('a', ValueType.ORGANISATION_UNIT); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java index fda17d090c38..386927acf176 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcOwnershipAnalyticsTableManagerTest.java @@ -76,7 +76,7 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.quick.JdbcConfiguration; import org.hisp.quick.StatementDialect; import org.junit.jupiter.api.BeforeEach; @@ -114,7 +114,7 @@ class JdbcOwnershipAnalyticsTableManagerTest extends DhisConvenienceTest { @Mock private PartitionManager partitionManager; - @Mock private DatabaseInfo databaseInfo; + @Mock private DatabaseInfoProvider databaseInfoProvider; @Mock private JdbcTemplate jdbcTemplate; @@ -157,7 +157,7 @@ public void setUp() { tableHookService, statementBuilder, partitionManager, - databaseInfo, + databaseInfoProvider, jdbcTemplate, jdbcConfiguration, analyticsExportSettings, diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/appmanager/AppMenuManager.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/appmanager/AppMenuManager.java index 0419de39bb5f..c09701fc59eb 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/appmanager/AppMenuManager.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/appmanager/AppMenuManager.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.appmanager; -import java.util.*; +import java.util.List; +import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/DefaultDataApprovalLevelService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/DefaultDataApprovalLevelService.java index 6b6dc9ae4d71..2a1470a6ccc5 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/DefaultDataApprovalLevelService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/DefaultDataApprovalLevelService.java @@ -381,7 +381,7 @@ public DataApprovalLevel getUserApprovalLevel( OrganisationUnit organisationUnit = null; for (OrganisationUnit unit : user.getOrganisationUnits()) { - if (organisationUnitService.isDescendant(orgUnit, unit)) { + if (orgUnit.isDescendant(unit)) { organisationUnit = unit; break; } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java index 36a3535dc2cf..a915de8b61ae 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataapproval/hibernate/HibernateDataApprovalStore.java @@ -63,7 +63,6 @@ import org.hisp.dhis.hibernate.jsonb.type.JsonbFunctions; import org.hisp.dhis.jdbc.StatementBuilder; import org.hisp.dhis.organisationunit.OrganisationUnit; -import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.period.PeriodStore; @@ -110,8 +109,6 @@ public class HibernateDataApprovalStore extends HibernateGenericStore getDataApprovalStatuses( if (orgUnits != null) { for (OrganisationUnit orgUnit : orgUnits) { - if (!organisationUnitService.isDescendant(orgUnit, userOrgUnits)) { + if (!orgUnit.isDescendant(userOrgUnits)) { log.debug("User " + user.getUsername() + " can't see orgUnit " + orgUnit.getName()); return new ArrayList<>(); // Unapprovable. diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationService.java index 19604090fd6c..faba8a5ce4d3 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationService.java @@ -30,7 +30,6 @@ import static java.lang.String.format; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; -import static org.hisp.dhis.program.notification.NotificationTrigger.SCHEDULED_DAYS_DUE_DATE; import static org.hisp.dhis.scheduling.JobProgress.FailurePolicy.SKIP_ITEM_OUTLIER; import com.google.common.base.Function; @@ -149,7 +148,7 @@ public class DefaultDataSetNotificationService implements DataSetNotificationSer @Override public void sendScheduledDataSetNotificationsForDay(Date day, JobProgress progress) { List templates = - dsntService.getScheduledNotifications(SCHEDULED_DAYS_DUE_DATE); + dsntService.getScheduledNotifications(DataSetNotificationTrigger.SCHEDULED_DAYS); if (templates == null || templates.isEmpty()) { log.info("No template found"); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationTemplateService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationTemplateService.java index 2a5bde56c6fc..4df24cadcc5f 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationTemplateService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/DefaultDataSetNotificationTemplateService.java @@ -30,7 +30,6 @@ import java.util.List; import lombok.RequiredArgsConstructor; import org.hisp.dhis.dataset.DataSet; -import org.hisp.dhis.program.notification.NotificationTrigger; import org.springframework.stereotype.Service; /** Created by zubair@dhis2.org on 20.07.17. */ @@ -57,7 +56,8 @@ public List getCompleteNotifications(DataSet dataSe } @Override - public List getScheduledNotifications(NotificationTrigger trigger) { + public List getScheduledNotifications( + DataSetNotificationTrigger trigger) { return store.getScheduledNotifications(trigger); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/HibernateDataSetNotificationTemplateStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/HibernateDataSetNotificationTemplateStore.java index b0a7d5c86248..738469afd01e 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/HibernateDataSetNotificationTemplateStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/dataset/notifications/HibernateDataSetNotificationTemplateStore.java @@ -32,7 +32,6 @@ import javax.persistence.criteria.CriteriaBuilder; import org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore; import org.hisp.dhis.dataset.DataSet; -import org.hisp.dhis.program.notification.NotificationTrigger; import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.user.CurrentUserService; import org.springframework.context.ApplicationEventPublisher; @@ -73,7 +72,8 @@ public List getNotificationsByTriggerType( } @Override - public List getScheduledNotifications(NotificationTrigger trigger) { + public List getScheduledNotifications( + DataSetNotificationTrigger trigger) { CriteriaBuilder builder = getCriteriaBuilder(); return getList( diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemProgramAttribute.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemProgramAttribute.java index 4333f80cc306..11695a5c9d4c 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemProgramAttribute.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/expression/dataitem/DimItemProgramAttribute.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.expression.dataitem; -import static org.hisp.dhis.common.DimensionItemType.*; +import static org.hisp.dhis.common.DimensionItemType.PROGRAM_ATTRIBUTE; import static org.hisp.dhis.parser.expression.ParserUtils.assumeExpressionProgramAttribute; import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java index dac3f37812ff..ce5c5f61cd0e 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/DefaultFileResourceService.java @@ -27,10 +27,6 @@ */ package org.hisp.dhis.fileresource; -import static java.lang.System.currentTimeMillis; -import static java.time.Duration.ofMillis; -import static java.time.Duration.ofSeconds; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -173,7 +169,7 @@ public List findOwnersByStorageKey(@CheckForNull String stora @Override @Transactional - public void saveFileResource(FileResource fileResource, File file) { + public void asyncSaveFileResource(FileResource fileResource, File file) { validateFileResource(fileResource); fileResource.setStorageStatus(FileResourceStorageStatus.PENDING); @@ -194,7 +190,7 @@ public void saveFileResource(FileResource fileResource, File file) { @Override @Transactional - public String saveFileResource(FileResource fileResource, byte[] bytes) { + public String asyncSaveFileResource(FileResource fileResource, byte[] bytes) { fileResource.setStorageStatus(FileResourceStorageStatus.PENDING); fileResourceStore.save(fileResource); entityManager.flush(); @@ -206,6 +202,22 @@ public String saveFileResource(FileResource fileResource, byte[] bytes) { return uid; } + @Override + @Transactional + public String syncSaveFileResource(FileResource fileResource, byte[] bytes) + throws ConflictException { + fileResource.setStorageStatus(FileResourceStorageStatus.PENDING); + fileResourceStore.save(fileResource); + entityManager.flush(); + + final String uid = fileResource.getUid(); + + String storageId = fileResourceContentStore.saveFileResourceContent(fileResource, bytes); + if (storageId == null) throw new ConflictException(ErrorCode.E6102); + + return uid; + } + @Override @Transactional public void deleteFileResource(String uid) { @@ -245,26 +257,9 @@ public void deleteFileResource(FileResource fileResource) { @Override @Nonnull public InputStream getFileResourceContent(FileResource fileResource) throws ConflictException { - return getFileResourceContent(fileResource, ofSeconds(10)); - } - - @Nonnull - @Override - public InputStream getFileResourceContent(FileResource fileResource, java.time.Duration timeout) - throws ConflictException { String key = fileResource.getStorageKey(); InputStream content = fileResourceContentStore.getFileResourceContent(key); - long since = currentTimeMillis(); - while (content == null && !timeout.minus(ofMillis(currentTimeMillis() - since)).isNegative()) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - content = fileResourceContentStore.getFileResourceContent(key); - } - if (content == null) - throw new ConflictException("File resource exists but content input stream was null"); + if (content == null) throw new ConflictException(ErrorCode.E6103); return content; } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java index 0c9ada0ea604..c57dd1b51b7a 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/fileresource/JCloudsFileResourceContentStore.java @@ -37,8 +37,14 @@ import java.io.OutputStream; import java.net.URI; import java.nio.file.Files; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Properties; import java.util.regex.Pattern; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; @@ -49,7 +55,11 @@ import org.hisp.dhis.external.conf.DhisConfigurationProvider; import org.hisp.dhis.external.location.LocationManager; import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.*; +import org.jclouds.blobstore.BlobRequestSigner; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.ContainerNotFoundException; +import org.jclouds.blobstore.LocalBlobRequestSigner; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.internal.RequestSigningUnsupported; import org.jclouds.domain.Credentials; @@ -220,9 +230,28 @@ public long getFileResourceContentLength(String key) { } @Override - public String saveFileResourceContent(FileResource fileResource, byte[] bytes) { - Blob blob = createBlob(fileResource, bytes); + public String saveFileResourceContent(@Nonnull FileResource fr, @Nonnull byte[] bytes) { + return saveFileResourceContent(fr, createBlob(fr, bytes), null); + } + + @Override + public String saveFileResourceContent(@Nonnull FileResource fr, @Nonnull File file) { + return saveFileResourceContent( + fr, + createBlob(fr, StringUtils.EMPTY, file, fr.getContentMd5()), + () -> { + try { + Files.deleteIfExists(file.toPath()); + } catch (IOException ioe) { + log.warn( + String.format("Temporary file '%s' could not be deleted.", file.toPath()), ioe); + } + }); + } + @CheckForNull + private String saveFileResourceContent( + @Nonnull FileResource fr, @CheckForNull Blob blob, @CheckForNull Runnable postPutCallback) { if (blob == null) { return null; } @@ -234,35 +263,18 @@ public String saveFileResourceContent(FileResource fileResource, byte[] bytes) { return null; } - log.debug(String.format("File resource saved with key: %s", fileResource.getStorageKey())); - - return fileResource.getStorageKey(); - } - - @Override - public String saveFileResourceContent(FileResource fileResource, File file) { - Blob blob = createBlob(fileResource, StringUtils.EMPTY, file, fileResource.getContentMd5()); - - if (blob == null) { - return null; - } - - blobStore.putBlob(config.container, blob); - - try { - Files.deleteIfExists(file.toPath()); - } catch (IOException ioe) { - log.warn(String.format("Temporary file '%s' could not be deleted.", file.toPath()), ioe); + if (postPutCallback != null) { + postPutCallback.run(); } - log.debug(String.format("File resource saved with key: %s", fileResource.getStorageKey())); + log.debug(String.format("File resource saved with key: %s", fr.getStorageKey())); - return fileResource.getStorageKey(); + return fr.getStorageKey(); } @Override public String saveFileResourceContent( - FileResource fileResource, Map imageFiles) { + @Nonnull FileResource fr, @Nonnull Map imageFiles) { if (imageFiles.isEmpty()) { return null; } @@ -282,7 +294,7 @@ public String saveFileResourceContent( return null; } - blob = createBlob(fileResource, entry.getKey().getDimension(), file, contentMd5); + blob = createBlob(fr, entry.getKey().getDimension(), file, contentMd5); if (blob != null) { try { @@ -300,7 +312,7 @@ public String saveFileResourceContent( } } - return fileResource.getStorageKey(); + return fr.getStorageKey(); } @Override @@ -371,7 +383,7 @@ private void deleteBlob(String key) { blobStore.removeBlob(config.container, key); } - private Blob createBlob(FileResource fileResource, byte[] bytes) { + private Blob createBlob(@Nonnull FileResource fileResource, @Nonnull byte[] bytes) { return blobStore .blobBuilder(fileResource.getStorageKey()) .payload(bytes) @@ -383,7 +395,10 @@ private Blob createBlob(FileResource fileResource, byte[] bytes) { } private Blob createBlob( - FileResource fileResource, String fileDimension, File file, String contentMd5) { + @Nonnull FileResource fileResource, + String fileDimension, + @Nonnull File file, + @Nonnull String contentMd5) { return blobStore .blobBuilder(StringUtils.join(fileResource.getStorageKey(), fileDimension)) .payload(file) diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java index bd6f9346ab3f..b314231b6e9d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/organisationunit/DefaultOrganisationUnitService.java @@ -41,7 +41,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -49,7 +48,6 @@ import org.hisp.dhis.cache.Cache; import org.hisp.dhis.cache.CacheProvider; import org.hisp.dhis.common.IdentifiableObjectManager; -import org.hisp.dhis.common.IdentifiableObjectUtils; import org.hisp.dhis.common.SortProperty; import org.hisp.dhis.commons.collection.ListUtils; import org.hisp.dhis.commons.filter.FilterUtils; @@ -454,55 +452,13 @@ public boolean isInUserHierarchy(User user, OrganisationUnit organisationUnit) { return false; } - return isDescendant(organisationUnit, user.getOrganisationUnits()); - } - - @Override - @Transactional - public boolean isDescendant(OrganisationUnit organisationUnit, Set ancestors) { - Objects.requireNonNull(organisationUnit); - - if (isEmpty(ancestors)) { - return false; - } - - Set ancestorUids = IdentifiableObjectUtils.getUidsAsSet(ancestors); - - OrganisationUnit unit = getOrganisationUnit(organisationUnit.getUid()); + OrganisationUnit unit = organisationUnitStore.getByUid(organisationUnit.getUid()); if (unit == null) { - unit = organisationUnit; - } - - while (unit != null) { - if (ancestorUids.contains(unit.getUid())) { - return true; - } - - unit = unit.getParent(); - } - - return false; - } - - @Transactional(readOnly = true) - @Override - public boolean isDescendant(OrganisationUnit organisationUnit, OrganisationUnit ancestor) { - if (ancestor == null) { return false; } - OrganisationUnit unit = getOrganisationUnit(organisationUnit.getUid()); - - while (unit != null) { - if (ancestor.equals(unit)) { - return true; - } - - unit = unit.getParent(); - } - - return false; + return unit.isDescendant(user.getOrganisationUnits()); } @Override @@ -524,7 +480,7 @@ public boolean isInUserDataViewHierarchy(User user, OrganisationUnit organisatio return false; } - return isDescendant(organisationUnit, user.getDataViewOrganisationUnitsWithFallback()); + return organisationUnit.isDescendant(user.getDataViewOrganisationUnitsWithFallback()); } @Override @@ -564,7 +520,7 @@ public boolean isInUserSearchHierarchy(User user, OrganisationUnit organisationU return false; } - return isDescendant(organisationUnit, user.getTeiSearchOrganisationUnitsWithFallback()); + return organisationUnit.isDescendant(user.getTeiSearchOrganisationUnitsWithFallback()); } @Override @@ -572,7 +528,7 @@ public boolean isInUserSearchHierarchy(User user, OrganisationUnit organisationU public boolean isInUserHierarchy(String uid, Set organisationUnits) { OrganisationUnit organisationUnit = organisationUnitStore.getByUid(uid); - return organisationUnit != null && isDescendant(organisationUnit, organisationUnits); + return organisationUnit != null && organisationUnit.isDescendant(organisationUnits); } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramElementsAndAttributesCollecter.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramElementsAndAttributesCollecter.java index fe047083fa78..9aaa729a0410 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramElementsAndAttributesCollecter.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/ProgramElementsAndAttributesCollecter.java @@ -28,8 +28,11 @@ package org.hisp.dhis.program; import static com.google.common.base.Preconditions.checkNotNull; -import static org.hisp.dhis.parser.expression.ParserUtils.*; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.*; +import static org.hisp.dhis.parser.expression.ParserUtils.assumeProgramExpressionProgramAttribute; +import static org.hisp.dhis.parser.expression.ParserUtils.assumeStageElementSyntax; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.A_BRACE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.HASH_BRACE; import java.util.Set; import org.hisp.dhis.parser.expression.antlr.ExpressionBaseListener; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/ProgramVariableItem.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/ProgramVariableItem.java index 6b7a61f93993..de9ecf33d361 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/ProgramVariableItem.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/variable/ProgramVariableItem.java @@ -27,7 +27,29 @@ */ package org.hisp.dhis.program.variable; -import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.*; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.ExprContext; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ANALYTICS_PERIOD_END; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ANALYTICS_PERIOD_START; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_COMPLETED_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_CREATION_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_CURRENT_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_DUE_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ENROLLMENT_COUNT; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ENROLLMENT_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ENROLLMENT_STATUS; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_EVENT_COUNT; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_EVENT_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_EVENT_STATUS; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_EXECUTION_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_INCIDENT_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ORG_UNIT_COUNT; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_PROGRAM_STAGE_ID; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_PROGRAM_STAGE_NAME; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_SCHEDULED_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_SCHEDULED_EVENT_COUNT; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_SYNC_DATE; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_TEI_COUNT; +import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_VALUE_COUNT; import static org.hisp.dhis.parser.expression.antlr.ExpressionParser.V_ZERO_POS_VALUE_COUNT; import com.google.common.collect.ImmutableMap; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/operators/TokenOperator.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/operators/TokenOperator.java index e668eab20f1e..f844060f853d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/operators/TokenOperator.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/operators/TokenOperator.java @@ -62,11 +62,27 @@ public Criterion getHibernateCriterion(QueryPath queryPath) { public Predicate getPredicate(CriteriaBuilder builder, Root root, QueryPath queryPath) { String value = caseSensitive ? getValue(String.class) : getValue(String.class).toLowerCase(); + Predicate defaultSearch = + builder.equal( + builder.function( + JsonbFunctions.REGEXP_SEARCH, + Boolean.class, + root.get(queryPath.getPath()), + builder.literal(TokenUtils.createRegex(value).toString())), + true); + + if (queryPath.getLocale() == null + || !queryPath.getProperty().isTranslatable() + || queryPath.getProperty().getTranslationKey() == null) { + return defaultSearch; + } return builder.equal( builder.function( - JsonbFunctions.REGEXP_SEARCH, + JsonbFunctions.SEARCH_TRANSLATION_TOKEN, Boolean.class, - root.get(queryPath.getPath()), + root.get("translations"), + builder.literal("{" + queryPath.getProperty().getTranslationKey() + "}"), + builder.literal(queryPath.getLocale().getLanguage()), builder.literal(TokenUtils.createRegex(value).toString())), true); } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/DefaultQueryPlanner.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/DefaultQueryPlanner.java index 4d590b379348..6dfbce8fdff6 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/DefaultQueryPlanner.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/DefaultQueryPlanner.java @@ -31,20 +31,27 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Locale; import javax.persistence.criteria.Path; import javax.persistence.criteria.Root; import lombok.RequiredArgsConstructor; import org.hisp.dhis.attribute.Attribute; import org.hisp.dhis.common.CodeGenerator; +import org.hisp.dhis.i18n.locale.LocaleManager; import org.hisp.dhis.query.Conjunction; import org.hisp.dhis.query.Criterion; import org.hisp.dhis.query.Disjunction; import org.hisp.dhis.query.Junction; import org.hisp.dhis.query.Query; import org.hisp.dhis.query.Restriction; +import org.hisp.dhis.query.operators.TokenOperator; import org.hisp.dhis.schema.Property; import org.hisp.dhis.schema.Schema; import org.hisp.dhis.schema.SchemaService; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.user.CurrentUserUtil; +import org.hisp.dhis.user.UserSettingKey; import org.springframework.stereotype.Component; /** @@ -54,6 +61,7 @@ @RequiredArgsConstructor public class DefaultQueryPlanner implements QueryPlanner { private final SchemaService schemaService; + private final SystemSettingManager systemSettingManager; @Override public QueryPlan planQuery(Query query) { @@ -202,6 +210,10 @@ private Query getQuery(Query query, boolean persistedOnly) { Restriction restriction = (Restriction) criterion; restriction.setQueryPath(getQueryPath(query.getSchema(), restriction.getPath())); + if (restriction.getOperator().getClass().isAssignableFrom(TokenOperator.class)) { + setQueryPathLocale(restriction); + } + if (restriction.getQueryPath().isPersisted() && !restriction.getQueryPath().haveAlias() && !Attribute.ObjectType.isValidType(restriction.getQueryPath().getPath())) { @@ -247,6 +259,10 @@ private Junction handleJunction(Query query, Junction queryJunction, boolean per Restriction restriction = (Restriction) criterion; restriction.setQueryPath(getQueryPath(query.getSchema(), restriction.getPath())); + if (restriction.getOperator().getClass().isAssignableFrom(TokenOperator.class)) { + setQueryPathLocale(restriction); + } + if (restriction.getQueryPath().isPersisted() && !restriction.getQueryPath().haveAlias(1) && !Attribute.ObjectType.isValidType(restriction.getQueryPath().getPath())) { @@ -270,4 +286,17 @@ private Junction handleJunction(Query query, Junction queryJunction, boolean per private boolean isFilterByAttributeId(Property curProperty, String propertyName) { return curProperty == null && CodeGenerator.isValidUid(propertyName); } + + private void setQueryPathLocale(Restriction restriction) { + Locale systemLocale = + systemSettingManager.getSystemSetting(SettingKey.DB_LOCALE, LocaleManager.DEFAULT_LOCALE); + Locale currentUserLocale = CurrentUserUtil.getUserSetting(UserSettingKey.DB_LOCALE); + if (currentUserLocale != null && !currentUserLocale.equals(systemLocale)) { + // Use translations jsonb column for querying with the current user locale. + restriction.getQueryPath().setLocale(currentUserLocale); + } else { + // Use default properties for querying. Don't use the translations jsonb column. + restriction.getQueryPath().setLocale(null); + } + } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/QueryPath.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/QueryPath.java index 9a6b3f7c272b..9a903c566817 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/QueryPath.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/query/planner/QueryPath.java @@ -30,15 +30,16 @@ import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import java.util.Arrays; -import lombok.AllArgsConstructor; +import java.util.Locale; import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.hisp.dhis.schema.Property; /** * @author Morten Olav Hansen */ @Getter -@AllArgsConstructor +@RequiredArgsConstructor public class QueryPath { private final Property property; @@ -48,6 +49,12 @@ public class QueryPath { private static final Joiner PATH_JOINER = Joiner.on("."); + /** + * If this locale is not null then the query must use the translations jsonb column instead of + * default properties. + */ + private Locale locale; + public QueryPath(Property property, boolean persisted) { this(property, persisted, new String[0]); } @@ -70,6 +77,14 @@ public boolean haveAlias(int n) { return alias != null && alias.length > n; } + public void setLocale(Locale locale) { + this.locale = locale; + } + + public Locale getLocale() { + return locale; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java index 28adb9f5c462..8f46b2a4eb4d 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobConfigurationService.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.scheduling; +import static org.hisp.dhis.jsontree.JsonBuilder.createArray; +import static org.hisp.dhis.scheduling.JobType.TRACKER_IMPORT_JOB; import static org.hisp.dhis.scheduling.JobType.values; import com.fasterxml.jackson.annotation.JsonProperty; @@ -40,6 +42,7 @@ import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -49,7 +52,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -67,11 +69,14 @@ import org.hisp.dhis.fileresource.FileResource; import org.hisp.dhis.fileresource.FileResourceDomain; import org.hisp.dhis.fileresource.FileResourceService; -import org.hisp.dhis.jsontree.*; +import org.hisp.dhis.jsontree.JsonMixed; +import org.hisp.dhis.jsontree.JsonNode; +import org.hisp.dhis.jsontree.JsonObject; import org.hisp.dhis.scheduling.JobType.Defaults; import org.hisp.dhis.schema.Property; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.tracker.imports.validation.ValidationCode; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -124,7 +129,7 @@ private void saveJobData(String uid, MimeType contentType, InputStream content) FileResourceDomain.JOB_DATA); fr.setUid(uid); fr.setAssigned(true); - fileResourceService.saveFileResource(fr, data); + fileResourceService.syncSaveFileResource(fr, data); } catch (IOException ex) { throw new ConflictException("Failed to create job data file resource: " + ex.getMessage()); } @@ -256,48 +261,69 @@ public List getStaleConfigurations(int staleForSeconds) { @Override @Transactional(readOnly = true) public List findJobRunErrors(@Nonnull JobRunErrorsParams params) { - Function toObject = - json -> { - JsonObject obj = JsonMixed.of(json); - List flatErrors = new ArrayList<>(); - JsonObject errors = obj.getObject("errors"); - errors - .node() - .members() - .forEach( - byObject -> - byObject - .getValue() - .members() - .forEach( - byCode -> - byCode - .getValue() - .elements() - .forEach( - error -> { - ErrorCode code = - ErrorCode.valueOf( - JsonMixed.of(error).getString("code").string()); - Object[] args = - JsonMixed.of(error) - .getArray("args") - .stringValues() - .toArray(new String[0]); - String msg = - MessageFormat.format(code.getMessage(), args); - flatErrors.add( - error - .extract() - .addMembers(e -> e.addString("message", msg))); - }))); - return JsonMixed.of( - errors - .node() - .replaceWith( - JsonBuilder.createArray(arr -> flatErrors.forEach(arr::addElement)))); - }; - return jobConfigurationStore.findJobRunErrors(params).map(toObject).toList(); + return jobConfigurationStore + .findJobRunErrors(params) + .map(json -> errorEntryWithMessages(json, params)) + .toList(); + } + + private JsonObject errorEntryWithMessages(String json, JobRunErrorsParams params) { + JsonObject entry = JsonMixed.of(json); + List flatErrors = new ArrayList<>(); + JobType type = entry.getString("type").parsed(JobType::valueOf); + String fileResourceId = entry.getString("file").string(); + JsonObject errors = entry.getObject("errors"); + String errorCodeNamespace = + type == TRACKER_IMPORT_JOB + ? ValidationCode.class.getSimpleName() + : ErrorCode.class.getSimpleName(); + errors + .node() + .members() + .forEach( + byObject -> + byObject + .getValue() + .members() + .forEach( + byCode -> + byCode + .getValue() + .elements() + .forEach(error -> flatErrors.add(errorWithMessage(type, error))))); + + return JsonMixed.of( + errors + .node() + .replaceWith(createArray(arr -> flatErrors.forEach(arr::addElement))) + .addMembers( + obj -> + obj.addString("codes", errorCodeNamespace) + .addMember("input", getJobInput(params, fileResourceId)))); + } + + private JsonNode getJobInput(JobRunErrorsParams params, String fileResourceId) { + if (!params.isIncludeInput()) return JsonNode.of("null"); + try { + byte[] bytes = + fileResourceService.copyFileResourceContent( + fileResourceService.getFileResource(fileResourceId)); + return JsonNode.of(new String(bytes, StandardCharsets.UTF_8)); + } catch (IOException ex) { + log.warn("Could not copy file content to error info for file: " + fileResourceId, ex); + return JsonNode.of("\"" + ex.getMessage() + "\""); + } + } + + private static JsonNode errorWithMessage(JobType type, JsonNode error) { + String codeName = JsonMixed.of(error).getString("code").string(); + String template = + type == TRACKER_IMPORT_JOB + ? ValidationCode.valueOf(codeName).getMessage() + : ErrorCode.valueOf(codeName).getMessage(); + Object[] args = JsonMixed.of(error).getArray("args").stringValues().toArray(new String[0]); + String msg = MessageFormat.format(template, args); + return error.extract().addMembers(e -> e.addString("message", msg)); } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java index ceb9d77c38c2..e461105b5491 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java @@ -46,7 +46,6 @@ import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.commons.util.DebugUtils; import org.hisp.dhis.eventhook.EventHookPublisher; -import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.leader.election.LeaderManager; import org.hisp.dhis.message.MessageService; @@ -284,8 +283,7 @@ private void updateProgress(@Nonnull String jobId) { if (job == null) return; try { JobProgress.Progress progress = job.getProgress(); - String errorCodes = - progress.getErrorCodes().stream().map(ErrorCode::name).sorted().collect(joining(" ")); + String errorCodes = progress.getErrorCodes().stream().sorted().collect(joining(" ")); jobConfigurationStore.updateProgress( jobId, jsonMapper.writeValueAsString(progress), errorCodes); } catch (JsonProcessingException ex) { diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerService.java index 6a66f2e6f657..0a62ea76dbcf 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerService.java @@ -29,13 +29,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import java.text.MessageFormat; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.feedback.ConflictException; -import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.scheduling.JobProgress.Progress; import org.springframework.stereotype.Service; @@ -127,15 +129,11 @@ public List getErrors(@Nonnull String jobId) { if (json == null) return List.of(); Progress progress = mapToProgress("{\"sequence\":[],\"errors\":" + json + "}"); if (progress == null) return List.of(); - Map>> map = progress.getErrors(); + Map>> map = progress.getErrors(); if (map.isEmpty()) return List.of(); - List errors = - map.values().stream() - .flatMap(e -> e.values().stream().flatMap(Collection::stream)) - .toList(); - errors.forEach( - e -> e.setMessage(MessageFormat.format(e.getCode().getMessage(), e.getArgs().toArray()))); - return errors; + return map.values().stream() + .flatMap(e -> e.values().stream().flatMap(Collection::stream)) + .toList(); } @Override diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/HibernateJobConfigurationStore.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/HibernateJobConfigurationStore.java index 3518285e8628..d75dc6817f85 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/HibernateJobConfigurationStore.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/HibernateJobConfigurationStore.java @@ -261,6 +261,7 @@ select jsonb_build_object( 'created', c.created, 'executed', c.lastexecuted, 'finished', c.lastfinished, + 'file', fr.uid, 'filesize', fr.contentlength, 'filetype', fr.contenttype, 'errors', c.progress -> 'errors') #>> '{}' diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java index c3583f06810f..21db10c661f3 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/JobScheduler.java @@ -34,8 +34,15 @@ import java.time.Instant; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.concurrent.*; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import lombok.RequiredArgsConstructor; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/RecordingJobProgress.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/RecordingJobProgress.java index f3367f15185a..f65df4e9af92 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/RecordingJobProgress.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/RecordingJobProgress.java @@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.message.MessageService; +import org.hisp.dhis.tracker.imports.validation.ValidationCode; import org.hisp.dhis.user.CurrentUserDetails; import org.hisp.dhis.user.CurrentUserUtil; @@ -150,6 +151,23 @@ public void addError( @CheckForNull String uid, @Nonnull String type, @Nonnull List args) { + addError(code.name(), uid, type, args); + } + + @Override + public void addError( + @Nonnull ValidationCode code, + @CheckForNull String uid, + @Nonnull String type, + @Nonnull List args) { + addError(code.name(), uid, type, args); + } + + private void addError( + @Nonnull String code, + @CheckForNull String uid, + @Nonnull String type, + @Nonnull List args) { try { // Note: we use empty string in case the UID is not known/defined yet to allow use in maps progress.addError(new Error(code, uid == null ? "" : uid, type, args)); diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/DhisOidcClientRegistration.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/DhisOidcClientRegistration.java index efee0309250c..7e91ccb292ba 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/DhisOidcClientRegistration.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/DhisOidcClientRegistration.java @@ -31,7 +31,11 @@ import com.nimbusds.jose.jwk.JWK; import java.security.interfaces.RSAPublicKey; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import lombok.Builder; import lombok.Data; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/GenericOidcProviderConfigParser.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/GenericOidcProviderConfigParser.java index f500edfcee25..a492834590b6 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/GenericOidcProviderConfigParser.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/security/oidc/GenericOidcProviderConfigParser.java @@ -27,7 +27,32 @@ */ package org.hisp.dhis.security.oidc; -import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.*; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.AUTHORIZATION_GRANT_TYPE; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.AUTHORIZATION_URI; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.CLIENT_AUTHENTICATION_METHOD; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.CLIENT_ID; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.CLIENT_SECRET; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.DISPLAY_ALIAS; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.ENABLE_LOGOUT; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.ENABLE_PKCE; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.END_SESSION_ENDPOINT; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.EXTERNAL_CLIENT_PREFIX; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.EXTRA_REQUEST_PARAMETERS; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.ISSUER_URI; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWK_SET_URL; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWK_URI; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWT_PRIVATE_KEY_ALIAS; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWT_PRIVATE_KEY_KEYSTORE_PASSWORD; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWT_PRIVATE_KEY_KEYSTORE_PATH; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.JWT_PRIVATE_KEY_PASSWORD; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.LOGIN_IMAGE; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.LOGIN_IMAGE_PADDING; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.MAPPING_CLAIM; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.PROVIDER_ID; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.REDIRECT_URL; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.SCOPES; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.TOKEN_URI; +import static org.hisp.dhis.security.oidc.provider.AbstractOidcProvider.USERINFO_URI; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/sms/config/SMPPClient.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/sms/config/SMPPClient.java index dad4ef84edab..44ec8723a174 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/sms/config/SMPPClient.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/sms/config/SMPPClient.java @@ -28,7 +28,11 @@ package org.hisp.dhis.sms.config; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.outboundmessage.OutboundMessage; import org.hisp.dhis.outboundmessage.OutboundMessageBatch; @@ -36,7 +40,16 @@ import org.hisp.dhis.sms.outbound.GatewayResponse; import org.jsmpp.InvalidResponseException; import org.jsmpp.PDUException; -import org.jsmpp.bean.*; +import org.jsmpp.bean.Address; +import org.jsmpp.bean.Alphabet; +import org.jsmpp.bean.ESMClass; +import org.jsmpp.bean.GeneralDataCoding; +import org.jsmpp.bean.MessageClass; +import org.jsmpp.bean.NumberingPlanIndicator; +import org.jsmpp.bean.RegisteredDelivery; +import org.jsmpp.bean.ReplaceIfPresentFlag; +import org.jsmpp.bean.SMSCDeliveryReceipt; +import org.jsmpp.bean.TypeOfNumber; import org.jsmpp.extra.NegativeResponseException; import org.jsmpp.extra.ResponseTimeoutException; import org.jsmpp.session.BindParameter; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/DefaultSystemService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/DefaultSystemService.java index bd1989aa20f5..9f87aae9c8f4 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/DefaultSystemService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/DefaultSystemService.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.system; +import static org.hisp.dhis.util.DateUtils.getPrettyInterval; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -35,9 +37,9 @@ import java.util.List; import java.util.Properties; import java.util.TimeZone; +import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.calendar.CalendarService; import org.hisp.dhis.commons.util.SystemUtils; import org.hisp.dhis.configuration.Configuration; @@ -48,8 +50,7 @@ import org.hisp.dhis.external.location.LocationManagerException; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; -import org.hisp.dhis.system.database.DatabaseInfo; -import org.hisp.dhis.util.DateUtils; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -63,26 +64,22 @@ */ @Slf4j @RequiredArgsConstructor -@Service("org.hisp.dhis.system.SystemService") +@Service public class DefaultSystemService implements SystemService, InitializingBean { - private final LocationManager locationManager; - - private final DatabaseInfo databaseInfo; + private final LocationManager locationManager; + private final DatabaseInfoProvider databaseInfoProvider; private final ConfigurationService configurationService; - private final DhisConfigurationProvider dhisConfig; - private final CalendarService calendarService; - - private final SystemSettingManager systemSettingManager; + private final SystemSettingManager settings; /** Variable holding fixed system info state. */ private SystemInfo systemInfo = null; @Override public void afterPropertiesSet() { - systemInfo = getFixedSystemInfo(); + systemInfo = getStableSystemInfo(); List info = List.of( @@ -92,7 +89,7 @@ public void afterPropertiesSet() { "Database name: " + systemInfo.getDatabaseInfo().getName(), "Java version: " + systemInfo.getJavaVersion()); - log.info(StringUtils.join(info, ", ")); + log.info(String.join(", ", info)); } // ------------------------------------------------------------------------- @@ -102,144 +99,99 @@ public void afterPropertiesSet() { @Override @Transactional(readOnly = true) public SystemInfo getSystemInfo() { - SystemInfo info = systemInfo != null ? systemInfo.instance() : null; - TimeZone tz = Calendar.getInstance().getTimeZone(); - - if (info == null) { - return null; - } + if (systemInfo == null) return null; + TimeZone tz = Calendar.getInstance().getTimeZone(); Date lastAnalyticsTableSuccess = - systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE); - String lastAnalyticsTableRuntime = - systemSettingManager.getStringSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_RUNTIME); + settings.getDateSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE); Date lastAnalyticsTablePartitionSuccess = - systemSettingManager.getDateSetting( - SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_UPDATE); - String lastAnalyticsTablePartitionRuntime = - systemSettingManager.getStringSetting( - SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_RUNTIME); - Date lastSystemMonitoringSuccess = - systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_SYSTEM_MONITORING_PUSH); - String systemName = systemSettingManager.getStringSetting(SettingKey.APPLICATION_TITLE); - String instanceBaseUrl = dhisConfig.getServerBaseUrl(); + settings.getDateSetting(SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_UPDATE); Date now = new Date(); - info.setCalendar(calendarService.getSystemCalendar().name()); - info.setDateFormat(calendarService.getSystemDateFormat().getJs()); - info.setServerDate(new Date()); - info.setServerTimeZoneId(tz.getID()); - info.setServerTimeZoneDisplayName(tz.getDisplayName()); - - info.setLastAnalyticsTableSuccess(lastAnalyticsTableSuccess); - info.setIntervalSinceLastAnalyticsTableSuccess( - DateUtils.getPrettyInterval(lastAnalyticsTableSuccess, now)); - info.setLastAnalyticsTableRuntime(lastAnalyticsTableRuntime); - - info.setLastAnalyticsTablePartitionSuccess(lastAnalyticsTablePartitionSuccess); - info.setIntervalSinceLastAnalyticsTablePartitionSuccess( - DateUtils.getPrettyInterval(lastAnalyticsTablePartitionSuccess, now)); - info.setLastAnalyticsTablePartitionRuntime(lastAnalyticsTablePartitionRuntime); - - info.setLastSystemMonitoringSuccess(lastSystemMonitoringSuccess); - - info.setSystemName(systemName); - info.setInstanceBaseUrl(instanceBaseUrl); - info.setEmailConfigured(systemSettingManager.emailConfigured()); - - setSystemMetadataVersionInfo(info); - - return info; + return systemInfo.toBuilder() + .databaseInfo(databaseInfoProvider.getDatabaseInfo()) + .calendar(calendarService.getSystemCalendar().name()) + .dateFormat(calendarService.getSystemDateFormat().getJs()) + .serverDate(now) + .serverTimeZoneId(tz.getID()) + .serverTimeZoneDisplayName(tz.getDisplayName()) + .lastAnalyticsTableSuccess(lastAnalyticsTableSuccess) + .intervalSinceLastAnalyticsTableSuccess(getPrettyInterval(lastAnalyticsTableSuccess, now)) + .lastAnalyticsTableRuntime( + settings.getStringSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_RUNTIME)) + .lastAnalyticsTablePartitionSuccess(lastAnalyticsTablePartitionSuccess) + .intervalSinceLastAnalyticsTablePartitionSuccess( + getPrettyInterval(lastAnalyticsTablePartitionSuccess, now)) + .lastAnalyticsTablePartitionRuntime( + settings.getStringSetting( + SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_RUNTIME)) + .lastSystemMonitoringSuccess( + settings.getDateSetting(SettingKey.LAST_SUCCESSFUL_SYSTEM_MONITORING_PUSH)) + .systemName(settings.getStringSetting(SettingKey.APPLICATION_TITLE)) + .instanceBaseUrl(dhisConfig.getServerBaseUrl()) + .emailConfigured(settings.emailConfigured()) + .isMetadataVersionEnabled(settings.getBooleanSetting(SettingKey.METADATAVERSION_ENABLED)) + .systemMetadataVersion(settings.getStringSetting(SettingKey.SYSTEM_METADATA_VERSION)) + .lastMetadataVersionSyncAttempt( + getLastMetadataVersionSyncAttempt( + settings.getDateSetting(SettingKey.LAST_SUCCESSFUL_METADATA_SYNC), + settings.getDateSetting(SettingKey.METADATA_LAST_FAILED_TIME))) + .build(); } - private SystemInfo getFixedSystemInfo() { + /** + * @return A {@link SystemInfo} with all properties set that are stable (immutable) after start + */ + private SystemInfo getStableSystemInfo() { Configuration config = configurationService.getConfiguration(); - - // --------------------------------------------------------------------- - // Version - // --------------------------------------------------------------------- - - SystemInfo info = loadBuildProperties(); - - // --------------------------------------------------------------------- - // External directory - // --------------------------------------------------------------------- - - info.setEnvironmentVariable(locationManager.getEnvironmentVariable()); - - try { - File directory = locationManager.getExternalDirectory(); - - info.setExternalDirectory(directory.getAbsolutePath()); - } catch (LocationManagerException ex) { - info.setExternalDirectory("Not set"); - } - - info.setFileStoreProvider(dhisConfig.getProperty(ConfigurationKey.FILESTORE_PROVIDER)); - info.setReadOnlyMode(dhisConfig.getProperty(ConfigurationKey.SYSTEM_READ_ONLY_MODE)); - info.setNodeId(dhisConfig.getProperty(ConfigurationKey.NODE_ID)); - info.setSystemMonitoringUrl(dhisConfig.getProperty(ConfigurationKey.SYSTEM_MONITORING_URL)); - info.setSystemId(config.getSystemId()); - info.setClusterHostname(dhisConfig.getProperty(ConfigurationKey.CLUSTER_HOSTNAME)); - info.setRedisEnabled(dhisConfig.isEnabled(ConfigurationKey.REDIS_ENABLED)); - - if (info.isRedisEnabled()) { - info.setRedisHostname(dhisConfig.getProperty(ConfigurationKey.REDIS_HOST)); - } - - // --------------------------------------------------------------------- - // Database - // --------------------------------------------------------------------- - - info.setDatabaseInfo(databaseInfo.instance()); - info.setReadReplicaCount( - Integer.valueOf(dhisConfig.getProperty(ConfigurationKey.ACTIVE_READ_REPLICAS))); - - // --------------------------------------------------------------------- - // System env variables and properties - // --------------------------------------------------------------------- - - try { - info.setJavaOpts(System.getenv("JAVA_OPTS")); - } catch (SecurityException ex) { - info.setJavaOpts("Unknown"); - } - Properties props = System.getProperties(); - - info.setJavaVersion(props.getProperty("java.version")); - info.setJavaVendor(props.getProperty("java.vendor")); - info.setOsName(props.getProperty("os.name")); - info.setOsArchitecture(props.getProperty("os.arch")); - info.setOsVersion(props.getProperty("os.version")); - - info.setMemoryInfo(SystemUtils.getMemoryString()); - info.setCpuCores(SystemUtils.getCpuCores()); - info.setEncryption(dhisConfig.getEncryptionStatus().isOk()); - - return info; + boolean redisEnabled = dhisConfig.isEnabled(ConfigurationKey.REDIS_ENABLED); + + return loadBuildProperties().toBuilder() + .environmentVariable(locationManager.getEnvironmentVariable()) + .externalDirectory(getExternalDirectory()) + .fileStoreProvider(dhisConfig.getProperty(ConfigurationKey.FILESTORE_PROVIDER)) + .readOnlyMode(dhisConfig.getProperty(ConfigurationKey.SYSTEM_READ_ONLY_MODE)) + .nodeId(dhisConfig.getProperty(ConfigurationKey.NODE_ID)) + .systemMonitoringUrl(dhisConfig.getProperty(ConfigurationKey.SYSTEM_MONITORING_URL)) + .systemId(config.getSystemId()) + .clusterHostname(dhisConfig.getProperty(ConfigurationKey.CLUSTER_HOSTNAME)) + .redisEnabled(redisEnabled) + .redisHostname(redisEnabled ? dhisConfig.getProperty(ConfigurationKey.REDIS_HOST) : null) + // Database + .databaseInfo(databaseInfoProvider.getDatabaseInfo()) + .readReplicaCount( + Integer.valueOf(dhisConfig.getProperty(ConfigurationKey.ACTIVE_READ_REPLICAS))) + // System env variables and properties + .javaOpts(getJavaOpts()) + .javaVersion(props.getProperty("java.version")) + .javaVendor(props.getProperty("java.vendor")) + .osName(props.getProperty("os.name")) + .osArchitecture(props.getProperty("os.arch")) + .osVersion(props.getProperty("os.version")) + .memoryInfo(SystemUtils.getMemoryString()) + .cpuCores(SystemUtils.getCpuCores()) + .encryption(dhisConfig.getEncryptionStatus().isOk()) + .build(); } public static SystemInfo loadBuildProperties() { - SystemInfo info = new SystemInfo(); ClassPathResource resource = new ClassPathResource("build.properties"); if (resource.isReadable()) { try (InputStream in = resource.getInputStream()) { Properties properties = new Properties(); - properties.load(in); - - info.setVersion(properties.getProperty("build.version")); - info.setRevision(properties.getProperty("build.revision")); - info.setJasperReportsVersion(properties.getProperty("jasperreports.version")); - String buildTime = properties.getProperty("build.time"); - DateTimeFormatter dateFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); - info.setBuildTime(new DateTime(dateFormat.parseDateTime(buildTime)).toDate()); + return SystemInfo.builder() + .version(properties.getProperty("build.version")) + .revision(properties.getProperty("build.revision")) + .jasperReportsVersion(properties.getProperty("jasperreports.version")) + .buildTime(new DateTime(dateFormat.parseDateTime(buildTime)).toDate()) + .build(); } catch (IOException ex) { // Do nothing } @@ -248,37 +200,37 @@ public static SystemInfo loadBuildProperties() { "build.properties is not available in the classpath. " + "Make sure you build the project with Maven before you start the embedded Jetty server."); } - - return info; + return SystemInfo.builder().build(); } - private void setSystemMetadataVersionInfo(SystemInfo info) { - Boolean isMetadataVersionEnabled = - systemSettingManager.getBooleanSetting(SettingKey.METADATAVERSION_ENABLED); - Date lastSuccessfulMetadataSync = - systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_METADATA_SYNC); - Date metadataLastFailedTime = - systemSettingManager.getDateSetting(SettingKey.METADATA_LAST_FAILED_TIME); - String systemMetadataVersion = - systemSettingManager.getStringSetting(SettingKey.SYSTEM_METADATA_VERSION); - Date lastMetadataVersionSyncAttempt = - getLastMetadataVersionSyncAttempt(lastSuccessfulMetadataSync, metadataLastFailedTime); + private static String getJavaOpts() { + try { + return System.getenv("JAVA_OPTS"); + } catch (SecurityException ex) { + return "Unknown"; + } + } - info.setIsMetadataVersionEnabled(isMetadataVersionEnabled); - info.setSystemMetadataVersion(systemMetadataVersion); - info.setLastMetadataVersionSyncAttempt(lastMetadataVersionSyncAttempt); + @Nonnull + private String getExternalDirectory() { + try { + File directory = locationManager.getExternalDirectory(); + return directory.getAbsolutePath(); + } catch (LocationManagerException ex) { + return "Not set"; + } } private Date getLastMetadataVersionSyncAttempt( Date lastSuccessfulMetadataSyncTime, Date lastFailedMetadataSyncTime) { if (lastSuccessfulMetadataSyncTime == null && lastFailedMetadataSyncTime == null) { return null; - } else if (lastSuccessfulMetadataSyncTime == null || lastFailedMetadataSyncTime == null) { + } + if (lastSuccessfulMetadataSyncTime == null || lastFailedMetadataSyncTime == null) { return (lastFailedMetadataSyncTime != null ? lastFailedMetadataSyncTime : lastSuccessfulMetadataSyncTime); } - return (lastSuccessfulMetadataSyncTime.compareTo(lastFailedMetadataSyncTime) < 0) ? lastFailedMetadataSyncTime : lastSuccessfulMetadataSyncTime; diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemInfo.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemInfo.java deleted file mode 100644 index e747df63cbcd..000000000000 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/system/SystemInfo.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Date; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.hisp.dhis.system.database.DatabaseInfo; -import org.springframework.beans.BeanUtils; - -/** - * @author Lars Helge Overland - */ -@Getter -@Setter -@NoArgsConstructor -public class SystemInfo { - // ------------------------------------------------------------------------- - // Transient properties - // ------------------------------------------------------------------------- - - @JsonProperty private String contextPath; - - @JsonProperty private String userAgent; - - // ------------------------------------------------------------------------- - // Volatile properties - // ------------------------------------------------------------------------- - - @JsonProperty private String calendar; - - @JsonProperty private String dateFormat; - - @JsonProperty private Date serverDate; - - @JsonProperty private String serverTimeZoneId; - - @JsonProperty private String serverTimeZoneDisplayName; - - @JsonProperty private Date lastAnalyticsTableSuccess; - - @JsonProperty private String intervalSinceLastAnalyticsTableSuccess; - - @JsonProperty private String lastAnalyticsTableRuntime; - - @JsonProperty private Date lastSystemMonitoringSuccess; - - @JsonProperty private Date lastAnalyticsTablePartitionSuccess; - - @JsonProperty private String intervalSinceLastAnalyticsTablePartitionSuccess; - - @JsonProperty private String lastAnalyticsTablePartitionRuntime; - - // ------------------------------------------------------------------------- - // Stable properties - // ------------------------------------------------------------------------- - - @JsonProperty private String version; - - @JsonProperty private String revision; - - @JsonProperty private Date buildTime; - - @JsonProperty private String jasperReportsVersion; - - @JsonProperty private String environmentVariable; - - @JsonProperty private String fileStoreProvider; - - @JsonProperty private String readOnlyMode; - - @JsonProperty private String nodeId; - - @JsonProperty private String javaVersion; - - @JsonProperty private String javaVendor; - - @JsonProperty private String javaOpts; - - @JsonProperty private String osName; - - @JsonProperty private String osArchitecture; - - @JsonProperty private String osVersion; - - @JsonProperty private String externalDirectory; - - @JsonProperty private DatabaseInfo databaseInfo; - - @JsonProperty private Integer readReplicaCount; - - @JsonProperty private String memoryInfo; - - @JsonProperty private Integer cpuCores; - - @JsonProperty private boolean encryption; - - @JsonProperty private boolean emailConfigured; - - @JsonProperty private boolean redisEnabled; - - @JsonProperty private String redisHostname; - - @JsonProperty private String systemId; - - @JsonProperty private String systemName; - - @JsonProperty private String systemMetadataVersion; - - @JsonProperty private String instanceBaseUrl; - - @JsonProperty private String systemMonitoringUrl; - - @JsonProperty private String clusterHostname; - - @JsonProperty private Boolean isMetadataVersionEnabled; - - @JsonProperty private Date lastMetadataVersionSyncAttempt; - - @JsonProperty private Boolean isMetadataSyncEnabled; - - public SystemInfo instance() { - SystemInfo info = new SystemInfo(); - BeanUtils.copyProperties(this, info); - // clear sensitive info may reset the data - info.setDatabaseInfo(databaseInfo == null ? null : databaseInfo.instance()); - return info; - } - - // ------------------------------------------------------------------------- - // Logic - // ------------------------------------------------------------------------- - - /** - * Clears sensitive system info properties. - * - *

Note that {@code systemId} must be present for {@link MonitoringService} to function. - */ - public void clearSensitiveInfo() { - this.jasperReportsVersion = null; - this.environmentVariable = null; - this.fileStoreProvider = null; - this.readOnlyMode = null; - this.nodeId = null; - this.javaVersion = null; - this.javaVendor = null; - this.javaOpts = null; - this.osName = null; - this.osArchitecture = null; - this.osVersion = null; - this.externalDirectory = null; - this.readReplicaCount = null; - this.memoryInfo = null; - this.cpuCores = null; - this.systemMonitoringUrl = null; - this.encryption = false; - this.redisEnabled = false; - this.redisHostname = null; - this.clusterHostname = null; - - if (this.databaseInfo != null) { - this.databaseInfo.clearSensitiveInfo(); - } - } -} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/DefaultTrackedEntityService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/DefaultTrackedEntityService.java index a4c760e6d439..2aabe00c9230 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/DefaultTrackedEntityService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/DefaultTrackedEntityService.java @@ -850,7 +850,7 @@ private boolean isLocalSearch(TrackedEntityQueryParams params, User user) { } for (OrganisationUnit ou : searchOrgUnits) { - if (!organisationUnitService.isDescendant(ou, localOrgUnits)) { + if (!ou.isDescendant(localOrgUnits)) { return false; } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_uz_UZ_Cyrl.properties b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_uz_UZ_Cyrl.properties index 6d7d8d72b9fc..16dae89a6c1b 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_uz_UZ_Cyrl.properties +++ b/dhis-2/dhis-services/dhis-service-core/src/main/resources/i18n_global_uz_UZ_Cyrl.properties @@ -41,7 +41,7 @@ dhis-web-maintenance=\u0422\u0435\u0445\u043d\u0438\u043a \u0445\u0438\u0437\u04 dhis-web-data-administration=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u0431\u043e\u0448\u049b\u0430\u0440\u0438\u0448 dhis-web-maintenance-settings=\u0421\u043e\u0437\u043b\u0430\u043c\u0430\u043b\u0430\u0440 dhis-web-dataentry=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u043a\u0438\u0440\u0438\u0442\u0438\u0448 -dhis-web-aggregate-data-entry=Data Entry (Beta) +dhis-web-aggregate-data-entry=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u041a\u0438\u0440\u0438\u0442\u0438\u0448 \u0443\u0447\u0443\u043d (Beta) dhis-web-sms=\u0421\u041c\u0421 dhis-web-import-export=\u0418\u043c\u043f\u043e\u0440\u0442/\u042d\u043a\u0441\u043f\u043e\u0440\u0442 dhis-web-data-quality=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0441\u0438\u0444\u0430\u0442\u0438 @@ -53,7 +53,7 @@ dhis-web-event-visualizer=\u04b2\u043e\u0434\u0438\u0441\u0430\u043b\u0430\u0440 dhis-web-tracker-capture=\u041a\u0443\u0437\u0430\u0442\u0443\u0432-\u041d\u0430\u0437\u043e\u0440\u0430\u0442 dhis-web-reports=\u04b2\u0438\u0441\u043e\u0431\u043e\u0442\u043b\u0430\u0440 dhis-web-approval=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u0442\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u0448 -dhis-web-approval-classic=Data Approval Classic +dhis-web-approval-classic=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u0442\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u043d\u0438\u0448\u0438 \u043a\u043b\u0430\u0441\u0441\u0438\u043a dhis-web-cache-cleaner=\u0411\u0440\u043e\u0443\u0437\u0435\u0440 \u043a\u0435\u0448\u0438\u043d\u0438 \u0442\u043e\u0437\u0430\u043b\u0430\u0448 dhis-web-mobile=\u041c\u043e\u0431\u0438\u043b (\u0421\u043c\u0430\u0440\u0442\u0444\u043e\u043d) dhis-web-dashboard-integration=\u0411\u043e\u0448\u049b\u0430\u0440\u0443\u0432 \u043f\u0430\u043d\u0435\u043b\u0438 @@ -109,7 +109,7 @@ M_dhis-web-maintenance=\u0422\u0430\u044a\u043c\u0438\u043d\u043e\u0442 \u0434\u M_dhis-web-data-administration=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u0431\u043e\u0448\u049b\u0430\u0440\u0438\u0448 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-settings=\u0421\u043e\u0437\u043b\u0430\u043c\u0430\u043b\u0430\u0440 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-dataentry=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u043a\u0438\u0440\u0438\u0442\u0438\u0448 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 -M_dhis-web-aggregate-data-entry=Data Entry (Beta) app +M_dhis-web-aggregate-data-entry=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u041a\u0438\u0440\u0438\u0442\u0438\u0448 \u0443\u0447\u0443\u043d (Beta) \u0434\u0430\u0441\u0442\u0443\u0440 M_dhis-web-import-export=\u0418\u043c\u043f\u043e\u0440\u0442-\u042d\u043a\u0441\u043f\u043e\u0440\u0442 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-validationrule=\u0422\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u0448 \u049a\u043e\u0438\u0434\u0430\u043b\u0430\u0440\u0438 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-data-quality=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0421\u0438\u0444\u0430\u0442\u0438 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 @@ -133,7 +133,7 @@ M_dhis-web-tracker-capture=\u041a\u0443\u0437\u0430\u0442\u0443\u0432-\u041d\u04 M_dhis-web-capture=\u041d\u0430\u0437\u043e\u0440\u0430\u0442\u0433\u0430 \u043e\u043b\u0438\u0448 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-reports=\u04b2\u0438\u0441\u043e\u0431\u043e\u0442\u043b\u0430\u0440 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-approval=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u0422\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u0448 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 -M_dhis-web-approval-classic=Data Approval Classic app +M_dhis-web-approval-classic=\u0414\u0430\u0441\u0442\u0443\u0440\u0434\u0430 \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u0442\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u043d\u0438\u0448\u0438 \u043a\u043b\u0430\u0441\u0441\u0438\u043a M_dhis-web-event-visualizer=\u04b2\u043e\u0434\u0438\u0441\u0430/\u0422\u0430\u0434\u0431\u0438\u0440 \u0412\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0442\u043e\u0440 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-cache-cleaner=\u0411\u0440\u0430\u0443\u0437\u0435\u0440 \u041a\u0435\u0448\u0438\u043d\u0438 \u0422\u043e\u0437\u0430\u043b\u0430\u0448 \u0434\u0430\u0441\u0442\u0443\u0440\u0438 M_dhis-web-translations=\u0422\u0430\u0440\u0436\u0438\u043c\u043e\u043d \u0434\u0430\u0441\u0442\u0443\u0440\u0438 @@ -200,7 +200,7 @@ F_INDICATORGROUPSET_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u0418\u043d\u0434\u043 F_INDICATORGROUPSET_DELETE=\u0418\u043d\u0434\u0438\u043a\u0430\u0442\u043e\u0440 \u0413\u0443\u0440\u0443\u04b3\u0438 \u0422\u045e\u043f\u043b\u0430\u043c\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_ORGANISATIONUNIT_ADD=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ORGANISATIONUNIT_DELETE=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u040e\u0447\u0438\u0440\u0438\u0448 -F_ORGANISATIONUNIT_MOVE=Move organisation unit +F_ORGANISATIONUNIT_MOVE=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u0438\u0440\u043b\u0438\u043a \u0436\u043e\u0439\u0438\u043d\u0438 \u045e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 F_ORGUNITGROUP_PUBLIC_ADD=\u041e\u0447\u0438\u049b \u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u0413\u0443\u0440\u0443\u04b3\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ORGUNITGROUP_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u0413\u0443\u0440\u0443\u04b3\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ORGUNITGROUP_DELETE=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u0413\u0443\u0440\u0443\u04b3\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 @@ -219,7 +219,7 @@ F_USERGROUP_LIST=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0 F_USER_GROUPS_READ_ONLY_ADD_MEMBERS=\u0424\u0430\u049b\u0430\u0442 \u040e\u049b\u0438\u0448 \u0443\u0447\u0443\u043d \u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 \u0413\u0443\u0440\u0443\u04b3\u043b\u0430\u0440\u0438\u0433\u0430 \u0410\u044a\u0437\u043e\u043b\u0430\u0440\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u041e\u043b\u0438\u0431 \u0442\u0430\u0448\u043b\u0430\u0448 F_USER_ADD=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_USER_DELETE=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 -F_USER_VIEW=View user +F_USER_VIEW=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438\u043d\u0438 \u043a\u045e\u0440\u0438\u0448 F_USER_ADD_WITHIN_MANAGED_GROUP=\u0411\u043e\u0448\u049b\u0430\u0440\u0438\u043b\u0430\u0434\u0438\u0433\u0430\u043d \u0413\u0443\u0440\u0443\u04b3 \u0438\u0447\u0438\u0434\u0430 \u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_USER_DELETE_WITHIN_MANAGED_GROUP=\u0411\u043e\u0448\u049b\u0430\u0440\u0438\u043b\u0430\u0434\u0438\u0433\u0430\u043d \u0413\u0443\u0440\u0443\u04b3 \u0438\u0447\u0438\u0434\u0430 \u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_USER_VIEW_WITHIN_MANAGED_GROUP=\u0411\u043e\u0448\u049b\u0430\u0440\u0438\u043b\u0430\u0434\u0438\u0433\u0430\u043d \u0413\u0443\u0440\u0443\u04b3 \u0438\u0447\u0438\u0434\u0430 \u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 \u041a\u045e\u0440\u0438\u0431 \u0447\u0438\u049b\u0438\u0448 @@ -235,10 +235,10 @@ F_REPORT_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u04b2\u0438\u0441\u043e\u0431\u04 F_REPORT_DELETE=\u04b2\u0438\u0441\u043e\u0431\u043e\u0442\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_REPORT_VIEW=\u04b2\u0438\u0441\u043e\u0431\u043e\u0442\u043d\u0438 \u041a\u045e\u0440\u0438\u0431 \u0447\u0438\u049b\u0438\u0448 F_DASHBOARD_PUBLIC_ADD=\u041e\u0447\u0438\u049b \u0411\u043e\u0448\u049b\u0430\u0440\u0443\u0432 \u043f\u0430\u043d\u0435\u043b\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 -F_VISUALIZATION_PUBLIC_ADD=Add/Update Public Visualization -F_VISUALIZATION_EXTERNAL=Visualization External Access -F_EVENT_VISUALIZATION_PUBLIC_ADD=Add/Update Public Event Visualization -F_EVENT_VISUALIZATION_EXTERNAL=Event Visualization External Access +F_VISUALIZATION_PUBLIC_ADD=\u041e\u043c\u043c\u0430\u0432\u0438\u0439 \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_VISUALIZATION_EXTERNAL=\u0422\u0430\u0448\u049b\u0438 \u043a\u0438\u0440\u0443\u0432\u0447\u0438\u043b\u0430\u0440\u043d\u0438 \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f\u043b\u0430\u0448 +F_EVENT_VISUALIZATION_PUBLIC_ADD=\u041e\u043c\u043c\u0430\u0432\u0438\u0439 \u04b3\u043e\u0434\u0438\u0441\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_EVENT_VISUALIZATION_EXTERNAL=\u0422\u0430\u0448\u049b\u0438 \u043a\u0438\u0440\u0443\u0432\u0447\u0438\u043b\u0430\u0440 \u04b3\u043e\u043b\u0430\u0442\u043b\u0430\u0440\u0438\u0434\u0430 \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f\u043b\u0430\u0448 F_DOCUMENT_PUBLIC_ADD=\u041e\u0447\u0438\u049b \u04b2\u0443\u0436\u0436\u0430\u0442 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_DOCUMENT_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u04b2\u0443\u0436\u0436\u0430\u0442 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_DOCUMENT_DELETE=\u04b2\u0443\u0436\u0436\u0430\u0442\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 @@ -294,7 +294,7 @@ F_TRACKED_ENTITY_INSTANCE_LIST=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\ F_TRACKED_ENTITY_INSTANCE_HISTORY=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0428\u0430\u0445\u0441 \u041d\u0430\u043c\u0443\u043d\u0430\u0441\u0438 \u0422\u0430\u0440\u0438\u0445\u0438\u043d\u0438 \u042e\u043a\u043b\u0430\u0448 F_TRACKED_ENTITY_INSTANCE_DASHBOARD=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0428\u0430\u0445\u0441 \u041d\u0430\u043c\u0443\u043d\u0430\u0438 \u0411\u043e\u0448\u049b\u0430\u0440\u0443\u0432 \u043f\u0430\u043d\u0435\u043b\u0438 F_TRACKED_ENTITY_INSTANCE_CHANGE_LOCATION=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0428\u0430\u0445\u0441 \u041d\u0430\u043c\u0443\u043d\u0430\u0438 \u0416\u043e\u0439\u043b\u0430\u0448\u0443\u0432\u0438\u043d\u0438 \u040e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u043d\u0433 -F_TRACKED_ENTITY_MERGE=Merge tracked entity instances +F_TRACKED_ENTITY_MERGE=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0448\u0430\u0445\u0441 \u043d\u0443\u0441\u0445\u0430\u043b\u0430\u0440\u0438\u043d\u0438 \u04b3\u0438\u0441\u043e\u0431\u0438\u043d\u0438 \u044e\u0440\u0438\u0442\u0438\u0448 F_TRACKED_ENTITY_COMMENT_ADD=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0428\u0430\u0445\u0441 \u041d\u0430\u043c\u0443\u043d\u0430\u0441\u0438 \u0418\u0437\u043e\u04b3 \u049a\u045e\u0448\u0438\u0448 F_TRACKED_ENTITY_COMMENT_DELETE=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0428\u0430\u0445\u0441 \u041d\u0430\u043c\u0443\u043d\u0430\u0441\u0438 \u0418\u0437\u043e\u04b3\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_PROGRAM_INDICATOR_MANAGEMENT=\u0414\u0430\u0441\u0442\u0443\u0440 \u0418\u043d\u0434\u0438\u043a\u0430\u0442\u043e\u0440\u043b\u0430\u0440\u0438\u043d\u0438 \u0411\u043e\u0448\u049b\u0430\u0440\u0438\u0448 @@ -347,9 +347,9 @@ F_VALIDATIONCRITERIA_DELETE=\u0422\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u04 F_ATTRIBUTE_PUBLIC_ADD=\u041e\u043c\u043c\u0430\u0432\u0438\u0439 \u0410\u0442\u0440\u0438\u0431\u0443\u0442\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ATTRIBUTE_PRIVATE_ADD=\u0425\u0443\u0441\u0443\u0441\u0438\u0439 \u0410\u0442\u0440\u0438\u0431\u0443\u0442\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ATTRIBUTE_DELETE=\u0410\u0442\u0440\u0438\u0431\u0443\u0442\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 -F_EXPRESSION_DIMENSION_ITEM_PUBLIC_ADD=Add/Update Public ExpressionDimensionItem -F_EXPRESSION_DIMENSION_ITEM_PRIVATE_ADD=Add/Update Private ExpressionDimensionItem -F_EXPRESSION_DIMENSION_ITEM_DELETE=Delete ExpressionDimensionItem +F_EXPRESSION_DIMENSION_ITEM_PUBLIC_ADD=\u041e\u0447\u0438\u049b \u0438\u0444\u043e\u0434\u0430 \u045e\u043b\u0447\u043e\u0432 \u044d\u043b\u0435\u043c\u0435\u043d\u0438\u0442\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_EXPRESSION_DIMENSION_ITEM_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u0438\u0444\u043e\u0434\u0430 \u045e\u043b\u0447\u043e\u0432 \u044d\u043b\u0435\u043c\u0435\u043d\u0438\u0442\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_EXPRESSION_DIMENSION_ITEM_DELETE=\u0418\u0444\u043e\u0434\u0430 \u045e\u043b\u0447\u043e\u0432 \u044d\u043b\u0435\u043c\u0435\u043d\u0438\u0442\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_OPTIONSET_PUBLIC_ADD=\u041e\u043c\u043c\u0430\u0432\u0438\u0439 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043b\u0430\u0440 \u0442\u045e\u043f\u043b\u0430\u043c\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_OPTIONSET_PRIVATE_ADD=\u0425\u0443\u0441\u0443\u0441\u0438\u0439 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043b\u0430\u0440 \u0442\u045e\u043f\u043b\u0430\u043c\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_OPTIONSET_DELETE=\u0412\u0430\u0440\u0438\u0430\u043d\u0442\u043b\u0430\u0440 \u0422\u045e\u043f\u043b\u0430\u043c\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 @@ -372,11 +372,11 @@ F_MOBILE_DELETE_SMS=\u0421\u041c\u0421\u043d\u0438 \u045e\u0447\u0438\u0440\u043 F_SEND_EMAIL=\u0435-\u0445\u0430\u0431\u0430\u0440 \u044e\u0431\u043e\u0440\u0438\u0448 F_VIEW_SETTINGS=\u0421\u043e\u0437\u043b\u0430\u043c\u0430\u043b\u0430\u0440\u043d\u0438 \u043a\u045e\u0440\u0438\u0448 F_PERFORM_MAINTENANCE=\u0422\u0430\u044a\u043c\u0438\u0440\u043b\u0430\u0448 \u0432\u0430\u0437\u0438\u0444\u0430\u043b\u0430\u0440\u0438\u043d\u0438 \u0431\u0430\u0436\u0430\u0440\u0438\u043d\u0433 -F_PERFORM_ANALYTICS_EXPLAIN=Perform analytics explain query +F_PERFORM_ANALYTICS_EXPLAIN=\u0410\u043d\u0430\u043b\u0438\u0442\u0438\u043a \u0441\u045e\u0440\u043e\u0432\u043d\u0438 \u0442\u0430\u04b3\u043b\u0438\u043b\u0438 \u0431\u0438\u043b\u0430\u043d \u0431\u0430\u0436\u0430\u0440\u0438\u043d\u0433 F_PRUNE_ORGANISATION_UNITS=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u045e\u043b\u0438\u043c\u043b\u0430\u0440\u043d\u0438 \u049b\u0438\u0441\u049b\u0430 \u043a\u045e\u0440\u0438\u043d\u0438\u0448\u0438 -F_ORGANISATION_UNIT_SPLIT=Split organisation unit +F_ORGANISATION_UNIT_SPLIT=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u0438\u0440\u043b\u0438\u043a\u043d\u0438 \u043f\u0430\u0440\u0447\u0430\u043b\u0430\u0448 F_ORGANISATION_UNIT_MERGE=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u045e\u043b\u0438\u043c\u043b\u0430\u0440\u043d\u0438 \u0431\u0438\u0440\u043b\u0430\u0448\u0442\u0438\u0440\u0438\u0448 -F_ORG_UNIT_PROFILE_ADD=Add organisation unit profile +F_ORG_UNIT_PROFILE_ADD=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u0438\u0440\u043b\u0438\u043a \u043f\u0440\u043e\u0444\u0438\u043b\u0438\u043d\u0438 \u049b\u045e\u0448\u0438\u0448 F_ELIMINATE_DUPLICATE_DATA_ELEMENTS=\u0422\u0430\u043a\u0440\u043e\u0440\u043b\u0430\u043d\u0430\u0451\u0442\u0433\u0430\u043d \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438\u043d\u0438 \u0439\u045e\u049b\u043e\u0442\u0438\u0448 F_GENERATE_MIN_MAX_VALUES=\u041c\u0438\u043d\u0438\u043c\u0430\u043b-\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b \u049b\u0438\u0439\u043c\u0430\u0442\u043b\u0430\u0440\u043d\u0438 \u044f\u0440\u0430\u0442\u0438\u043d\u0433 F_IMPORT_OTHER_SYSTEMS=\u0411\u043e\u0448\u049b\u0430 \u0442\u0438\u0437\u0438\u043c\u043b\u0430\u0440\u0434\u0430\u043d \u0438\u043c\u043f\u043e\u0440\u0442 \u049b\u0438\u043b\u0438\u0448 @@ -392,15 +392,15 @@ F_REPLICATE_USER=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0 F_LOCALE_ADD=\u0422\u0438\u043b \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_LOCALE_DELETE=\u0422\u0438\u043b \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u0438\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 F_PREDICTOR_RUN=\u0411\u0430\u0448\u043e\u0440\u0430\u0442\u043d\u0438 \u0438\u0448\u0433\u0430 \u0442\u0443\u0448\u0438\u0440\u0438\u0448 -F_TRACKED_ENTITY_MANAGEMENT=Manage tracked entities -F_TRACKED_ENTITY_UPDATE=Update tracked entities +F_TRACKED_ENTITY_MANAGEMENT=\u041a\u0443\u0437\u0430\u0442\u0443\u0432 \u043d\u0430\u0437\u043e\u0440\u0430\u0442 \u043e\u0431\u044a\u0435\u043a\u0442\u0438\u043d\u0438 \u0431\u043e\u0448\u049b\u0430\u0440\u0438\u0448 +F_TRACKED_ENTITY_UPDATE=\u041a\u0443\u0437\u0430\u0442\u0443\u0432 \u043d\u0430\u0437\u043e\u0440\u0430\u0442 \u043e\u0431\u044a\u0435\u043a\u0442\u0438\u043d\u0438 \u044f\u043d\u0433\u0438\u043b\u0430\u0448 F_VALIDATIONRULEGROUP_ADD=\u0422\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u0448 \u049a\u043e\u0438\u0434\u0430\u0441\u0438 \u0413\u0443\u0440\u0443\u04b3\u043b\u0430\u0440\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_SQLVIEW_ADD=SQL \u041a\u045e\u0440\u0438\u0431 \u0447\u0438\u049b\u0438\u0448\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_ORGUNITGROUPSET_ADD=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0411\u045e\u043b\u0438\u043c \u0413\u0443\u0440\u0443\u04b3\u0438 \u0422\u045e\u043f\u043b\u0430\u043c\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_OPTIONSET_ADD=\u0412\u0430\u0440\u0438\u0430\u043d\u0442\u043b\u0430\u0440 \u0422\u045e\u043f\u043b\u0430\u043c\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_INSERT_CUSTOM_JS_CSS=\u041c\u0430\u0445\u0441\u0443\u0441 Javascript \u0432\u0430 CSS \u043d\u0438 \u0436\u043e\u0439\u043b\u0430\u0448\u0442\u0438\u0440\u0438\u043d\u0433 F_VIEW_UNAPPROVED_DATA=\u0422\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u043d\u043c\u0430\u0433\u0430\u043d \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u043a\u045e\u0440\u0438\u043d\u0433 -F_PROGRAM_RULE_MANAGEMENT=Manage program rule +F_PROGRAM_RULE_MANAGEMENT=\u0414\u0430\u0441\u0442\u0443\u0440 \u049b\u043e\u0438\u0434\u0430\u0441\u0438\u043d\u0438 \u0431\u043e\u0448\u049b\u0430\u0440\u0438\u0448 F_PROGRAM_RULE_ADD=\u0414\u0430\u0441\u0442\u0443\u0440 \u049a\u043e\u0438\u0434\u0430\u043b\u0430\u0440\u0438 \u049a\u045e\u0448\u0438\u0448 F_PROGRAM_RULE_UPDATE=\u0414\u0430\u0441\u0442\u0442\u0443\u0440 \u049a\u043e\u0438\u0434\u0430\u043b\u0430\u0440\u0438 \u042f\u043d\u0433\u0438\u043b\u0430\u0448 F_OAUTH2_CLIENT_MANAGE=OAuth2 Client \u043d\u0438 \u0431\u043e\u0448\u049b\u0430\u0440\u0438\u0448 @@ -426,20 +426,20 @@ F_PROGRAM_ADD=\u0414\u0430\u0441\u0442\u0443\u0440 \u049a\u045e\u0448\u0438\u044 F_REPORT_EXTERNAL=\u0422\u0430\u0448\u049b\u0438 \u043a\u0438\u0440\u0438\u0448 \u04b2\u0443\u049b\u0443\u049b\u0438 \u04b3\u0430\u049b\u0438\u0434\u0430 \u04b2\u0438\u0441\u043e\u0431\u043e\u0442 F_UNCOMPLETE_EVENT=\u0422\u0443\u0433\u0430\u043b\u043b\u0430\u043d\u043c\u0430\u0433\u0430\u043d \u04b3\u043e\u0434\u0438\u0441\u0430/\u0442\u0430\u0434\u0431\u0438\u0440\u043b\u0430\u0440 F_SKIP_DATA_IMPORT_AUDIT=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0438\u043c\u043f\u043e\u0440\u0442\u0438 \u0430\u0443\u0434\u0438\u0442\u0438\u043d\u0438 \u045e\u0442\u043a\u0430\u0437\u0438\u0431 \u044e\u0431\u043e\u0440\u0438\u043d\u0433 -F_VIEW_SERVER_INFO=View server information -F_DATA_APPROVAL_WORKFLOW=Add/Update Data Approval Workflow -F_DATA_APPROVAL_LEVEL=Add/Update Data Approval Level -F_AGGREGATE_DATA_EXCHANGE_PUBLIC_ADD=Add/Update Public Aggregate Data Exchange -F_AGGREGATE_DATA_EXCHANGE_PRIVATE_ADD=Add/Update Private Aggregate Data Exchange -F_AGGREGATE_DATA_EXCHANGE_DELETE=Delete Aggregate Data Exchange -F_AGGREGATE_DATA_EXCHANGE=Aggregate Data Exchange -F_EVENT_HOOK_PUBLIC_ADD=Add/Update Public Event Hook -F_EVENT_HOOK_PRIVATE_ADD=Add/Update Private Event Hook -F_EVENT_HOOK_DELETE=Delete Event Hook -F_ROUTE_PUBLIC_ADD=Add/Update Public Route -F_ROUTE_PRIVATE_ADD=Add/Update Private Route -F_ROUTE_PUBLIC_DELETE=Delete Route -F_IMPERSONATE_USERS=Impersonate user +F_VIEW_SERVER_INFO=\u0421\u0435\u0440\u0432\u0435\u0440 \u0430\u0445\u0431\u043e\u0440\u043e\u0442\u0438\u043d\u0438 \u043a\u045e\u0440\u0438\u0448 +F_DATA_APPROVAL_WORKFLOW=\u0418\u0448 \u0436\u0430\u0440\u0430\u0451\u043d\u0438 \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_DATA_APPROVAL_LEVEL=\u0411\u043e\u0441\u049b\u0438\u0447 \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_AGGREGATE_DATA_EXCHANGE_PUBLIC_ADD=\u0419\u0438\u0493\u0438\u043b\u0433\u0430\u043d \u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u042d\u043b\u0435\u043c\u0435\u043d\u0442\u0438\u043d\u0438 \u040e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 +F_AGGREGATE_DATA_EXCHANGE_PRIVATE_ADD=\u041e\u0447\u0438\u049b \u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u042d\u043b\u0435\u043c\u0435\u043d\u0442\u0438\u043d\u0438 \u040e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 +F_AGGREGATE_DATA_EXCHANGE_DELETE=\u040e\u0447\u0438\u0440\u0438\u043b\u0433\u0430\u043d \u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u042d\u043b\u0435\u043c\u0435\u043d\u0442\u0438\u043d\u0438 \u040e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 +F_AGGREGATE_DATA_EXCHANGE=\u0419\u0438\u0493\u0438\u043b\u0433\u0430\u043d \u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u040e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 +F_EVENT_HOOK_PUBLIC_ADD=\u041e\u0447\u0438\u049b \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u0434\u0430\u0433\u0438 \u04b3\u043e\u0434\u0438\u0441\u0430\u043b\u0430\u0440 \u043a\u0443\u0437\u0430\u0442\u0443\u0432\u0447\u0438\u0441\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_EVENT_HOOK_PRIVATE_ADD=\u0401\u043f\u0438\u049b \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u0434\u0430\u0433\u0438 \u04b3\u043e\u0434\u0438\u0441\u0430\u043b\u0430\u0440 \u043a\u0443\u0437\u0430\u0442\u0443\u0432\u0447\u0438\u0441\u0438\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_EVENT_HOOK_DELETE=\u04b2\u043e\u0434\u0438\u0441\u0430/\u0422\u0430\u0434\u0431\u0438\u0440\u043d\u0438 \u043a\u0443\u0437\u0430\u0442\u0443\u0432\u0447\u0438\u043d\u0438 \u045e\u0447\u0438\u0440\u0438\u0448 +F_ROUTE_PUBLIC_ADD=\u0423\u043c\u0443\u043c\u0438\u0439 \u0439\u045e\u043d\u0430\u043b\u0438\u0448\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_ROUTE_PRIVATE_ADD=\u0425\u0443\u0441\u0443\u0441\u0438\u0439 \u0439\u045e\u043d\u0430\u043b\u0438\u0448\u043d\u0438 \u049a\u045e\u0448\u0438\u0448/\u042f\u043d\u0433\u0438\u043b\u0430\u0448 +F_ROUTE_PUBLIC_DELETE=\u0419\u045e\u043d\u0430\u043b\u0438\u0448\u043d\u0438 \u040e\u0447\u0438\u0440\u0438\u0448 +F_IMPERSONATE_USERS=\u0420\u043e\u0431\u043e\u0442 \u0444\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 #-- Common ---------------------------------------------------------------------# @@ -544,7 +544,7 @@ item=\u042d\u043b\u0435\u043c\u0435\u043d\u0442 action=\u0410\u043c\u0430\u043b sub_total=\u041e\u0440\u0430\u043b\u0438\u049b \u0439\u0438\u0493\u0438\u043d\u0434\u0438 total=\u0416\u0430\u043c\u0438 -impersonate_user=Impersonate user +impersonate_user=\u0420\u043e\u0431\u043e\u0442 \u0444\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438 username_in_password=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438\u0418\u0441\u043c\u0438 \u043f\u0430\u0440\u043e\u043b\u043d\u0438\u043d\u0433 \u049b\u0438\u0441\u043c\u0438 \u0431\u045e\u043b\u0438\u0448\u0438 \u043c\u0443\u043c\u043a\u0438\u043d \u044d\u043c\u0430\u0441 email_in_password=\u0415-\u043f\u043e\u0447\u0442\u0430 \u043f\u0430\u0440\u043e\u043b\u043d\u0438\u043d\u0433 \u049b\u0438\u0441\u043c\u0438 \u0431\u045e\u043b\u0438\u0448\u0438 \u043c\u0443\u043c\u043a\u0438\u043d \u044d\u043c\u0430\u0441 username_email_in_password=\u0424\u043e\u0439\u0434\u0430\u043b\u0430\u043d\u0443\u0432\u0447\u0438\u0418\u0441\u043c\u0438/\u0423-\u043f\u043e\u0447\u0442\u0430 \u043f\u0430\u0440\u043e\u043b\u043d\u0438\u043d\u0433 \u049b\u0438\u0441\u043c\u0438 \u0431\u045e\u043b\u0438\u0448\u0438 \u043c\u0443\u043c\u043a\u0438\u043d \u044d\u043c\u0430\u0441 @@ -608,7 +608,7 @@ BiWeekly=\u0416\u0443\u0444\u0442 \u04b3\u0430\u0444\u0442\u0430 Monthly=\u041e\u0439\u043b\u0438\u043a BiMonthly=\u0416\u0443\u0444\u0442 \u043e\u0439\u043b\u0438\u043a Quarterly=\u04b2\u0430\u0440 \u043a\u0432\u0430\u0440\u0442\u0430\u043b -QuarterlyNov=Quarterly November +QuarterlyNov=\u0427\u043e\u0440\u0430\u043a\u043b\u0438\u043a \u041d\u043e\u044f\u0431\u0440\u044c SixMonthly=\u042f\u0440\u0438\u043c \u0439\u0438\u043b\u043b\u0438\u043a SixMonthlyApril=\u042f\u0440\u0438\u043c \u0439\u0438\u043b\u043b\u0438\u043a \u0410\u043f\u0440\u0435\u043b\u044c SixMonthlyNov=\u042f\u0440\u0438\u043c \u0439\u0438\u043b\u043b\u0438\u043a \u041d\u043e\u044f\u0431\u0440\u044c @@ -627,35 +627,35 @@ Every=\u04b2\u0430\u0440 \u0431\u0438\u0440 format.Daily.startDate=\u0439\u0439\u0439\u0439/\u043e\u0439/\u043a\u0443\u043d format.Daily.endDate= -format.Weekly.startDate=yyyy 'W' w +format.Weekly.startDate=yyyy ' \u0439 ' w format.Weekly.endDate = -format.WeeklyWednesday.startDate=yyyy 'W' w +format.WeeklyWednesday.startDate=yyyy ' \u0439 ' w format.WeeklyWednesday.endDate= -format.WeeklyThursday.startDate=yyyy 'W' w +format.WeeklyThursday.startDate=yyyy ' \u0439 ' w format.WeeklyThursday.endDate= -format.WeeklySaturday.startDate=yyyy 'W' w +format.WeeklySaturday.startDate=yyyy ' \u0439 ' w format.WeeklySaturday.endDate= -format.WeeklySunday.startDate=yyyy 'W' w +format.WeeklySunday.startDate=yyyy ' \u0439 ' w format.WeeklySunday.endDate= -format.BiWeekly.startDate=yyyy 'W' w 'to ' -format.BiWeekly.endDate='W' w +format.BiWeekly.startDate=yyyy ' \u0439 ' w '\u0433\u0430' +format.BiWeekly.endDate='\u0419 ' w format.Monthly.startDate=MMMM yyyy format.Monthly.endDate = -format.BiMonthly.startDate=MMM 'to ' +format.BiMonthly.startDate=\u043e\u0439 '\u0433\u0430 ' format.BiMonthly.endDate=MMM yyyy -format.Quarterly.startDate=MMM 'to ' +format.Quarterly.startDate=\u043e\u0439 '\u0433\u0430 ' format.Quarterly.endDate=MMM yyyy -format.QuarterlyNov.startDate=MMM 'to ' +format.QuarterlyNov.startDate=\u043e\u0439 '\u0433\u0430 ' format.QuarterlyNov.endDate=MMM yyyy -format.SixMonthly.startDate=MMM 'to ' +format.SixMonthly.startDate=\u043e\u0439 '\u0433\u0430 ' format.SixMonthly.endDate=MMM yyyy -format.SixMonthlyApril.startDate=MMM 'to ' +format.SixMonthlyApril.startDate=\u043e\u0439 '\u0433\u0430 ' format.SixMonthlyApril.endDate=MMM yyyy -format.SixMonthlyNov.startDate=MMM 'to ' +format.SixMonthlyNov.startDate=\u043e\u0439 '\u0433\u0430 ' format.SixMonthlyNov.endDate=MMM yyyy format.Yearly.startDate=yyyy format.Yearly.endDate = -format.TwoYearly.startDate=yyyy 'and ' +format.TwoYearly.startDate=yyyy '\u0432\u0430' format.TwoYearly.endDate=yyyy format.OnChange.startDate=yyyy-MM-dd '- ' format.OnChange.endDate=\u0439\u0439\u0439\u0439/\u043e\u0439/\u043a\u0443\u043d @@ -663,26 +663,26 @@ format.Survey.startDate=\u0439\u0439\u0439\u0439/\u043e\u0439/\u043a\u0443\u043d format.Survey.endDate = format.Relative.startDate=yyyy-MM-dd '- ' format.Relative.endDate=\u0439\u0439\u0439\u0439/\u043e\u0439/\u043a\u0443\u043d -format.FinancialApril.startDate=MMM yyyy 'to ' +format.FinancialApril.startDate=MMM yyyy '\u0433\u0430 ' format.FinancialApril.endDate=MMM yyyy -format.FinancialJuly.startDate=MMM yyyy 'to ' +format.FinancialJuly.startDate=MMM yyyy '\u0433\u0430 ' format.FinancialJuly.endDate=MMM yyyy -format.FinancialOct.startDate=MMM yyyy 'to ' +format.FinancialOct.startDate=MMM yyyy '\u0433\u0430 ' format.FinancialOct.endDate=MMM yyyy -format.FinancialNov.startDate=MMM yyyy 'to ' +format.FinancialNov.startDate=MMM yyyy '\u0433\u0430 ' format.FinancialNov.endDate=MMM yyyy #-- extended format for selected periods --------------------------------------# -format.BiMonthly.startDate.ext=MMMM '- ' +format.BiMonthly.startDate.ext=\u041e\u0439\u041e\u0439 '- ' format.BiMonthly.endDate.ext=MMMM yyyy -format.Quarterly.startDate.ext=MMMM '- ' +format.Quarterly.startDate.ext=\u041e\u0439\u041e\u0439 '- ' format.Quarterly.endDate.ext=MMMM yyyy -format.SixMonthly.startDate.ext=MMMM '- ' +format.SixMonthly.startDate.ext=\u041e\u0439\u041e\u0439 '- ' format.SixMonthly.endDate.ext=MMMM yyyy -format.SixMonthlyApril.startDate.ext=MMMM '- ' +format.SixMonthlyApril.startDate.ext=\u041e\u0439\u041e\u0439 '- ' format.SixMonthlyApril.endDate.ext=MMMM yyyy -format.SixMonthlyNov.startDate.ext=MMMM '- ' +format.SixMonthlyNov.startDate.ext=\u041e\u0439\u041e\u0439 '- ' format.SixMonthlyNov.endDate.ext=MMMM yyyy format.FinancialApril.startDate.ext=MMMM yyyy '- ' format.FinancialApril.endDate.ext=MMMM yyyy @@ -1697,7 +1697,7 @@ sign_in_with_openID=OpenID \u043e\u0440\u049b\u0430\u043b\u0438 \u0442\u0438\u04 change_language=\u0442\u0438\u043b\u043d\u0438 \u0442\u0430\u043d\u043b\u0430\u043d\u0433 login_with_google=Google \u043e\u0440\u049b\u0430\u043b\u0438 \u0442\u0438\u0437\u0438\u043c\u0433\u0430 \u043a\u0438\u0440\u0438\u043d\u0433 login_with_azure=\u041c\u0438\u043a\u0440\u043e\u0441\u043e\u0444\u0442 \u043e\u0440\u049b\u0430\u043b\u0438 \u0442\u0438\u0437\u0438\u043c\u0433\u0430 \u043a\u0438\u0440\u0438\u043d\u0433 -oidc_login_failed=Failed to login with OIDC +oidc_login_failed=OIDC \u043e\u0440\u049b\u0430\u043b\u0438 \u0442\u0438\u0437\u0438\u043c\u0433\u0430 \u043a\u0438\u0440\u043e\u043b\u043c\u0430\u0434\u0438 two_factor_code=\u041a\u043e\u0434 #-- New User Account page -----------------------------------------------------# @@ -1737,8 +1737,8 @@ deletion_in_progress=\u0418\u043b\u043e\u0432\u0430 \u04b3\u043e\u0437\u0438\u04 #-- Data value error codes ----------------------------------------------------# data_element_or_type_null_or_empty=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u0438 \u0451\u043a\u0438 \u0442\u0443\u0440\u0438 \u043d\u043e\u043b\u044c \u0451\u043a\u0438 \u0431\u045e\u0448 -data_element_lacks_option_set=Data elements with value type MULTI_TEXT must have an option set -value_not_valid_option=Data value contains value(s) other than the valid options of the data element's option set +data_element_lacks_option_set=MULTI_TEXT \u0442\u0438\u043f\u0434\u0430\u0433\u0438 \u049b\u0438\u0439\u043c\u0430\u0442\u0433\u0430 \u044d\u0433\u0430 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043b\u0430\u0440 \u0442\u0443\u0440\u0438\u0433\u0430 \u044d\u0433\u0430 \u0431\u045e\u043b\u0438\u0448\u0438 \u043b\u043e\u0437\u0438\u043c +value_not_valid_option=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438\u0434\u0430 \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u0438 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043b\u0430\u0440 \u0442\u045e\u043f\u043b\u0430\u043c\u0438\u043d\u0438\u043d\u0433 \u0436\u043e\u0440\u0438\u0439 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043b\u0430\u0440\u0438\u0434\u0430\u043d \u0431\u043e\u0448\u049b\u0430 \u049b\u0438\u0439\u043c\u0430\u0442(\u043b\u0430\u0440) \u043c\u0430\u0432\u0436\u0443\u0434 value_length_greater_than_max_length=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b \u0443\u0437\u0443\u043d\u043b\u0430\u043a\u0434\u0430\u043d \u043e\u0448\u0438\u049b\u0440\u043e\u049b value_not_numeric=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u0440\u0430\u049b\u0430\u043c\u043b\u0438 \u044d\u043c\u0430\u0441 value_required_but_not_provided=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u0442\u0430\u043b\u0430\u0431 \u044d\u0442\u0438\u043b\u0430\u0434\u0438, \u043b\u0435\u043a\u0438\u043d \u0443 \u043a\u0438\u0440\u0438\u0442\u0438\u043b\u043c\u0430\u0433\u0430\u043d \u044d\u0434\u0438 @@ -1751,7 +1751,7 @@ value_is_zero_and_not_zero_significant=\u041c\u0430\u044a\u043b\u0443\u043c\u043 value_not_bool=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u043c\u0430\u043d\u0442\u0438\u049b\u0430\u043d \u0445\u0430\u0442\u043e value_not_true_only=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u043d\u043e\u0442\u045e\u0493\u0440\u0438 value_not_valid_date=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u04b3\u0430\u049b\u0438\u049b\u0438\u0439 \u0441\u0430\u043d\u0430 \u044d\u043c\u0430\u0441 -value_not_valid_letter=Data value is not a valid letter +value_not_valid_letter=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u049b\u0438\u0439\u043c\u0430\u0442\u0438 \u04b3\u0430\u049b\u0438\u049b\u0438\u0439 \u04b3\u0430\u0440\u0444 \u044d\u043c\u0430\u0441 stored_by_length_greater_than_max_length=\u041c\u0430\u043a\u0441\u0438\u043c\u0430\u043b \u0443\u0437\u0443\u043d\u043b\u0438\u043a\u0434\u0430\u043d \u043a\u045e\u043f\u0440\u043e\u049b \u0441\u0430\u049b\u043b\u0430\u043d\u0434\u0438 comment_length_greater_than_max_length=\u0418\u0437\u043e\u04b3 \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b \u0440\u0443\u0445\u0441\u0430\u0442 \u0431\u0435\u0440\u0438\u043b\u0433\u0430\u043d \u0443\u0437\u0443\u043d\u043b\u0438\u043a\u0434\u0430\u043d \u043e\u0448\u0438\u049b\u0440\u043e\u049b @@ -1805,8 +1805,8 @@ enrollment_date=\u049a\u0430\u0431\u0443\u043b \u049b\u0438\u043b\u0438\u043d\u0 current_date=\u04b2\u043e\u0437\u0438\u0440\u0433\u0438 \u0441\u0430\u043d\u0430 value_count=\u049a\u0438\u0439\u043c\u0430\u0442\u043b\u0430\u0440 \u0441\u043e\u043d\u0438 zero_pos_value_count=\u041d\u043e\u043b\u044c \u0451\u043a\u0438 \u043c\u0443\u0441\u0431\u0430\u0442 \u049b\u0438\u0439\u043c\u0430\u0442\u043b\u0430\u0440 \u0441\u043e\u043d\u0438 -event_count=Event count (ACTIVE or COMPLETED status only) -scheduled_event_count=Event count (SCHEDULE status only) +event_count=\u04b2\u043e\u0434\u0438\u0441\u0430\u043b\u0430\u0440 \u0441\u043e\u043d\u0438 (\u0444\u0430\u049b\u0430\u0442 \u0410\u041a\u0422\u0418\u0412 \u0451\u043a\u0438 \u0422\u0423\u0413\u0410\u041b\u041b\u0410\u041d\u0413\u0410\u041d \u0441\u0442\u0430\u0442\u0443\u0441\u0434\u0430\u0433\u0438\u043b\u0430\u0440) +scheduled_event_count=\u04b2\u043e\u0434\u0438\u0441\u0430\u043b\u0430\u0440 \u0441\u043e\u043d\u0438 (\u0444\u0430\u049b\u0430\u0442 \u0420\u0415\u0416\u0410\u041b\u0410\u0428\u0422\u0418\u0420\u0418\u041b\u0413\u0410\u041d \u0441\u0442\u0430\u0442\u0443\u0441\u0434\u0430\u0433\u0438\u043b\u0430\u0440) org_unit_count=\u0422\u0430\u0448\u043a\u0438\u043b\u0438\u0439 \u0431\u045e\u043b\u0438\u043c \u0441\u043e\u043d\u0438 enrollment_count=\u0420\u045e\u0439\u0445\u0430\u0442\u0433\u0430 \u043e\u043b\u0438\u043d\u0433\u0430\u043d\u043b\u0430\u0440 \u0441\u043e\u043d\u0438 tei_count=\u041a\u0443\u0437\u0430\u0442\u0443\u0432\u0434\u0430\u0433\u0438 \u0448\u0430\u0445\u0441 \u043d\u0430\u043c\u0443\u043d\u0430\u0441\u0438 \u0441\u043e\u043d\u0438 @@ -1906,16 +1906,16 @@ notification_comment_update=\u0438\u0437\u043e\u04b3\u043b\u0430\u043d\u0433\u04 # -- Start Data Integrity Checks--------------------------------------------# #Java based checks -data_integrity.data_elements_without_data_sets.section=Data Elements -data_integrity.data_elements_without_data_sets.name=Data elements lacking data Sets +data_integrity.data_elements_without_data_sets.section=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438 +data_integrity.data_elements_without_data_sets.name=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0442\u045e\u043f\u043b\u0430\u043c\u0438\u0433\u0430 \u044d\u0433\u0430 \u0431\u045e\u043b\u043c\u0430\u0433\u0430\u043d \u043c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438 data_integrity.data_elements_without_data_sets.description=Lists all data elements that have no data sets -data_integrity.data_elements_without_groups.section=Data Elements +data_integrity.data_elements_without_groups.section=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438 data_integrity.data_elements_without_groups.name=Data elements lacking groups data_integrity.data_elements_without_groups.description=Lists all data elements that have no data element groups -data_integrity.data_elements_assigned_to_data_sets_with_different_period_types.section=Data Elements +data_integrity.data_elements_assigned_to_data_sets_with_different_period_types.section=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438 data_integrity.data_elements_assigned_to_data_sets_with_different_period_types.name=Data elements assigned to data sets with different period type data_integrity.data_elements_assigned_to_data_sets_with_different_period_types.description=Lists all data elements that are assigned to at least one data set that has a different period type -data_integrity.data_elements_violating_exclusive_group_sets.section=Data Elements +data_integrity.data_elements_violating_exclusive_group_sets.section=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440\u0438 data_integrity.data_elements_violating_exclusive_group_sets.name=Data elements with conflicting exclusive group sets data_integrity.data_elements_violating_exclusive_group_sets.description=Lists all data elements that are a members of more than one exclusive data element set belonging to the same data element group set data_integrity.data_elements_violating_exclusive_group_sets.recommendation=Either remove the data element from all but one exclusive group set or consider if the set really should be an exclusive group set diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/fileresource/FileResourceServiceTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/fileresource/FileResourceServiceTest.java index ddb65176e77d..f211fdbdac70 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/fileresource/FileResourceServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/fileresource/FileResourceServiceTest.java @@ -102,7 +102,7 @@ void verifySaveFile() { File file = new File(""); - subject.saveFileResource(fileResource, file); + subject.asyncSaveFileResource(fileResource, file); verify(fileResourceStore).save(fileResource); verify(entityManager).flush(); @@ -122,7 +122,8 @@ void verifySaveIllegalFileTypeResourceA() { "very_evil_script.html", "text/html", 1024, "md5", FileResourceDomain.USER_AVATAR); File file = new File("very_evil_script.html"); - assertThrows(IllegalQueryException.class, () -> subject.saveFileResource(fileResource, file)); + assertThrows( + IllegalQueryException.class, () -> subject.asyncSaveFileResource(fileResource, file)); } @Test @@ -136,7 +137,8 @@ void verifySaveIllegalFileTypeResourceB() { FileResourceDomain.MESSAGE_ATTACHMENT); File file = new File("suspicious_program.rpm"); - assertThrows(IllegalQueryException.class, () -> subject.saveFileResource(fileResource, file)); + assertThrows( + IllegalQueryException.class, () -> subject.asyncSaveFileResource(fileResource, file)); } @Test @@ -157,7 +159,7 @@ void verifySaveImageFile() { fileResource.setUid("imageUid1"); - subject.saveFileResource(fileResource, file); + subject.asyncSaveFileResource(fileResource, file); verify(fileResourceStore).save(fileResource); verify(entityManager).flush(); @@ -211,7 +213,7 @@ void verifySaveOrgUnitImageFile() { fileResource.setUid("imageUid1"); - subject.saveFileResource(fileResource, file); + subject.asyncSaveFileResource(fileResource, file); verify(fileResourceStore).save(fileResource); verify(entityManager).flush(); diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/DefaultQueryServiceTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/DefaultQueryServiceTest.java index 422042621a6f..c35b8a212d27 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/DefaultQueryServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/DefaultQueryServiceTest.java @@ -42,6 +42,7 @@ import org.hisp.dhis.query.planner.QueryPlanner; import org.hisp.dhis.schema.SchemaService; import org.hisp.dhis.schema.descriptors.OrganisationUnitSchemaDescriptor; +import org.hisp.dhis.setting.SystemSettingManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -65,9 +66,11 @@ class DefaultQueryServiceTest { @Mock private SchemaService schemaService; + @Mock private SystemSettingManager systemSettingManager; + @BeforeEach public void setUp() { - QueryPlanner queryPlanner = new DefaultQueryPlanner(schemaService); + QueryPlanner queryPlanner = new DefaultQueryPlanner(schemaService, systemSettingManager); subject = new DefaultQueryService( queryParser, queryPlanner, criteriaQueryEngine, inMemoryQueryEngine); diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/planner/DefaultQueryPlannerTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/planner/DefaultQueryPlannerTest.java index c0268674b7b6..5a289026af85 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/planner/DefaultQueryPlannerTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/query/planner/DefaultQueryPlannerTest.java @@ -44,6 +44,7 @@ import org.hisp.dhis.schema.SchemaService; import org.hisp.dhis.schema.descriptors.DataElementSchemaDescriptor; import org.hisp.dhis.schema.descriptors.OrganisationUnitSchemaDescriptor; +import org.hisp.dhis.setting.SystemSettingManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -60,9 +61,11 @@ class DefaultQueryPlannerTest { @Mock private SchemaService schemaService; + @Mock private SystemSettingManager systemSettingManager; + @BeforeEach public void setUp() { - this.subject = new DefaultQueryPlanner(schemaService); + this.subject = new DefaultQueryPlanner(schemaService, systemSettingManager); } @Test diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/system/SystemInfoTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/system/SystemInfoTest.java index c418936f735f..de72bbcc6905 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/system/SystemInfoTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/system/SystemInfoTest.java @@ -27,33 +27,60 @@ */ package org.hisp.dhis.system; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + import org.hisp.dhis.system.database.DatabaseInfo; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -/** - * Unit tests for {@link SystemInfo}. - * - * @author Volker Schmidt - */ +/** Unit tests for {@link SystemInfo}. */ class SystemInfoTest { - private DatabaseInfo databaseInfo; - - private SystemInfo systemInfo; + @Test + void testWithoutSensitiveInfo() { + SystemInfo info = + SystemInfo.builder() + .jasperReportsVersion("x") + .environmentVariable("x") + .fileStoreProvider("x") + .readOnlyMode("x") + .nodeId("x") + .javaVersion("x") + .javaVendor("x") + .javaOpts("x") + .osName("x") + .osArchitecture("x") + .externalDirectory("x") + .readReplicaCount(-1) + .memoryInfo("x") + .cpuCores(-1) + .systemMonitoringUrl("x") + .encryption(true) + .redisEnabled(true) + .redisHostname("x") + .databaseInfo(DatabaseInfo.builder().name("x").build()) + .build(); - @BeforeEach - void setUp() { - databaseInfo = new DatabaseInfo(); - systemInfo = new SystemInfo(); - systemInfo.setDatabaseInfo(databaseInfo); - } + info = info.withoutSensitiveInfo(); - @Test - void instance() { - final SystemInfo si = systemInfo.instance(); - Assertions.assertNotSame(systemInfo, si); - Assertions.assertNotSame(systemInfo.getDatabaseInfo(), si.getDatabaseInfo()); + assertNull(info.getJasperReportsVersion()); + assertNull(info.getEnvironmentVariable()); + assertNull(info.getFileStoreProvider()); + assertNull(info.getReadOnlyMode()); + assertNull(info.getNodeId()); + assertNull(info.getJavaVersion()); + assertNull(info.getJavaVendor()); + assertNull(info.getJavaOpts()); + assertNull(info.getOsName()); + assertNull(info.getOsArchitecture()); + assertNull(info.getExternalDirectory()); + assertNull(info.getReadReplicaCount()); + assertNull(info.getMemoryInfo()); + assertNull(info.getCpuCores()); + assertNull(info.getSystemMonitoringUrl()); + assertNull(info.getRedisHostname()); + assertFalse(info.isRedisEnabled()); + assertFalse(info.isEncryption()); + assertNull(info.getDatabaseInfo().getName()); } } diff --git a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/trackedentity/job/TrackerTrigramIndexingJobTest.java b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/trackedentity/job/TrackerTrigramIndexingJobTest.java index 1f7a0e310e19..1258d62b7fab 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/trackedentity/job/TrackerTrigramIndexingJobTest.java +++ b/dhis-2/dhis-services/dhis-service-core/src/test/java/org/hisp/dhis/trackedentity/job/TrackerTrigramIndexingJobTest.java @@ -27,7 +27,13 @@ */ package org.hisp.dhis.trackedentity.job; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/dataset/DefaultCompleteDataSetRegistrationExchangeService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/dataset/DefaultCompleteDataSetRegistrationExchangeService.java index d5b1b2b653d1..20141e225467 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/dataset/DefaultCompleteDataSetRegistrationExchangeService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/dataset/DefaultCompleteDataSetRegistrationExchangeService.java @@ -666,9 +666,7 @@ private void validateOrgUnitInUserHierarchy( boolean inUserHierarchy = mdCaches .getOrgUnitInHierarchyMap() - .get( - mdProps.orgUnit.getUid(), - () -> organisationUnitService.isDescendant(mdProps.orgUnit, userOrgUnits)); + .get(mdProps.orgUnit.getUid(), () -> mdProps.orgUnit.isDescendant(userOrgUnits)); if (!inUserHierarchy) { throw new ImportConflictException( @@ -726,8 +724,7 @@ private void validateAttrOptCombo( aocOrgUnitKey, () -> { Set aocOrgUnits = aoc.getOrganisationUnits(); - return aocOrgUnits == null - || organisationUnitService.isDescendant(mdProps.orgUnit, aocOrgUnits); + return aocOrgUnits == null || mdProps.orgUnit.isDescendant(aocOrgUnits); }); if (!isOrgUnitValidForAoc) { diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/DefaultMonitoringService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/DefaultMonitoringService.java index 984c95385ae8..ca431e5a704f 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/DefaultMonitoringService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/monitoring/DefaultMonitoringService.java @@ -96,8 +96,7 @@ public void pushMonitoringInfo() { return; } - SystemInfo systemInfo = systemService.getSystemInfo(); - systemInfo.clearSensitiveInfo(); + SystemInfo systemInfo = systemService.getSystemInfo().withoutSensitiveInfo(); if (StringUtils.isBlank(systemInfo.getSystemId())) { log.warn("System ID not available, aborting monitoring request"); diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfDataEntryFormUtil.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfDataEntryFormUtil.java index 741ce96e437c..a510cf1312da 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfDataEntryFormUtil.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfDataEntryFormUtil.java @@ -37,7 +37,7 @@ import com.lowagie.text.pdf.AcroFields; import com.lowagie.text.pdf.PdfPCell; import com.lowagie.text.pdf.PdfReader; -import java.awt.*; +import java.awt.Color; import java.io.InputStream; import java.util.ArrayList; import java.util.List; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfFormFontSettings.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfFormFontSettings.java index d6ab3066a354..32d92741e507 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfFormFontSettings.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/pdfform/PdfFormFontSettings.java @@ -30,7 +30,7 @@ import com.lowagie.text.Font; import com.lowagie.text.FontFactory; import com.lowagie.text.pdf.BaseFont; -import java.awt.*; +import java.awt.Color; import java.util.HashMap; import java.util.Locale; import java.util.Map; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java index 55e9842b059b..912296a1de26 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/webmessage/WebMessageUtils.java @@ -27,9 +27,10 @@ */ package org.hisp.dhis.dxf2.webmessage; +import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist; + import java.sql.SQLException; import java.util.List; -import java.util.Optional; import java.util.function.Supplier; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.dxf2.importsummary.ImportStatus; @@ -273,19 +274,4 @@ public static ErrorCode getErrorCode(SQLException ex) { } return ErrorCode.E7145; } - - /** - * Utility method to detect if the {@link SQLException} refers to a missing relation in the - * database. - * - * @param ex a {@link SQLException} to analyze - * @return true if the error is a missing relation error, false otherwise - */ - public static boolean relationDoesNotExist(SQLException ex) { - if (ex != null) { - return Optional.of(ex).map(SQLException::getSQLState).filter("42P01"::equals).isPresent(); - } - - return false; - } } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/EventDataValueAggregatorTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/EventDataValueAggregatorTest.java index 95505b2ddfe1..fcd85e0f166a 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/EventDataValueAggregatorTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/context/EventDataValueAggregatorTest.java @@ -28,7 +28,10 @@ package org.hisp.dhis.dxf2.deprecated.tracker.importer.context; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import com.google.common.collect.Lists; import com.google.common.collect.Sets; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/preprocess/EnrollmentPreProcessorTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/preprocess/EnrollmentPreProcessorTest.java index 0146b2e060a7..c1caef4aafdd 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/preprocess/EnrollmentPreProcessorTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/preprocess/EnrollmentPreProcessorTest.java @@ -30,9 +30,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import static org.hisp.dhis.DhisConvenienceTest.*; import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; import static org.hisp.dhis.DhisConvenienceTest.createProgram; +import static org.hisp.dhis.DhisConvenienceTest.createTrackedEntity; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/validation/EnrollmentRepeatableStageCheckTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/validation/EnrollmentRepeatableStageCheckTest.java index a56a543a7dbc..8d26207f359d 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/validation/EnrollmentRepeatableStageCheckTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/insert/validation/EnrollmentRepeatableStageCheckTest.java @@ -27,10 +27,10 @@ */ package org.hisp.dhis.dxf2.deprecated.tracker.importer.insert.validation; -import static org.hisp.dhis.DhisConvenienceTest.*; import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; import static org.hisp.dhis.DhisConvenienceTest.createProgram; import static org.hisp.dhis.DhisConvenienceTest.createProgramStage; +import static org.hisp.dhis.DhisConvenienceTest.createTrackedEntity; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/shared/validation/EnrollmentCheckTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/shared/validation/EnrollmentCheckTest.java index 6d5ffcad596c..cdcbb3e987ef 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/shared/validation/EnrollmentCheckTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/importer/shared/validation/EnrollmentCheckTest.java @@ -27,9 +27,9 @@ */ package org.hisp.dhis.dxf2.deprecated.tracker.importer.shared.validation; -import static org.hisp.dhis.DhisConvenienceTest.*; import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; import static org.hisp.dhis.DhisConvenienceTest.createProgram; +import static org.hisp.dhis.DhisConvenienceTest.createTrackedEntity; import static org.mockito.Mockito.when; import com.google.common.collect.Lists; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/importsummary/ImportSummariesTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/importsummary/ImportSummariesTest.java index a539be7aeb14..458de701cf11 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/importsummary/ImportSummariesTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/importsummary/ImportSummariesTest.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.dxf2.importsummary; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/DefaultMetadataExportServiceTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/DefaultMetadataExportServiceTest.java index c5bd942d5542..b3c475fe7ce3 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/DefaultMetadataExportServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/DefaultMetadataExportServiceTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.dxf2.metadata; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.when; import java.util.Arrays; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/attribute/MetadataAttributeCheckTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/attribute/MetadataAttributeCheckTest.java index 65fc4dc9ee26..e82fe41fa83c 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/attribute/MetadataAttributeCheckTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/attribute/MetadataAttributeCheckTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.dxf2.metadata.attribute; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import com.google.common.collect.Lists; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/JobConfigurationObjectBundleHookTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/JobConfigurationObjectBundleHookTest.java index 78a39b3cab62..e717189c3428 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/JobConfigurationObjectBundleHookTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/objectbundle/hooks/JobConfigurationObjectBundleHookTest.java @@ -216,7 +216,7 @@ void validateDelayForFixedIntervalTypeJobs() { jobConfiguration.setUid(jobConfigUid); jobConfiguration.setJobType(JobType.CONTINUOUS_ANALYTICS_TABLE); jobConfiguration.setSchedulingType(SchedulingType.FIXED_DELAY); - jobConfiguration.setJobParameters(new ContinuousAnalyticsJobParameters(1, null, null)); + jobConfiguration.setJobParameters(new ContinuousAnalyticsJobParameters(1, null, null, true)); List errorReports = hook.validate(jobConfiguration, null); assertEquals(1, errorReports.size()); diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/sync/MetadataSyncDelegateTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/sync/MetadataSyncDelegateTest.java index facdd5e88a45..9ecb0f2ef2f7 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/sync/MetadataSyncDelegateTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/metadata/sync/MetadataSyncDelegateTest.java @@ -66,8 +66,7 @@ class MetadataSyncDelegateTest { void testShouldVerifyIfStopSyncReturnFalseIfNoSystemVersionInLocal() { String versionSnapshot = "{\"system:\": {\"date\":\"2016-05-24T05:27:25.128+0000\", \"version\": \"2.26\"}, \"name\":\"testVersion\",\"created\":\"2016-05-26T11:43:59.787+0000\",\"type\":\"BEST_EFFORT\",\"id\":\"ktwh8PHNwtB\",\"hashCode\":\"12wa32d4f2et3tyt5yu6i\"}"; - SystemInfo systemInfo = new SystemInfo(); - when(systemService.getSystemInfo()).thenReturn(systemInfo); + when(systemService.getSystemInfo()).thenReturn(SystemInfo.builder().build()); boolean shouldStopSync = metadataSyncDelegate.shouldStopSync(versionSnapshot); assertFalse(shouldStopSync); } @@ -76,9 +75,7 @@ void testShouldVerifyIfStopSyncReturnFalseIfNoSystemVersionInLocal() { void testShouldVerifyIfStopSyncReturnFalseIfNoSystemVersionInRemote() { String versionSnapshot = "{\"system:\": {\"date\":\"2016-05-24T05:27:25.128+0000\", \"version\": \"2.26\"}, \"name\":\"testVersion\",\"created\":\"2016-05-26T11:43:59.787+0000\",\"type\":\"BEST_EFFORT\",\"id\":\"ktwh8PHNwtB\",\"hashCode\":\"12wa32d4f2et3tyt5yu6i\"}"; - SystemInfo systemInfo = new SystemInfo(); - systemInfo.setVersion("2.26"); - when(systemService.getSystemInfo()).thenReturn(systemInfo); + when(systemService.getSystemInfo()).thenReturn(SystemInfo.builder().version("2.26").build()); boolean shouldStopSync = metadataSyncDelegate.shouldStopSync(versionSnapshot); assertFalse(shouldStopSync); } @@ -89,9 +86,7 @@ void testShouldVerifyIfStopSyncReturnTrueIfDHISVersionMismatch() throws IOExcept "{\"system:\": {\"date\":\"2016-06-24T05:27:25.128+0000\", \"version\": \"2.26\"}, \"name\":\"testVersion\",\"created\":\"2016-05-26T11:43:59.787+0000\",\"type\":\"BEST_EFFORT\",\"id\":\"ktwh8PHNwtB\"," + "\"hashCode\":\"12wa32d4f2et3tyt5yu6i\"}"; String systemNodeString = "{\"date\":\"2016-06-24T05:27:25.128+0000\", \"version\": \"2.26\"}"; - SystemInfo systemInfo = new SystemInfo(); - systemInfo.setVersion("2.25"); - when(systemService.getSystemInfo()).thenReturn(systemInfo); + when(systemService.getSystemInfo()).thenReturn(SystemInfo.builder().version("2.25").build()); when(metadataSystemSettingService.getStopMetadataSyncSetting()).thenReturn(true); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonNode = mapper.readTree(systemNodeString); @@ -107,9 +102,7 @@ void testShouldVerifyIfStopSyncReturnFalseIfDHISVersionSame() throws IOException String versionSnapshot = "{\"system:\": {\"date\":\"2016-05-24T05:27:25.128+0000\", \"version\": \"2.26\"}, \"name\":\"testVersion\",\"created\":\"2016-05-26T11:43:59.787+0000\",\"type\":\"BEST_EFFORT\",\"id\":\"ktwh8PHNwtB\",\"hashCode\":\"12wa32d4f2et3tyt5yu6i\"}"; String systemNodeString = "{\"date\":\"2016-05-24T05:27:25.128+0000\", \"version\": \"2.26\"}"; - SystemInfo systemInfo = new SystemInfo(); - systemInfo.setVersion("2.26"); - when(systemService.getSystemInfo()).thenReturn(systemInfo); + when(systemService.getSystemInfo()).thenReturn(SystemInfo.builder().version("2.26").build()); when(metadataSystemSettingService.getStopMetadataSyncSetting()).thenReturn(true); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonNode = mapper.readTree(systemNodeString); @@ -124,10 +117,7 @@ void testShouldVerifyIfStopSyncReturnFalseIfDHISVersionSame() throws IOException void testShouldVerifyIfStopSyncReturnFalseIfStopSyncIsNotSet() { String versionSnapshot = "{\"system:\": {\"date\":\"2016-05-24T05:27:25.128+0000\", \"version\": \"2.26\"}, \"name\":\"testVersion\",\"created\":\"2016-05-26T11:43:59.787+0000\",\"type\":\"BEST_EFFORT\",\"id\":\"ktwh8PHNwtB\",\"hashCode\":\"12wa32d4f2et3tyt5yu6i\"}"; - SystemInfo systemInfo = new SystemInfo(); - systemInfo.setVersion("2.26"); - - when(systemService.getSystemInfo()).thenReturn(systemInfo); + when(systemService.getSystemInfo()).thenReturn(SystemInfo.builder().version("2.26").build()); when(metadataSystemSettingService.getStopMetadataSyncSetting()).thenReturn(false); boolean shouldStopSync = metadataSyncDelegate.shouldStopSync(versionSnapshot); assertFalse(shouldStopSync); diff --git a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/config/ProgramRuleConfig.java b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/config/ProgramRuleConfig.java index caf53e737f85..a68854d4f7ca 100644 --- a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/config/ProgramRuleConfig.java +++ b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/config/ProgramRuleConfig.java @@ -29,7 +29,12 @@ import org.hisp.dhis.constant.ConstantService; import org.hisp.dhis.programrule.ProgramRuleVariableService; -import org.hisp.dhis.programrule.engine.*; +import org.hisp.dhis.programrule.engine.NotificationImplementableRuleService; +import org.hisp.dhis.programrule.engine.ProgramRuleEngine; +import org.hisp.dhis.programrule.engine.ProgramRuleEngineListener; +import org.hisp.dhis.programrule.engine.ProgramRuleEntityMapperService; +import org.hisp.dhis.programrule.engine.ServerSideImplementableRuleService; +import org.hisp.dhis.programrule.engine.SupplementaryDataProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/DefaultProgramRuleEntityMapperService.java b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/DefaultProgramRuleEntityMapperService.java index 2d277f2bca9a..7f38593abb7a 100644 --- a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/DefaultProgramRuleEntityMapperService.java +++ b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/DefaultProgramRuleEntityMapperService.java @@ -33,7 +33,11 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; @@ -44,11 +48,45 @@ import org.hisp.dhis.i18n.I18nManager; import org.hisp.dhis.program.Enrollment; import org.hisp.dhis.program.Event; -import org.hisp.dhis.programrule.*; +import org.hisp.dhis.programrule.ProgramRule; +import org.hisp.dhis.programrule.ProgramRuleAction; +import org.hisp.dhis.programrule.ProgramRuleActionType; +import org.hisp.dhis.programrule.ProgramRuleService; +import org.hisp.dhis.programrule.ProgramRuleVariable; +import org.hisp.dhis.programrule.ProgramRuleVariableService; +import org.hisp.dhis.programrule.ProgramRuleVariableSourceType; import org.hisp.dhis.rules.DataItem; import org.hisp.dhis.rules.ItemValueType; import org.hisp.dhis.rules.Option; -import org.hisp.dhis.rules.models.*; +import org.hisp.dhis.rules.models.AttributeType; +import org.hisp.dhis.rules.models.Rule; +import org.hisp.dhis.rules.models.RuleAction; +import org.hisp.dhis.rules.models.RuleActionAssign; +import org.hisp.dhis.rules.models.RuleActionCreateEvent; +import org.hisp.dhis.rules.models.RuleActionDisplayKeyValuePair; +import org.hisp.dhis.rules.models.RuleActionDisplayText; +import org.hisp.dhis.rules.models.RuleActionErrorOnCompletion; +import org.hisp.dhis.rules.models.RuleActionHideField; +import org.hisp.dhis.rules.models.RuleActionHideProgramStage; +import org.hisp.dhis.rules.models.RuleActionHideSection; +import org.hisp.dhis.rules.models.RuleActionScheduleMessage; +import org.hisp.dhis.rules.models.RuleActionSendMessage; +import org.hisp.dhis.rules.models.RuleActionSetMandatoryField; +import org.hisp.dhis.rules.models.RuleActionShowError; +import org.hisp.dhis.rules.models.RuleActionShowWarning; +import org.hisp.dhis.rules.models.RuleActionWarningOnCompletion; +import org.hisp.dhis.rules.models.RuleAttributeValue; +import org.hisp.dhis.rules.models.RuleDataValue; +import org.hisp.dhis.rules.models.RuleEnrollment; +import org.hisp.dhis.rules.models.RuleEvent; +import org.hisp.dhis.rules.models.RuleValueType; +import org.hisp.dhis.rules.models.RuleVariable; +import org.hisp.dhis.rules.models.RuleVariableAttribute; +import org.hisp.dhis.rules.models.RuleVariableCalculatedValue; +import org.hisp.dhis.rules.models.RuleVariableCurrentEvent; +import org.hisp.dhis.rules.models.RuleVariableNewestEvent; +import org.hisp.dhis.rules.models.RuleVariableNewestStageEvent; +import org.hisp.dhis.rules.models.RuleVariablePreviousEvent; import org.hisp.dhis.rules.utils.RuleEngineUtils; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.trackedentityattributevalue.TrackedEntityAttributeValue; diff --git a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEngine.java b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEngine.java index b109df5b206a..de38cdd5c9fd 100644 --- a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEngine.java +++ b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/ProgramRuleEngine.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.programrule.engine; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; @@ -46,7 +49,12 @@ import org.hisp.dhis.rules.RuleEngine; import org.hisp.dhis.rules.RuleEngineContext; import org.hisp.dhis.rules.RuleEngineIntent; -import org.hisp.dhis.rules.models.*; +import org.hisp.dhis.rules.models.RuleEffect; +import org.hisp.dhis.rules.models.RuleEffects; +import org.hisp.dhis.rules.models.RuleEnrollment; +import org.hisp.dhis.rules.models.RuleEvent; +import org.hisp.dhis.rules.models.RuleValidationResult; +import org.hisp.dhis.rules.models.TriggerEnvironment; import org.hisp.dhis.trackedentityattributevalue.TrackedEntityAttributeValue; /** diff --git a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/RuleActionSendMessageImplementer.java b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/RuleActionSendMessageImplementer.java index 291e2152c84a..63c1d7a04641 100644 --- a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/RuleActionSendMessageImplementer.java +++ b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/engine/RuleActionSendMessageImplementer.java @@ -37,7 +37,8 @@ import org.hisp.dhis.program.EnrollmentService; import org.hisp.dhis.program.Event; import org.hisp.dhis.program.EventService; -import org.hisp.dhis.program.notification.*; +import org.hisp.dhis.program.notification.ProgramNotificationTemplate; +import org.hisp.dhis.program.notification.ProgramNotificationTemplateService; import org.hisp.dhis.program.notification.event.ProgramRuleEnrollmentEvent; import org.hisp.dhis.program.notification.event.ProgramRuleStageEvent; import org.hisp.dhis.rules.models.RuleAction; diff --git a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/hibernate/HibernateProgramRuleStore.java b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/hibernate/HibernateProgramRuleStore.java index d093280d2a3a..e20a9d79bcd3 100644 --- a/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/hibernate/HibernateProgramRuleStore.java +++ b/dhis-2/dhis-services/dhis-service-program-rule/src/main/java/org/hisp/dhis/programrule/hibernate/HibernateProgramRuleStore.java @@ -36,7 +36,11 @@ import org.hibernate.Session; import org.hisp.dhis.common.hibernate.HibernateIdentifiableObjectStore; import org.hisp.dhis.program.Program; -import org.hisp.dhis.programrule.*; +import org.hisp.dhis.programrule.ProgramRule; +import org.hisp.dhis.programrule.ProgramRuleActionEvaluationEnvironment; +import org.hisp.dhis.programrule.ProgramRuleActionEvaluationTime; +import org.hisp.dhis.programrule.ProgramRuleActionType; +import org.hisp.dhis.programrule.ProgramRuleStore; import org.hisp.dhis.query.JpaQueryUtils; import org.hisp.dhis.security.acl.AclService; import org.hisp.dhis.user.CurrentUserService; diff --git a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/mapgeneration/InternalMapObject.java b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/mapgeneration/InternalMapObject.java index 2f769a38b97c..7b3dbf9ce528 100644 --- a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/mapgeneration/InternalMapObject.java +++ b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/mapgeneration/InternalMapObject.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.mapgeneration; -import java.awt.*; +import java.awt.Color; import org.geotools.data.DataUtilities; import org.geotools.feature.SchemaException; import org.geotools.styling.SLD; diff --git a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/pushanalysis/DefaultPushAnalysisService.java b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/pushanalysis/DefaultPushAnalysisService.java index e2b1d7912270..36ca62b4539a 100644 --- a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/pushanalysis/DefaultPushAnalysisService.java +++ b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/pushanalysis/DefaultPushAnalysisService.java @@ -441,7 +441,7 @@ private String saveFileResource(FileResource fileResource, byte[] bytes) { fileResource.setAssigned(true); - String fileResourceUid = fileResourceService.saveFileResource(fileResource, bytes); + String fileResourceUid = fileResourceService.asyncSaveFileResource(fileResource, bytes); externalFileResource.setFileResource(fileResourceService.getFileResource(fileResourceUid)); diff --git a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java index 8417a297c943..133818de4df6 100644 --- a/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java +++ b/dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/validation/ValidationRunContext.java @@ -29,10 +29,17 @@ import static java.util.Objects.requireNonNull; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import lombok.*; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; import org.hisp.dhis.category.CategoryOption; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.category.CategoryOptionGroup; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java index 5ba70ac49ffe..dc8010750a34 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/event/JdbcEventStore.java @@ -169,7 +169,9 @@ class JdbcEventStore implements EventStore { private static final String COLUMN_EVENT_LAST_UPDATED_BY = "ev_lastupdatedbyuserinfo"; private static final String COLUMN_EVENT_CREATED_BY = "ev_createdbyuserinfo"; private static final String COLUMN_EVENT_CREATED = "ev_created"; + private static final String COLUMN_EVENT_CREATED_AT_CLIENT = "ev_createdatclient"; private static final String COLUMN_EVENT_LAST_UPDATED = "ev_lastupdated"; + private static final String COLUMN_EVENT_LAST_UPDATED_AT_CLIENT = "ev_lastupdatedatclient"; private static final String COLUMN_EVENT_COMPLETED_BY = "ev_completedby"; private static final String COLUMN_EVENT_ATTRIBUTE_OPTION_COMBO_UID = "coc_uid"; private static final String COLUMN_EVENT_COMPLETED_DATE = "ev_completeddate"; @@ -206,7 +208,9 @@ class JdbcEventStore implements EventStore { entry("lastUpdatedBy", COLUMN_EVENT_LAST_UPDATED_BY), entry("createdBy", COLUMN_EVENT_CREATED_BY), entry("created", COLUMN_EVENT_CREATED), + entry("createdAtClient", COLUMN_EVENT_CREATED_AT_CLIENT), entry("lastUpdated", COLUMN_EVENT_LAST_UPDATED), + entry("lastUpdatedAtClient", COLUMN_EVENT_LAST_UPDATED_AT_CLIENT), entry("completedBy", COLUMN_EVENT_COMPLETED_BY), entry("attributeOptionCombo.uid", COLUMN_EVENT_ATTRIBUTE_OPTION_COMBO_UID), entry("completedDate", COLUMN_EVENT_COMPLETED_DATE), @@ -329,10 +333,13 @@ private List fetchEvents(EventQueryParams queryParams, PageParams pagePar event.setScheduledDate(resultSet.getTimestamp(COLUMN_EVENT_SCHEDULED_DATE)); event.setOccurredDate(resultSet.getTimestamp(COLUMN_EVENT_OCCURRED_DATE)); event.setCreated(resultSet.getTimestamp(COLUMN_EVENT_CREATED)); + event.setCreatedAtClient(resultSet.getTimestamp(COLUMN_EVENT_CREATED_AT_CLIENT)); event.setCreatedByUserInfo( EventUtils.jsonToUserInfo( resultSet.getString(COLUMN_EVENT_CREATED_BY), jsonMapper)); event.setLastUpdated(resultSet.getTimestamp(COLUMN_EVENT_LAST_UPDATED)); + event.setLastUpdatedAtClient( + resultSet.getTimestamp(COLUMN_EVENT_LAST_UPDATED_AT_CLIENT)); event.setLastUpdatedByUserInfo( EventUtils.jsonToUserInfo( resultSet.getString(COLUMN_EVENT_LAST_UPDATED_BY), jsonMapper)); @@ -745,10 +752,15 @@ private String getEventSelectQuery( .append(", ") .append("ev.created as ") .append(COLUMN_EVENT_CREATED) + .append(", ") + .append("ev.createdatclient as ") + .append(COLUMN_EVENT_CREATED_AT_CLIENT) .append(", ev.createdbyuserinfo as ") .append(COLUMN_EVENT_CREATED_BY) .append(", ev.lastupdated as ") .append(COLUMN_EVENT_LAST_UPDATED) + .append(", ev.lastupdatedatclient as ") + .append(COLUMN_EVENT_LAST_UPDATED_AT_CLIENT) .append(", ev.lastupdatedbyuserinfo as ") .append(COLUMN_EVENT_LAST_UPDATED_BY) .append(", ") diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/DefaultTrackedEntityService.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/DefaultTrackedEntityService.java index b1b05572139a..f8be15e7aea2 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/DefaultTrackedEntityService.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/DefaultTrackedEntityService.java @@ -576,7 +576,7 @@ private boolean isLocalSearch(TrackedEntityQueryParams params, User user) { } for (OrganisationUnit ou : searchOrgUnits) { - if (!organisationUnitService.isDescendant(ou, localOrgUnits)) { + if (!ou.isDescendant(localOrgUnits)) { return false; } } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/TrackedEntityMapper.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/TrackedEntityMapper.java index cf2e8e0337b8..45e51647bcb7 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/TrackedEntityMapper.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/mappers/TrackedEntityMapper.java @@ -63,5 +63,6 @@ public interface TrackedEntityMapper extends PreheatMapper { @Mapping(target = "name") @Mapping(target = "attributeValues") @Mapping(target = "user") + @Mapping(target = "parent") OrganisationUnit map(OrganisationUnit organisationUnit); } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/supplier/ProgramOrgUnitsSupplier.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/supplier/ProgramOrgUnitsSupplier.java index ee4b26b85a5b..e58cfe53252f 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/supplier/ProgramOrgUnitsSupplier.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/preheat/supplier/ProgramOrgUnitsSupplier.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.tracker.imports.preheat.supplier; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.hisp.dhis.common.IdentifiableObject; import org.hisp.dhis.organisationunit.OrganisationUnit; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/Error.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/Error.java index 4038def54b20..880bbf30ff93 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/Error.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/Error.java @@ -29,7 +29,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Value; import org.hisp.dhis.common.OpenApi; @@ -37,43 +41,24 @@ @Builder @OpenApi.Shared(name = "TrackerImportError") public class Error { - private final String errorMessage; - private final String errorCode; - - private final String trackerType; - - private final String uid; + @Nonnull @JsonProperty String message; + @Nonnull @JsonProperty String errorCode; + @Nonnull @JsonProperty String trackerType; + @Nonnull @JsonProperty String uid; + @EqualsAndHashCode.Exclude @Nonnull @JsonProperty List args; @JsonCreator public Error( - @JsonProperty("message") String errorMessage, - @JsonProperty("errorCode") String errorCode, - @JsonProperty("trackerType") String trackerType, - @JsonProperty("uid") String uid) { - this.errorMessage = errorMessage; + @Nonnull @JsonProperty("message") String message, + @Nonnull @JsonProperty("errorCode") String errorCode, + @Nonnull @JsonProperty("trackerType") String trackerType, + @Nonnull @JsonProperty("uid") String uid, + @CheckForNull @JsonProperty("args") List args) { + this.message = message; this.errorCode = errorCode; this.trackerType = trackerType; this.uid = uid; - } - - @JsonProperty - public String getErrorCode() { - return errorCode; - } - - @JsonProperty - public String getMessage() { - return errorMessage; - } - - @JsonProperty - public String getTrackerType() { - return trackerType; - } - - @JsonProperty - public String getUid() { - return uid; + this.args = args == null ? List.of() : args; } } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/ValidationReport.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/ValidationReport.java index 6e32596d894e..d67f61a2b4ed 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/ValidationReport.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/report/ValidationReport.java @@ -75,10 +75,11 @@ private static List convertToError(List errors) { .map( e -> Error.builder() - .errorMessage(e.getMessage()) + .message(e.getMessage()) .errorCode(e.getCode()) .trackerType(e.getType()) .uid(e.getUid()) + .args(e.getArgs()) .build()) .collect(Collectors.toList()); } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Error.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Error.java index 58c2694653bf..a9857f1a9e4c 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Error.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Error.java @@ -27,18 +27,18 @@ */ package org.hisp.dhis.tracker.imports.validation; +import java.util.List; import lombok.Value; import org.hisp.dhis.tracker.TrackerType; @Value public class Error implements Validation { - String message; + String message; ValidationCode code; - TrackerType type; - String uid; + List args; public ValidationCode getErrorCode() { return code; @@ -67,4 +67,9 @@ public String getType() { public String getUid() { return uid; } + + @Override + public List getArgs() { + return args.stream().map(obj -> obj == null ? null : obj.toString()).toList(); + } } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/PersistablesFilter.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/PersistablesFilter.java index e33bb5855277..867054cc842e 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/PersistablesFilter.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/PersistablesFilter.java @@ -321,15 +321,15 @@ private void addErrorsForChildren( } private static Error error(ValidationCode code, TrackerDto notPersistable, TrackerDto reason) { - String message = - MessageFormat.format( - code.getMessage(), - notPersistable.getTrackerType().getName(), - notPersistable.getUid(), - reason.getTrackerType().getName(), - reason.getUid()); - - return new Error(message, code, notPersistable.getTrackerType(), notPersistable.getUid()); + Object[] args = { + notPersistable.getTrackerType().getName(), + notPersistable.getUid(), + reason.getTrackerType().getName(), + reason.getUid() + }; + String message = MessageFormat.format(code.getMessage(), args); + return new Error( + message, code, notPersistable.getTrackerType(), notPersistable.getUid(), List.of(args)); } /** diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Reporter.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Reporter.java index 49648c5c2379..73a38ca10724 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Reporter.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Reporter.java @@ -28,6 +28,7 @@ package org.hisp.dhis.tracker.imports.validation; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumMap; import java.util.HashSet; import java.util.List; @@ -158,7 +159,8 @@ public boolean addError(TrackerDto dto, ValidationCode code, Object... args) { MessageFormatter.format(idSchemes, code.getMessage(), args), code, dto.getTrackerType(), - dto.getUid())); + dto.getUid(), + args == null ? List.of() : Arrays.asList(args))); return true; } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Validation.java b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Validation.java index f0a4a51c5de2..229de7adc478 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Validation.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/validation/Validation.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.tracker.imports.validation; +import java.util.List; + /** * Validation represents an issue found by the validation process. It contains information that help * the client to understand and fix the problem. @@ -39,4 +41,8 @@ public interface Validation { String getType(); String getUid(); + + default List getArgs() { + return List.of(); + } } diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/SetMandatoryFieldExecutorTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/SetMandatoryFieldExecutorTest.java index 78f3eae603f1..f232a71a6230 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/SetMandatoryFieldExecutorTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/SetMandatoryFieldExecutorTest.java @@ -28,7 +28,7 @@ package org.hisp.dhis.tracker.imports.programrule.executor.enrollment; import static org.hisp.dhis.tracker.imports.programrule.ProgramRuleIssue.error; -import static org.hisp.dhis.tracker.imports.validation.ValidationCode.*; +import static org.hisp.dhis.tracker.imports.validation.ValidationCode.E1306; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/ValidationExecutorTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/ValidationExecutorTest.java index 5b8ff344ee11..5918b900250f 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/ValidationExecutorTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/programrule/executor/enrollment/ValidationExecutorTest.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.tracker.imports.programrule.executor.enrollment; -import static org.hisp.dhis.tracker.imports.domain.EnrollmentStatus.*; +import static org.hisp.dhis.tracker.imports.domain.EnrollmentStatus.ACTIVE; +import static org.hisp.dhis.tracker.imports.domain.EnrollmentStatus.COMPLETED; import static org.hisp.dhis.tracker.imports.programrule.IssueType.ERROR; import static org.hisp.dhis.tracker.imports.programrule.IssueType.WARNING; import static org.hisp.dhis.tracker.imports.programrule.ProgramRuleIssue.error; diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/report/TrackerBundleImportReportTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/report/TrackerBundleImportReportTest.java index 0ecabc5ed491..d2f5d2761e5c 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/report/TrackerBundleImportReportTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/report/TrackerBundleImportReportTest.java @@ -71,7 +71,7 @@ class TrackerBundleImportReportTest { @InjectMocks private DefaultTrackerImportService trackerImportService; - private ObjectMapper jsonMapper = JacksonObjectMapperConfig.staticJsonMapper(); + private final ObjectMapper jsonMapper = JacksonObjectMapperConfig.staticJsonMapper(); @Test void testImportReportErrors() { @@ -135,7 +135,8 @@ void testSerializingAndDeserializingImportReport() throws JsonProcessingExceptio "Could not find OrganisationUnit: ``, linked to Tracked Entity.", ValidationCode.E1049.name(), TRACKED_ENTITY.name(), - "BltTZV9HvEZ"))); + "BltTZV9HvEZ", + List.of("BltTZV9HvEZ")))); tvr.addWarnings( List.of( new Warning( @@ -198,8 +199,8 @@ void testSerializingAndDeserializingImportReport() throws JsonProcessingExceptio deserializedReportTrackerTypeReport.getStats()); // Verify Validation Report - Error Reports assertEquals( - toSerializeReport.getValidationReport().getErrors().get(0).getErrorMessage(), - deserializedReport.getValidationReport().getErrors().get(0).getErrorMessage()); + toSerializeReport.getValidationReport().getErrors().get(0).getMessage(), + deserializedReport.getValidationReport().getErrors().get(0).getMessage()); assertEquals( toSerializeReport.getValidationReport().getErrors().get(0).getErrorCode(), deserializedReport.getValidationReport().getErrors().get(0).getErrorCode()); @@ -245,7 +246,8 @@ private ValidationReport createValidationReport() { "Could not find OrganisationUnit: ``, linked to Tracked Entity.", ValidationCode.E1049.name(), TRACKED_ENTITY.name(), - "BltTZV9HvEZ")), + "BltTZV9HvEZ", + List.of("BltTZV9HvEZ"))), List.of( new Warning( "ProgramStage `l8oDIfJJhtg` does not allow user assignment", diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ReporterTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ReporterTest.java index b6cb4acb7b99..f10e22052b98 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ReporterTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ReporterTest.java @@ -30,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.List; import org.hisp.dhis.tracker.TrackerType; import org.hisp.dhis.tracker.imports.TrackerIdSchemeParams; import org.junit.jupiter.api.Test; @@ -75,7 +76,8 @@ void hasWarningReportNotFound() { } private Error eventError() { - return new Error("some error", ValidationCode.E1000, TrackerType.EVENT, "JgDfHAGzzfS"); + return new Error( + "some error", ValidationCode.E1000, TrackerType.EVENT, "JgDfHAGzzfS", List.of()); } private Warning eventWarning() { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ValidationResultTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ValidationResultTest.java index e714a0ee5c4e..ffddf319a64f 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ValidationResultTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/ValidationResultTest.java @@ -30,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.List; import java.util.Set; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.tracker.TrackerType; @@ -77,7 +78,7 @@ private Error newError(ValidationCode code) { } private Error newError(String uid, ValidationCode code) { - return new Error("", code, TrackerType.EVENT, uid); + return new Error("", code, TrackerType.EVENT, uid, List.of()); } private Warning newWarning() { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/AllTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/AllTest.java index b463fac888c3..b70f34ee3be4 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/AllTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/AllTest.java @@ -148,7 +148,8 @@ public boolean needsToRun(TrackerImportStrategy strategy) { * tracker type, uid or error code. */ private static void addError(Reporter reporter, String message) { - reporter.addError(new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid")); + reporter.addError( + new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid", List.of())); } private List actualErrorMessages() { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/EachTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/EachTest.java index 3930ca6298ee..70e746965bdf 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/EachTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/EachTest.java @@ -151,7 +151,8 @@ private static Enrollment enrollment(String uid, String... notes) { * tracker type, uid or error code. */ private static void addError(Reporter reporter, String message) { - reporter.addError(new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid")); + reporter.addError( + new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid", List.of())); } private List actualErrorMessages() { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/FieldTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/FieldTest.java index e2dc21d45980..64ce221f2287 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/FieldTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/FieldTest.java @@ -74,7 +74,7 @@ void testFieldWithValidator() { Validator isValidUid = (r, b, uid) -> { // to demonstrate that we are getting the trackedEntity field - r.addError(new Error(uid, E1000, ENROLLMENT, uid)); + r.addError(new Error(uid, E1000, ENROLLMENT, uid, List.of(uid))); }; Validator validator = field(Enrollment::getTrackedEntity, isValidUid); @@ -172,7 +172,8 @@ void testFieldWithPredicateSucceeding() { * tracker type, uid or error code. */ private static void addError(Reporter reporter, String message) { - reporter.addError(new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid")); + reporter.addError( + new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid", List.of())); } private List actualErrorMessages() { diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/SeqTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/SeqTest.java index 207b09c860f1..21e3a14955ed 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/SeqTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/SeqTest.java @@ -243,7 +243,8 @@ public void validate(Reporter reporter, TrackerBundle bundle, String input) { * tracker type, uid or error code. */ private static void addError(Reporter reporter, String message) { - reporter.addError(new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid")); + reporter.addError( + new Error(message, ValidationCode.E9999, TrackerType.TRACKED_ENTITY, "uid", List.of())); } /** diff --git a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/RepeatedEventsValidatorTest.java b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/RepeatedEventsValidatorTest.java index 338d55164001..4d946674c41a 100644 --- a/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/RepeatedEventsValidatorTest.java +++ b/dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/imports/validation/validator/event/RepeatedEventsValidatorTest.java @@ -191,7 +191,8 @@ void testTwoEventsInNotRepeatableProgramStageWhenOneIsInvalidArePassingValidatio List events = Lists.newArrayList(invalidEvent, notRepeatableEvent("B")); bundle.setEvents(events); events.forEach(e -> bundle.setStrategy(e, TrackerImportStrategy.CREATE_AND_UPDATE)); - reporter.addError(new Error("", E9999, invalidEvent.getTrackerType(), invalidEvent.getUid())); + reporter.addError( + new Error("", E9999, invalidEvent.getTrackerType(), invalidEvent.getUid(), List.of())); validator.validate(reporter, bundle, bundle.getEvents()); diff --git a/dhis-2/dhis-services/dhis-service-validation/pom.xml b/dhis-2/dhis-services/dhis-service-validation/pom.xml index 5e721f1085d7..9bad82dfb298 100644 --- a/dhis-2/dhis-services/dhis-service-validation/pom.xml +++ b/dhis-2/dhis-services/dhis-service-validation/pom.xml @@ -34,6 +34,10 @@ org.hisp.dhis dhis-service-schema + + org.hisp.dhis + dhis-service-setting + org.hisp.dhis dhis-support-commons @@ -128,5 +132,10 @@ mockito-junit-jupiter test + + org.junit.jupiter + junit-jupiter-params + test + diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutlierDetectionService.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutlierDetectionService.java deleted file mode 100644 index e2712072cb52..000000000000 --- a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutlierDetectionService.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.outlierdetection; - -import java.io.IOException; -import java.io.OutputStream; -import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.feedback.ErrorMessage; - -/** - * Outlier detection service. - * - * @author Lars Helge Overland - */ -public interface OutlierDetectionService { - /** - * Validates the given request. - * - * @param request the {@link OutlierDetectionRequest}. - * @throws IllegalQueryException if request is invalid. - */ - void validate(OutlierDetectionRequest request) throws IllegalQueryException; - - /** - * Validates the given request. - * - * @param request the {@link OutlierDetectionRequest}. - * @return an {@link ErrorMessage} if request is invalid, or null if valid. - */ - ErrorMessage validateForErrorMessage(OutlierDetectionRequest request); - - /** - * Creates a {@link OutlierDetectionRequest} from the given query. - * - * @param query the {@link OutlierDetectionQuery}. - * @return a {@link OutlierDetectionRequest}. - */ - OutlierDetectionRequest getFromQuery(OutlierDetectionQuery query); - - /** - * Returns outlier data values for the given request. - * - * @param request the {@link OutlierDetectionRequest}. - * @return a {@link OutlierDetectionResponse}. - * @throws IllegalQueryException if request is invalid. - */ - OutlierDetectionResponse getOutlierValues(OutlierDetectionRequest request) - throws IllegalQueryException; - - /** - * Writes outlier data values for the given request as CSV to the given output stream. - * - * @param request the {@link OutlierDetectionRequest}. - * @param out the {@link OutputStream} to write to. - * @throws IllegalQueryException if request is invalid. - */ - void getOutlierValuesAsCsv(OutlierDetectionRequest request, OutputStream out) - throws IllegalQueryException, IOException; -} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutliersSqlParamName.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutliersSqlParamName.java new file mode 100644 index 000000000000..b0e484f7ba87 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/OutliersSqlParamName.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection; + +/** Enum for named params of parametrized sql query */ +public enum OutliersSqlParamName { + // ZScore (modified ZScore) factor. + // For example the threshold=3 means all data lying outside 3 sigma (3 * standard deviation) + // are considered as the outliers + THRESHOLD("threshold"), + DATA_ELEMENT_IDS("data_element_ids"), + START_DATE("start_date"), + END_DATE("end_date"), + // start date criteria of statistic data collection (the stats will be based on data starting on + // this date) + DATA_START_DATE("data_start_date"), + // start date criteria of statistic data collection (the stats will be based on data ending on + // this date) + DATA_END_DATE("data_end_date"), + MAX_RESULTS("max_results"); + private final String key; + + OutliersSqlParamName(String key) { + this.key = key; + } + + public String getKey() { + return key; + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/parser/OutlierDetectionQueryParser.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/parser/OutlierDetectionQueryParser.java new file mode 100644 index 000000000000..f82677ef23c7 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/parser/OutlierDetectionQueryParser.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.parser; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.springframework.stereotype.Component; + +/** Parse and transform the incoming query params into the OutlierDetectionRequest. */ +@Component +@AllArgsConstructor +public class OutlierDetectionQueryParser { + + private final IdentifiableObjectManager idObjectManager; + + /** + * Creates a {@link OutlierDetectionRequest} from the given query. + * + * @param query the {@link OutlierDetectionQuery}. + * @return a {@link OutlierDetectionRequest}. + */ + public OutlierDetectionRequest getFromQuery(OutlierDetectionQuery query) { + OutlierDetectionRequest.Builder request = new OutlierDetectionRequest.Builder(); + + List dataSets = idObjectManager.getByUid(DataSet.class, query.getDs()); + + // Re-fetch data elements to maintain access control + + List de = + dataSets.stream() + .map(DataSet::getDataElements) + .flatMap(Collection::stream) + .filter(d -> d.getValueType().isNumeric()) + .map(DataElement::getUid) + .collect(Collectors.toList()); + + de.addAll(query.getDe()); + + List dataElements = idObjectManager.getByUid(DataElement.class, de); + List orgUnits = + idObjectManager.getByUid(OrganisationUnit.class, query.getOu()); + + request + .withDataElements(dataElements) + .withStartEndDate(query.getStartDate(), query.getEndDate()) + .withOrgUnits(orgUnits) + .withDataStartDate(query.getDataStartDate()) + .withDataEndDate(query.getDataEndDate()); + + if (query.getAlgorithm() != null) { + request.withAlgorithm(query.getAlgorithm()); + } + + if (query.getThreshold() != null) { + request.withThreshold(query.getThreshold()); + } + + if (query.getOrderBy() != null) { + request.withOrderBy(query.getOrderBy()); + } + + if (query.getMaxResults() != null) { + request.withMaxResults(query.getMaxResults()); + } + + return request.build(); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZScoreSqlStatementProcessor.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZScoreSqlStatementProcessor.java new file mode 100644 index 000000000000..d035a9623eeb --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZScoreSqlStatementProcessor.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.DATA_ELEMENT_IDS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.END_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.MAX_RESULTS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.START_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.THRESHOLD; + +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.outlierdetection.Order; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.OutliersSqlParamName; +import org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; +import org.springframework.stereotype.Component; + +/** + * The class is related to outlier data detection based on z-score and modified z-score. The + * analytics tables are used for it. + * + *

This both implements the {@link OutlierDetectionAlgorithm#Z_SCORE} and {@link + * OutlierDetectionAlgorithm#MOD_Z_SCORE}. Usual z-score uses the mean as middle value whereas the + * modified z-score uses the median as middle value or more mathematically correct as the + * measure of central tendency. + */ +@Component +public class AnalyticsZScoreSqlStatementProcessor implements OutlierSqlStatementProcessor { + + /** + * The function retries the sql statement for inspection of outliers. Following scores are in use: + * + *

Z-Score abs(xi – μ) / σ where: xi: A single data value μ: The mean of the dataset σ: The + * standard deviation of the dataset + * + *

Modified z-score = 0.6745 * abs(xi – x̃) / MAD where: xi: A single data value x̃: The median + * of the dataset MAD: The median absolute deviation of the dataset 0.6745: conversion factor + * (0.75 percentiles) * + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return sql statement for the outlier detection and related data + */ + @Override + public String getSqlStatement(OutlierDetectionRequest request) { + if (request == null) { + return StringUtils.EMPTY; + } + + String ouPathClause = OutlierDetectionUtils.getOrgUnitPathClause(request.getOrgUnits(), "ax"); + + boolean modifiedZ = request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; + + String middleValue = modifiedZ ? " ax.percentile_middle_value" : " ax.avg_middle_value"; + + String order = + request.getOrderBy() == Order.MEAN_ABS_DEV + ? "middle_value_abs_dev" + : request.getOrderBy().getKey(); + String thresholdParam = OutliersSqlParamName.THRESHOLD.getKey(); + + String sql = + "select * from (select " + + "ax.dataelementid, " + + "ax.de_uid, " + + "ax.ou_uid, " + + "ax.coc_uid, " + + "ax.aoc_uid, " + + "ax.de_name, " + + "ax.ou_name, " + + "ax.coc_name, " + + "ax.aoc_name, " + + "ax.value, " + + "ax.pestartdate as pe_start_date, " + + "ax.pt_name, " + + middleValue + + " as middle_value, " + + "ax.std_dev as std_dev, " + + "ax.mad as mad, " + + "abs(ax.value::double precision - " + + middleValue + + ") as middle_value_abs_dev, "; + if (modifiedZ) { + sql += + "(case when ax.mad = 0 then 0 " + + " else 0.6745 * abs(ax.value::double precision - " + + middleValue + + " ) / ax.mad " + + " end) as z_score, "; + } else { + sql += + "(case when ax.std_dev = 0 then 0 " + + " else abs(ax.value::double precision - " + + middleValue + + " ) / ax.std_dev " + + " end) as z_score, "; + } + sql += + middleValue + + " - (ax.std_dev * :" + + thresholdParam + + ") as lower_bound, " + + middleValue + + " + (ax.std_dev * :" + + thresholdParam + + ") as upper_bound " + + "from analytics ax " + + "where dataelementid in (:" + + DATA_ELEMENT_IDS.getKey() + + ") " + + "and " + + ouPathClause + + " " + + "and ax.pestartdate >= :" + + START_DATE.getKey() + + " " + + "and ax.peenddate <= :" + + END_DATE.getKey() + + ") t1 " + + "where t1.z_score > :" + + thresholdParam + + " " + + "order by " + + order + + " desc " + + "limit :" + + MAX_RESULTS.getKey() + + " "; + + return sql; + } + + /** + * To avoid the sql injection and decrease the load of the database engine (query plan caching) + * the named params are in use. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return named params for parametrized sql query + */ + @Override + public SqlParameterSource getSqlParameterSource(OutlierDetectionRequest request) { + return new MapSqlParameterSource() + .addValue(THRESHOLD.getKey(), request.getThreshold()) + .addValue(DATA_ELEMENT_IDS.getKey(), request.getDataElementIds()) + .addValue(START_DATE.getKey(), request.getStartDate()) + .addValue(END_DATE.getKey(), request.getEndDate()) + .addValue(MAX_RESULTS.getKey(), request.getMaxResults()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessor.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessor.java new file mode 100644 index 000000000000..c5c641ee0d6b --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessor.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.DATA_ELEMENT_IDS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.END_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.MAX_RESULTS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.START_DATE; + +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; +import org.springframework.stereotype.Component; + +@Component +public class MinMaxSqlStatementProcessor implements OutlierSqlStatementProcessor { + + /** + * The function retries the sql statement for inspection of outliers. Min and max values are + * inspected. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return sql statement for the outlier detection and related data + */ + @Override + public String getSqlStatement(OutlierDetectionRequest request) { + if (request == null) { + return StringUtils.EMPTY; + } + + String ouPathClause = OutlierDetectionUtils.getOrgUnitPathClause(request.getOrgUnits(), "ou"); + + return "select de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, " + + "de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, " + + "pe.startdate as pe_start_date, pt.name as pt_name, " + + "dv.value::double precision as value, dv.followup as follow_up, " + + "least(abs(dv.value::double precision - mm.minimumvalue), " + + "abs(dv.value::double precision - mm.maximumvalue)) as bound_abs_dev, " + + "mm.minimumvalue as lower_bound, " + + "mm.maximumvalue as upper_bound " + + "from datavalue dv " + + "inner join dataelement de on dv.dataelementid = de.dataelementid " + + "inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid " + + "inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid " + + "inner join period pe on dv.periodid = pe.periodid " + + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " + + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " + + + // Min-max value join + "inner join minmaxdataelement mm on (dv.dataelementid = mm.dataelementid " + + "and dv.sourceid = mm.sourceid and dv.categoryoptioncomboid = mm.categoryoptioncomboid) " + + "where dv.dataelementid in (:data_element_ids) " + + "and pe.startdate >= :start_date " + + "and pe.enddate <= :end_date " + + "and " + + ouPathClause + + " " + + "and dv.deleted is false " + + + // Filter for values outside the min-max range + "and (dv.value::double precision < mm.minimumvalue or dv.value::double precision > mm.maximumvalue) " + + + // Order and limit + "order by bound_abs_dev desc " + + "limit :max_results;"; + } + + /** + * To avoid the sql injection and decrease the load of the database engine (query plan caching) + * the named params are in use. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return named params for parametrized sql query + */ + @Override + public SqlParameterSource getSqlParameterSource(OutlierDetectionRequest request) { + return new MapSqlParameterSource() + .addValue(DATA_ELEMENT_IDS.getKey(), request.getDataElementIds()) + .addValue(START_DATE.getKey(), request.getStartDate()) + .addValue(END_DATE.getKey(), request.getEndDate()) + .addValue(MAX_RESULTS.getKey(), request.getMaxResults()); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/OutlierSqlStatementProcessor.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/OutlierSqlStatementProcessor.java new file mode 100644 index 000000000000..966c7bf2eee8 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/OutlierSqlStatementProcessor.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; + +public interface OutlierSqlStatementProcessor { + + /** + * Creates a parametrised SQL statement for outliers. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return SQL statement as a string. + */ + String getSqlStatement(OutlierDetectionRequest request); + + /** + * Retrieve SQL parameters for outliers SQL statement + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return teh instance of {@link SqlParameterSource}. + */ + SqlParameterSource getSqlParameterSource(OutlierDetectionRequest request); +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessor.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessor.java new file mode 100644 index 000000000000..5890d5a8421e --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessor.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.DATA_ELEMENT_IDS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.DATA_END_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.DATA_START_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.END_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.MAX_RESULTS; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.START_DATE; +import static org.hisp.dhis.outlierdetection.OutliersSqlParamName.THRESHOLD; + +import java.util.Date; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.outlierdetection.Order; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; +import org.springframework.stereotype.Component; + +@Component +public class ZscoreSqlStatementProcessor implements OutlierSqlStatementProcessor { + + /** + * The function retries the sql statement for inspection of outliers. Z-Score and modified Z-Score + * is in use. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return sql statement for the outlier detection and related data + */ + @Override + public String getSqlStatement(OutlierDetectionRequest request) { + if (request == null) { + return StringUtils.EMPTY; + } + + String ouPathClause = OutlierDetectionUtils.getOrgUnitPathClause(request.getOrgUnits(), "ou"); + String dataStartDateClause = getDataStartDateClause(request.getDataStartDate()); + String dataEndDateClause = getDataEndDateClause(request.getDataEndDate()); + + boolean modifiedZ = request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; + String middleStatsCalc = + modifiedZ + ? "percentile_cont(0.5) within group(order by dv.value::double precision)" + : "avg(dv.value::double precision)"; + + String order = + request.getOrderBy() == Order.MEAN_ABS_DEV + ? "middle_value_abs_dev" + : request.getOrderBy().getKey(); + + String thresholdParam = THRESHOLD.getKey(); + + return "select dvs.de_uid, dvs.ou_uid, dvs.coc_uid, dvs.aoc_uid, " + + "dvs.de_name, dvs.ou_name, dvs.coc_name, dvs.aoc_name, dvs.value, dvs.follow_up, " + + "dvs.pe_start_date, dvs.pt_name, " + + "stats.middle_value as middle_value, " + + "stats.std_dev as std_dev, " + + "abs(dvs.value::double precision - stats.middle_value) as middle_value_abs_dev, " + + "abs(dvs.value::double precision - stats.middle_value) / stats.std_dev as z_score, " + + "stats.middle_value - (stats.std_dev * :" + + thresholdParam + + ") as lower_bound, " + + "stats.middle_value + (stats.std_dev * :" + + thresholdParam + + ") as upper_bound " + + + // Data value query + "from (" + + "select dv.dataelementid, dv.sourceid, dv.periodid, " + + "dv.categoryoptioncomboid, dv.attributeoptioncomboid, " + + "de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, " + + "de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, " + + "pe.startdate as pe_start_date, pt.name as pt_name, " + + "dv.value as value, dv.followup as follow_up " + + "from datavalue dv " + + "inner join dataelement de on dv.dataelementid = de.dataelementid " + + "inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid " + + "inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid " + + "inner join period pe on dv.periodid = pe.periodid " + + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " + + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " + + "where dv.dataelementid in (:" + + DATA_ELEMENT_IDS.getKey() + + ") " + + "and pe.startdate >= :" + + START_DATE.getKey() + + " " + + "and pe.enddate <= :" + + END_DATE.getKey() + + " " + + "and " + + ouPathClause + + " " + + "and dv.deleted is false" + + ") as dvs " + + + // Mean or Median and std dev mapping query + "inner join (" + + "select dv.dataelementid as dataelementid, dv.sourceid as sourceid, " + + "dv.categoryoptioncomboid as categoryoptioncomboid, " + + "dv.attributeoptioncomboid as attributeoptioncomboid, " + + middleStatsCalc + + " as middle_value, " + + "stddev_pop(dv.value::double precision) as std_dev " + + "from datavalue dv " + + "inner join period pe on dv.periodid = pe.periodid " + + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " + + "where dv.dataelementid in (:" + + DATA_ELEMENT_IDS.getKey() + + ") " + + dataStartDateClause + + dataEndDateClause + + "and " + + ouPathClause + + " " + + "and dv.deleted is false " + + "group by dv.dataelementid, dv.sourceid, dv.categoryoptioncomboid, dv.attributeoptioncomboid" + + ") as stats " + + + // Query join + "on dvs.dataelementid = stats.dataelementid " + + "and dvs.sourceid = stats.sourceid " + + "and dvs.categoryoptioncomboid = stats.categoryoptioncomboid " + + "and dvs.attributeoptioncomboid = stats.attributeoptioncomboid " + + "where stats.std_dev != 0.0 " + + + // Filter on z-score threshold + "and (abs(dvs.value::double precision - stats.middle_value) / stats.std_dev) >= :" + + thresholdParam + + " " + + + // Order and limit + "order by " + + order + + " desc " + + "limit :" + + MAX_RESULTS.getKey() + + ";"; + } + + /** + * To avoid the sql injection and decrease the load of the database engine (query plan caching) + * the named params are in use. + * + * @param request the instance of {@link OutlierDetectionRequest}. + * @return named params for parametrized sql query + */ + @Override + public SqlParameterSource getSqlParameterSource(OutlierDetectionRequest request) { + return new MapSqlParameterSource() + .addValue(THRESHOLD.getKey(), request.getThreshold()) + .addValue(DATA_ELEMENT_IDS.getKey(), request.getDataElementIds()) + .addValue(START_DATE.getKey(), request.getStartDate()) + .addValue(END_DATE.getKey(), request.getEndDate()) + .addValue(DATA_START_DATE.getKey(), request.getDataStartDate()) + .addValue(DATA_END_DATE.getKey(), request.getDataEndDate()) + .addValue(MAX_RESULTS.getKey(), request.getMaxResults()); + } + + private String getDataStartDateClause(Date dataStartDate) { + return dataStartDate != null + ? "and pe.startdate >= :" + DATA_START_DATE.getKey() + " " + : StringUtils.EMPTY; + } + + private String getDataEndDateClause(Date dataStartDate) { + return dataStartDate != null + ? "and pe.enddate <= :" + DATA_END_DATE.getKey() + " " + : StringUtils.EMPTY; + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AbstractOutlierDetectionManager.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AbstractOutlierDetectionManager.java new file mode 100644 index 000000000000..0e98ebe14893 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AbstractOutlierDetectionManager.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.service; + +import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.withExceptionHandling; +import static org.hisp.dhis.period.PeriodType.getIsoPeriod; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.calendar.Calendar; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.outlierdetection.processor.OutlierSqlStatementProcessor; +import org.hisp.dhis.period.PeriodType; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; + +/** + * Manager for database queries related to outlier data detection (Z-Score, modified Z-Score, + * Min-Max values). + */ +@Slf4j +public abstract class AbstractOutlierDetectionManager { + private final NamedParameterJdbcTemplate jdbcTemplate; + private final OutlierSqlStatementProcessor sqlStatementProcessor; + + protected AbstractOutlierDetectionManager( + NamedParameterJdbcTemplate jdbcTemplate, OutlierSqlStatementProcessor sqlStatementProcessor) { + this.jdbcTemplate = jdbcTemplate; + this.sqlStatementProcessor = sqlStatementProcessor; + } + + /** + * Retrieves all outliers. + * + * @param request the {@link OutlierDetectionRequest}. + * @return list of the OutlierValue instances for api response + */ + public List getOutlierValues(OutlierDetectionRequest request) { + String sql = sqlStatementProcessor.getSqlStatement(request); + SqlParameterSource params = sqlStatementProcessor.getSqlParameterSource(request); + Calendar calendar = PeriodType.getCalendar(); + boolean modifiedZ = request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; + + return withExceptionHandling( + () -> jdbcTemplate.query(sql, params, getRowMapper(calendar, modifiedZ))) + .orElse(null); + } + + /** + * Returns a {@link RowMapper} for {@link OutlierValue}. + * + * @param calendar the {@link Calendar}. + * @return a {@link RowMapper}. + */ + protected abstract RowMapper getRowMapper( + final Calendar calendar, boolean modifiedZ); + + /** + * Maps incoming database set into the api response element. + * + * @param calendar the {@link Calendar}. + * @param rs the {@link ResultSet}. + * @return single OutlierValue instance + * @throws SQLException + */ + protected OutlierValue getOutlierValue(Calendar calendar, ResultSet rs) throws SQLException { + OutlierValue outlier = new OutlierValue(); + + String isoPeriod = getIsoPeriod(calendar, rs.getString("pt_name"), rs.getDate("pe_start_date")); + + outlier.setDe(rs.getString("de_uid")); + outlier.setDeName(rs.getString("de_name")); + outlier.setPe(isoPeriod); + outlier.setOu(rs.getString("ou_uid")); + outlier.setOuName(rs.getString("ou_name")); + outlier.setCoc(rs.getString("coc_uid")); + outlier.setCocName(rs.getString("coc_name")); + outlier.setAoc(rs.getString("aoc_uid")); + outlier.setAocName(rs.getString("aoc_name")); + outlier.setValue(rs.getDouble("value")); + + return outlier; + } + + /** + * The values for outlier identification are added to OutlierValue instance. + * + * @param outlierValue the {@link OutlierValue} + * @param rs the {@link ResultSet} + * @param modifiedZ boolean flag (false means z-score to be applied) + * @throws SQLException + */ + protected void addZScoreBasedParamsToOutlierValue( + OutlierValue outlierValue, ResultSet rs, boolean modifiedZ) throws SQLException { + if (modifiedZ) { + outlierValue.setMedian(rs.getDouble("middle_value")); + } else { + outlierValue.setMean(rs.getDouble("middle_value")); + } + + outlierValue.setAbsDev(rs.getDouble("middle_value_abs_dev")); + outlierValue.setZScore(outlierValue.getStdDev() == 0 ? Double.NaN : rs.getDouble("z_score")); + outlierValue.setLowerBound(rs.getDouble("lower_bound")); + outlierValue.setUpperBound(rs.getDouble("upper_bound")); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsOutlierDetectionService.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsOutlierDetectionService.java new file mode 100644 index 000000000000..4c573f97168a --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsOutlierDetectionService.java @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.service; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.common.Grid; +import org.hisp.dhis.common.GridHeader; +import org.hisp.dhis.common.IllegalQueryException; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; +import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.system.grid.GridUtils; +import org.hisp.dhis.system.grid.ListGrid; +import org.springframework.stereotype.Service; + +@Slf4j +@AllArgsConstructor +@Service +public class AnalyticsOutlierDetectionService { + + private final AnalyticsZScoreOutlierDetectionManager zScoreOutlierDetection; + + /** + * Transform the incoming request into api response (json). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public Grid getOutlierValues(OutlierDetectionRequest request) throws IllegalQueryException { + List outlierValues = zScoreOutlierDetection.getOutlierValues(request); + + Grid grid = new ListGrid(); + setHeaders(grid, request); + setMetaData(grid, request, outlierValues); + setRows(grid, outlierValues, request); + + return grid; + } + + /** + * Transform the incoming request into api response (csv download). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsCsv(OutlierDetectionRequest request, Writer writer) + throws IllegalQueryException, IOException { + GridUtils.toCsv(getOutlierValues(request), writer); + } + + /** + * Transform the incoming request into api response (xml). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsXml(OutlierDetectionRequest request, OutputStream outputStream) + throws IllegalQueryException { + GridUtils.toXml(getOutlierValues(request), outputStream); + } + + /** + * Transform the incoming request into api response (xls download). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsXls(OutlierDetectionRequest request, OutputStream outputStream) + throws IllegalQueryException, IOException { + GridUtils.toXls(getOutlierValues(request), outputStream); + } + + /** + * Transform the incoming request into api response (html). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsHtml(OutlierDetectionRequest request, Writer writer) + throws IllegalQueryException { + GridUtils.toHtml(getOutlierValues(request), writer); + } + + /** + * Transform the incoming request into api response (html with css). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsHtmlCss(OutlierDetectionRequest request, Writer writer) + throws IllegalQueryException { + GridUtils.toHtmlCss(getOutlierValues(request), writer); + } + + private void setHeaders(Grid grid, OutlierDetectionRequest request) { + boolean isModifiedZScore = request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; + + grid.addHeader(new GridHeader("data element", ValueType.TEXT)); + grid.addHeader(new GridHeader("data element name", ValueType.TEXT)); + grid.addHeader(new GridHeader("period", ValueType.TEXT)); + grid.addHeader(new GridHeader("organisation unit", ValueType.TEXT)); + grid.addHeader(new GridHeader("organisation unit name", ValueType.TEXT)); + grid.addHeader(new GridHeader("category option", ValueType.TEXT)); + grid.addHeader(new GridHeader("category option name", ValueType.TEXT)); + grid.addHeader(new GridHeader("attribute option", ValueType.TEXT)); + grid.addHeader(new GridHeader("attribute option name", ValueType.TEXT)); + grid.addHeader(new GridHeader("value", ValueType.NUMBER)); + grid.addHeader(new GridHeader(isModifiedZScore ? "median" : "mean", ValueType.NUMBER)); + grid.addHeader( + new GridHeader( + isModifiedZScore ? "median absolute deviation" : "stdDev", ValueType.NUMBER)); + grid.addHeader(new GridHeader("absDev", ValueType.NUMBER)); + grid.addHeader( + new GridHeader(isModifiedZScore ? "modified zScore" : "zScore", ValueType.NUMBER)); + grid.addHeader(new GridHeader("lowerBound", ValueType.NUMBER)); + grid.addHeader(new GridHeader("upperBound", ValueType.NUMBER)); + } + + private void setMetaData( + Grid grid, OutlierDetectionRequest request, List outlierValues) { + grid.addMetaData("algorithm", request.getAlgorithm()); + grid.addMetaData("threshold", request.getThreshold()); + grid.addMetaData("orderBy", request.getOrderBy().getKey()); + grid.addMetaData("maxResults", request.getMaxResults()); + grid.addMetaData("count", outlierValues.size()); + } + + private void setRows( + Grid grid, List outlierValues, OutlierDetectionRequest request) { + outlierValues.forEach( + v -> { + boolean isModifiedZScore = + request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; + grid.addRow(); + grid.addValue(v.getDe()); + grid.addValue(v.getDeName()); + grid.addValue(v.getPe()); + grid.addValue(v.getOu()); + grid.addValue(v.getOuName()); + grid.addValue(v.getCoc()); + grid.addValue(v.getCocName()); + grid.addValue(v.getAoc()); + grid.addValue(v.getAocName()); + grid.addValue(v.getValue()); + grid.addValue(isModifiedZScore ? v.getMedian() : v.getMean()); + grid.addValue(v.getStdDev()); + grid.addValue(v.getAbsDev()); + grid.addValue(v.getZScore()); + grid.addValue(v.getLowerBound()); + grid.addValue(v.getUpperBound()); + }); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsZScoreOutlierDetectionManager.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsZScoreOutlierDetectionManager.java new file mode 100644 index 000000000000..d7995caf8074 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/AnalyticsZScoreOutlierDetectionManager.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.service; + +import java.sql.ResultSet; +import java.sql.SQLException; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.calendar.Calendar; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.outlierdetection.processor.OutlierSqlStatementProcessor; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +/** + * Manager for database queries related to outlier data detection based on z-score and modified + * z-score. + * + *

This both implements the {@link OutlierDetectionAlgorithm#Z_SCORE} and {@link + * OutlierDetectionAlgorithm#MOD_Z_SCORE}. Usual z-score uses the mean as middle value whereas the + * modified z-score uses the median as middle value or more mathematically correct as the + * measure of central tendency. + */ +@Slf4j +@Repository +public class AnalyticsZScoreOutlierDetectionManager extends AbstractOutlierDetectionManager { + protected AnalyticsZScoreOutlierDetectionManager( + NamedParameterJdbcTemplate jdbcTemplate, + @Qualifier("analyticsZScoreSqlStatementProcessor") + OutlierSqlStatementProcessor sqlStatementProcessor) { + super(jdbcTemplate, sqlStatementProcessor); + } + + /** {@inheritDoc} */ + @Override + protected RowMapper getRowMapper(Calendar calendar, boolean modifiedZ) { + return (rs, rowNum) -> { + OutlierValue outlierValue = getOutlierValue(calendar, rs); + addZScoreBasedParamsToOutlierValue(outlierValue, rs, modifiedZ); + + return outlierValue; + }; + } + + /** {@inheritDoc} */ + @Override + protected void addZScoreBasedParamsToOutlierValue( + OutlierValue outlierValue, ResultSet rs, boolean modifiedZ) throws SQLException { + if (modifiedZ) { + outlierValue.setStdDev(rs.getDouble("mad")); + } else { + outlierValue.setStdDev(rs.getDouble("std_dev")); + } + + super.addZScoreBasedParamsToOutlierValue(outlierValue, rs, modifiedZ); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/DefaultOutlierDetectionService.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/DefaultOutlierDetectionService.java index 46ceec22deb9..94430bfc1c1b 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/DefaultOutlierDetectionService.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/DefaultOutlierDetectionService.java @@ -28,24 +28,14 @@ package org.hisp.dhis.outlierdetection.service; import java.io.IOException; -import java.io.OutputStream; -import java.util.Collection; +import java.io.Writer; import java.util.List; -import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.dataelement.DataElement; -import org.hisp.dhis.dataset.DataSet; -import org.hisp.dhis.feedback.ErrorCode; -import org.hisp.dhis.feedback.ErrorMessage; -import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.outlierdetection.OutlierDetectionMetadata; -import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; import org.hisp.dhis.outlierdetection.OutlierValue; import org.hisp.dhis.system.util.JacksonCsvUtils; import org.springframework.stereotype.Service; @@ -54,120 +44,36 @@ * @author Lars Helge Overland */ @Slf4j +@AllArgsConstructor @Service -@RequiredArgsConstructor -public class DefaultOutlierDetectionService implements OutlierDetectionService { - private static final int MAX_LIMIT = 10_000; - - private final IdentifiableObjectManager idObjectManager; +public class DefaultOutlierDetectionService { private final ZScoreOutlierDetectionManager zScoreOutlierDetection; - private final MinMaxOutlierDetectionManager minMaxOutlierDetection; - @Override - public void validate(OutlierDetectionRequest request) throws IllegalQueryException { - ErrorMessage error = validateForErrorMessage(request); - - if (error != null) { - log.warn( - String.format( - "Outlier detection request validation failed, code: '%s', message: '%s'", - error.getErrorCode(), error.getMessage())); - - throw new IllegalQueryException(error); - } - } - - @Override - public ErrorMessage validateForErrorMessage(OutlierDetectionRequest request) { - ErrorMessage error = null; - - if (request.getDataElements().isEmpty()) { - error = new ErrorMessage(ErrorCode.E2200); - } else if (request.getStartDate() == null || request.getEndDate() == null) { - error = new ErrorMessage(ErrorCode.E2201); - } else if (request.getStartDate().after(request.getEndDate())) { - error = new ErrorMessage(ErrorCode.E2202); - } else if (request.getOrgUnits().isEmpty()) { - error = new ErrorMessage(ErrorCode.E2203); - } else if (request.getThreshold() <= 0) { - error = new ErrorMessage(ErrorCode.E2204); - } else if (request.getMaxResults() <= 0) { - error = new ErrorMessage(ErrorCode.E2205); - } else if (request.getMaxResults() > MAX_LIMIT) { - error = new ErrorMessage(ErrorCode.E2206, MAX_LIMIT); - } else if (request.hasDataStartEndDate() - && request.getDataStartDate().after(request.getDataEndDate())) { - error = new ErrorMessage(ErrorCode.E2207); - } - - return error; - } - - @Override - public OutlierDetectionRequest getFromQuery(OutlierDetectionQuery query) { - OutlierDetectionRequest.Builder request = new OutlierDetectionRequest.Builder(); - - List dataSets = idObjectManager.getByUid(DataSet.class, query.getDs()); - - // Re-fetch data elements to maintain access control - - List de = - dataSets.stream() - .map(DataSet::getDataElements) - .flatMap(Collection::stream) - .filter(d -> d.getValueType().isNumeric()) - .map(DataElement::getUid) - .collect(Collectors.toList()); - - de.addAll(query.getDe()); - - List dataElements = idObjectManager.getByUid(DataElement.class, de); - List orgUnits = - idObjectManager.getByUid(OrganisationUnit.class, query.getOu()); - - request - .withDataElements(dataElements) - .withStartEndDate(query.getStartDate(), query.getEndDate()) - .withOrgUnits(orgUnits) - .withDataStartDate(query.getDataStartDate()) - .withDataEndDate(query.getDataEndDate()); - - if (query.getAlgorithm() != null) { - request.withAlgorithm(query.getAlgorithm()); - } - - if (query.getThreshold() != null) { - request.withThreshold(query.getThreshold()); - } - - if (query.getOrderBy() != null) { - request.withOrderBy(query.getOrderBy()); - } - - if (query.getMaxResults() != null) { - request.withMaxResults(query.getMaxResults()); - } - - return request.build(); - } - - @Override + /** + * Transform the incoming request into api response (json). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ public OutlierDetectionResponse getOutlierValues(OutlierDetectionRequest request) throws IllegalQueryException { - validate(request); - final OutlierDetectionResponse response = new OutlierDetectionResponse(); response.setOutlierValues(getOutliers(request)); response.setMetadata(getMetadata(request, response.getOutlierValues())); return response; } - @Override - public void getOutlierValuesAsCsv(OutlierDetectionRequest request, OutputStream out) + /** + * Transform the incoming request into api response (csv download). + * + * @param request the {@link OutlierDetectionRequest}. + * @return the {@link OutlierDetectionResponse}. + */ + public void getOutlierValuesAsCsv(OutlierDetectionRequest request, Writer writer) throws IllegalQueryException, IOException { - JacksonCsvUtils.toCsv(getOutlierValues(request).getOutlierValues(), OutlierValue.class, out); + JacksonCsvUtils.toCsv(getOutlierValues(request).getOutlierValues(), OutlierValue.class, writer); } /** @@ -197,15 +103,9 @@ private OutlierDetectionMetadata getMetadata( * @return a list of {@link OutlierValue}. */ private List getOutliers(OutlierDetectionRequest request) { - switch (request.getAlgorithm()) { - case Z_SCORE: - case MOD_Z_SCORE: - return zScoreOutlierDetection.getOutlierValues(request); - case MIN_MAX: - return minMaxOutlierDetection.getOutlierValues(request); - default: - throw new IllegalStateException( - String.format("Outlier detection algorithm not supported: %s", request.getAlgorithm())); - } + return switch (request.getAlgorithm()) { + case Z_SCORE, MOD_Z_SCORE -> zScoreOutlierDetection.getOutlierValues(request); + case MIN_MAX -> minMaxOutlierDetection.getOutlierValues(request); + }; } } diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/MinMaxOutlierDetectionManager.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/MinMaxOutlierDetectionManager.java index f4b5184771f2..41bc022628fb 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/MinMaxOutlierDetectionManager.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/MinMaxOutlierDetectionManager.java @@ -27,23 +27,13 @@ */ package org.hisp.dhis.outlierdetection.service; -import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.getOrgUnitPathClause; -import static org.hisp.dhis.period.PeriodType.getIsoPeriod; - -import java.util.List; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.calendar.Calendar; -import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.feedback.ErrorCode; -import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierValue; -import org.hisp.dhis.period.PeriodType; -import org.springframework.dao.DataIntegrityViolationException; +import org.hisp.dhis.outlierdetection.processor.OutlierSqlStatementProcessor; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; -import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Repository; /** @@ -52,105 +42,26 @@ * @author Lars Helge Overland */ @Slf4j -@RequiredArgsConstructor @Repository -public class MinMaxOutlierDetectionManager { - private final NamedParameterJdbcTemplate jdbcTemplate; - - /** - * Returns a list of outlier data values based on min-max values for the given request. - * - * @param request the {@link OutlierDetectionRequest}. - * @return a list of {@link OutlierValue}. - */ - public List getOutlierValues(OutlierDetectionRequest request) { - final String ouPathClause = getOrgUnitPathClause(request.getOrgUnits()); - - final String sql = - "select de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, " - + "de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, " - + "pe.startdate as pe_start_date, pt.name as pt_name, " - + "dv.value::double precision as value, dv.followup as follow_up, " - + "least(abs(dv.value::double precision - mm.minimumvalue), " - + "abs(dv.value::double precision - mm.maximumvalue)) as bound_abs_dev, " - + "mm.minimumvalue as lower_bound, " - + "mm.maximumvalue as upper_bound " - + "from datavalue dv " - + "inner join dataelement de on dv.dataelementid = de.dataelementid " - + "inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid " - + "inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid " - + "inner join period pe on dv.periodid = pe.periodid " - + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " - + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " - + - // Min-max value join - "inner join minmaxdataelement mm on (dv.dataelementid = mm.dataelementid " - + "and dv.sourceid = mm.sourceid and dv.categoryoptioncomboid = mm.categoryoptioncomboid) " - + "where dv.dataelementid in (:data_element_ids) " - + "and pe.startdate >= :start_date " - + "and pe.enddate <= :end_date " - + "and " - + ouPathClause - + " " - + "and dv.deleted is false " - + - // Filter for values outside the min-max range - "and (dv.value::double precision < mm.minimumvalue or dv.value::double precision > mm.maximumvalue) " - + - // Order and limit - "order by bound_abs_dev desc " - + "limit :max_results;"; - - final SqlParameterSource params = - new MapSqlParameterSource() - .addValue("data_element_ids", request.getDataElementIds()) - .addValue("start_date", request.getStartDate()) - .addValue("end_date", request.getEndDate()) - .addValue("max_results", request.getMaxResults()); - - final Calendar calendar = PeriodType.getCalendar(); - - try { - return jdbcTemplate.query(sql, params, getRowMapper(calendar)); - } catch (DataIntegrityViolationException ex) { - // Casting non-numeric data to double, catching exception is faster - // than filtering - - log.error(ErrorCode.E2208.getMessage(), ex); - - throw new IllegalQueryException(ErrorCode.E2208); - } +public class MinMaxOutlierDetectionManager extends AbstractOutlierDetectionManager { + protected MinMaxOutlierDetectionManager( + NamedParameterJdbcTemplate jdbcTemplate, + @Qualifier("minMaxSqlStatementProcessor") + OutlierSqlStatementProcessor sqlStatementProcessor) { + super(jdbcTemplate, sqlStatementProcessor); } - /** - * Returns a {@link RowMapper} for {@link OutlierValue}. - * - * @param calendar the {@link Calendar}. - * @return a {@link RowMapper}. - */ - private RowMapper getRowMapper(final Calendar calendar) { + /** {@inheritDoc} */ + @Override + protected RowMapper getRowMapper(Calendar calendar, boolean modifiedZ) { return (rs, rowNum) -> { - final OutlierValue outlier = new OutlierValue(); - - final String isoPeriod = - getIsoPeriod(calendar, rs.getString("pt_name"), rs.getDate("pe_start_date")); - - outlier.setDe(rs.getString("de_uid")); - outlier.setDeName(rs.getString("de_name")); - outlier.setPe(isoPeriod); - outlier.setOu(rs.getString("ou_uid")); - outlier.setOuName(rs.getString("ou_name")); - outlier.setCoc(rs.getString("coc_uid")); - outlier.setCocName(rs.getString("coc_name")); - outlier.setAoc(rs.getString("aoc_uid")); - outlier.setAocName(rs.getString("aoc_name")); - outlier.setValue(rs.getDouble("value")); - outlier.setAbsDev(rs.getDouble("bound_abs_dev")); - outlier.setLowerBound(rs.getDouble("lower_bound")); - outlier.setUpperBound(rs.getDouble("upper_bound")); - outlier.setFollowup(rs.getBoolean("follow_up")); + OutlierValue outlierValue = getOutlierValue(calendar, rs); + outlierValue.setAbsDev(rs.getDouble("bound_abs_dev")); + outlierValue.setLowerBound(rs.getDouble("lower_bound")); + outlierValue.setUpperBound(rs.getDouble("upper_bound")); + outlierValue.setFollowup(rs.getBoolean("follow_up")); - return outlier; + return outlierValue; }; } } diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/ZScoreOutlierDetectionManager.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/ZScoreOutlierDetectionManager.java index 3a33bb9c2141..77631969655d 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/ZScoreOutlierDetectionManager.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/service/ZScoreOutlierDetectionManager.java @@ -27,27 +27,16 @@ */ package org.hisp.dhis.outlierdetection.service; -import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.getDataEndDateClause; -import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.getDataStartDateClause; -import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.getOrgUnitPathClause; -import static org.hisp.dhis.period.PeriodType.getIsoPeriod; - -import java.util.List; -import lombok.RequiredArgsConstructor; +import java.sql.ResultSet; +import java.sql.SQLException; import lombok.extern.slf4j.Slf4j; import org.hisp.dhis.calendar.Calendar; -import org.hisp.dhis.common.IllegalQueryException; -import org.hisp.dhis.feedback.ErrorCode; -import org.hisp.dhis.outlierdetection.Order; import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; -import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierValue; -import org.hisp.dhis.period.PeriodType; -import org.springframework.dao.DataIntegrityViolationException; +import org.hisp.dhis.outlierdetection.processor.OutlierSqlStatementProcessor; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; -import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Repository; /** @@ -61,165 +50,34 @@ * @author Lars Helge Overland */ @Slf4j -@RequiredArgsConstructor @Repository -public class ZScoreOutlierDetectionManager { - private final NamedParameterJdbcTemplate jdbcTemplate; - - /** - * Returns a list of outlier data values based on z-score for the given request. - * - * @param request the {@link OutlierDetectionRequest}. - * @return a list of {@link OutlierValue}. - */ - public List getOutlierValues(OutlierDetectionRequest request) { - final String ouPathClause = getOrgUnitPathClause(request.getOrgUnits()); - final String dataStartDateClause = getDataStartDateClause(request.getDataStartDate()); - final String dataEndDateClause = getDataEndDateClause(request.getDataEndDate()); - - final boolean modifiedZ = request.getAlgorithm() == OutlierDetectionAlgorithm.MOD_Z_SCORE; - final String middle_stats_calc = - modifiedZ - ? "percentile_cont(0.5) within group(order by dv.value::double precision)" - : "avg(dv.value::double precision)"; - - String order = - request.getOrderBy() == Order.MEAN_ABS_DEV - ? "middle_value_abs_dev" - : request.getOrderBy().getKey(); - - final String sql = - "select dvs.de_uid, dvs.ou_uid, dvs.coc_uid, dvs.aoc_uid, " - + "dvs.de_name, dvs.ou_name, dvs.coc_name, dvs.aoc_name, dvs.value, dvs.follow_up, " - + "dvs.pe_start_date, dvs.pt_name, " - + "stats.middle_value as middle_value, " - + "stats.std_dev as std_dev, " - + "abs(dvs.value::double precision - stats.middle_value) as middle_value_abs_dev, " - + "abs(dvs.value::double precision - stats.middle_value) / stats.std_dev as z_score, " - + "stats.middle_value - (stats.std_dev * :threshold) as lower_bound, " - + "stats.middle_value + (stats.std_dev * :threshold) as upper_bound " - + - // Data value query - "from (" - + "select dv.dataelementid, dv.sourceid, dv.periodid, " - + "dv.categoryoptioncomboid, dv.attributeoptioncomboid, " - + "de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, " - + "de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, " - + "pe.startdate as pe_start_date, pt.name as pt_name, " - + "dv.value as value, dv.followup as follow_up " - + "from datavalue dv " - + "inner join dataelement de on dv.dataelementid = de.dataelementid " - + "inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid " - + "inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid " - + "inner join period pe on dv.periodid = pe.periodid " - + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " - + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " - + "where dv.dataelementid in (:data_element_ids) " - + "and pe.startdate >= :start_date " - + "and pe.enddate <= :end_date " - + "and " - + ouPathClause - + " " - + "and dv.deleted is false" - + ") as dvs " - + - // Mean or Median and std dev mapping query - "inner join (" - + "select dv.dataelementid as dataelementid, dv.sourceid as sourceid, " - + "dv.categoryoptioncomboid as categoryoptioncomboid, " - + "dv.attributeoptioncomboid as attributeoptioncomboid, " - + middle_stats_calc - + " as middle_value, " - + "stddev_pop(dv.value::double precision) as std_dev " - + "from datavalue dv " - + "inner join period pe on dv.periodid = pe.periodid " - + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " - + "where dv.dataelementid in (:data_element_ids) " - + dataStartDateClause - + dataEndDateClause - + "and " - + ouPathClause - + " " - + "and dv.deleted is false " - + "group by dv.dataelementid, dv.sourceid, dv.categoryoptioncomboid, dv.attributeoptioncomboid" - + ") as stats " - + - // Query join - "on dvs.dataelementid = stats.dataelementid " - + "and dvs.sourceid = stats.sourceid " - + "and dvs.categoryoptioncomboid = stats.categoryoptioncomboid " - + "and dvs.attributeoptioncomboid = stats.attributeoptioncomboid " - + "where stats.std_dev != 0.0 " - + - // Filter on z-score threshold - "and (abs(dvs.value::double precision - stats.middle_value) / stats.std_dev) >= :threshold " - + - // Order and limit - "order by " - + order - + " desc " - + "limit :max_results;"; - - final SqlParameterSource params = - new MapSqlParameterSource() - .addValue("threshold", request.getThreshold()) - .addValue("data_element_ids", request.getDataElementIds()) - .addValue("start_date", request.getStartDate()) - .addValue("end_date", request.getEndDate()) - .addValue("data_start_date", request.getDataStartDate()) - .addValue("data_end_date", request.getDataEndDate()) - .addValue("max_results", request.getMaxResults()); +public class ZScoreOutlierDetectionManager extends AbstractOutlierDetectionManager { - final Calendar calendar = PeriodType.getCalendar(); - - try { - return jdbcTemplate.query(sql, params, getRowMapper(calendar, modifiedZ)); - } catch (DataIntegrityViolationException ex) { - // Casting non-numeric data to double, catching exception is faster - // than filtering - - log.error(ErrorCode.E2208.getMessage(), ex); - - throw new IllegalQueryException(ErrorCode.E2208); - } + protected ZScoreOutlierDetectionManager( + NamedParameterJdbcTemplate jdbcTemplate, + @Qualifier("zscoreSqlStatementProcessor") + OutlierSqlStatementProcessor sqlStatementProcessor) { + super(jdbcTemplate, sqlStatementProcessor); } - /** - * Returns a {@link RowMapper} for {@link OutlierValue}. - * - * @param calendar the {@link Calendar}. - * @return a {@link RowMapper}. - */ - private RowMapper getRowMapper(final Calendar calendar, boolean modifiedZ) { + /** {@inheritDoc} */ + @Override + protected RowMapper getRowMapper(Calendar calendar, boolean modifiedZ) { return (rs, rowNum) -> { - final OutlierValue outlier = new OutlierValue(); + OutlierValue outlierValue = getOutlierValue(calendar, rs); + addZScoreBasedParamsToOutlierValue(outlierValue, rs, modifiedZ); + outlierValue.setFollowup(rs.getBoolean("follow_up")); - final String isoPeriod = - getIsoPeriod(calendar, rs.getString("pt_name"), rs.getDate("pe_start_date")); + return outlierValue; + }; + } - outlier.setDe(rs.getString("de_uid")); - outlier.setDeName(rs.getString("de_name")); - outlier.setPe(isoPeriod); - outlier.setOu(rs.getString("ou_uid")); - outlier.setOuName(rs.getString("ou_name")); - outlier.setCoc(rs.getString("coc_uid")); - outlier.setCocName(rs.getString("coc_name")); - outlier.setAoc(rs.getString("aoc_uid")); - outlier.setAocName(rs.getString("aoc_name")); - outlier.setValue(rs.getDouble("value")); - if (modifiedZ) { - outlier.setMedian(rs.getDouble("middle_value")); - } else { - outlier.setMean(rs.getDouble("middle_value")); - } - outlier.setStdDev(rs.getDouble("std_dev")); - outlier.setAbsDev(rs.getDouble("middle_value_abs_dev")); - outlier.setZScore(rs.getDouble("z_score")); - outlier.setLowerBound(rs.getDouble("lower_bound")); - outlier.setUpperBound(rs.getDouble("upper_bound")); - outlier.setFollowup(rs.getBoolean("follow_up")); + /** {@inheritDoc} */ + @Override + protected void addZScoreBasedParamsToOutlierValue( + OutlierValue outlierValue, ResultSet rs, boolean modifiedZ) throws SQLException { + outlierValue.setStdDev(rs.getDouble("std_dev")); - return outlier; - }; + super.addZScoreBasedParamsToOutlierValue(outlierValue, rs, modifiedZ); } } diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtils.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtils.java index a74074da9f17..77b3feb6de77 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtils.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtils.java @@ -27,50 +27,74 @@ */ package org.hisp.dhis.outlierdetection.util; -import java.util.Date; +import static org.hisp.dhis.feedback.ErrorCode.E2208; +import static org.hisp.dhis.feedback.ErrorCode.E7131; +import static org.hisp.dhis.util.SqlExceptionUtils.relationDoesNotExist; + import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.common.IllegalQueryException; +import org.hisp.dhis.common.QueryRuntimeException; import org.hisp.dhis.commons.util.TextUtils; +import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.util.SqlExceptionUtils; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.jdbc.BadSqlGrammarException; /** * @author Lars Helge Overland */ +@Slf4j public class OutlierDetectionUtils { + /** * Returns an organisation unit 'path' "like" clause for the given list of {@link * OrganisationUnit}. * - * @param query the list of {@link OrganisationUnit}. + * @param orgUnits the list of {@link OrganisationUnit}. * @return an organisation unit 'path' "like" clause. */ - public static String getOrgUnitPathClause(List orgUnits) { - String sql = "("; + public static String getOrgUnitPathClause(List orgUnits, String pathAlias) { + StringBuilder sql = new StringBuilder("("); + orgUnits.forEach( + ou -> + sql.append(pathAlias).append(".\"path\" like '").append(ou.getPath()).append("%' or ")); - for (OrganisationUnit ou : orgUnits) { - sql += "ou.\"path\" like '" + ou.getPath() + "%' or "; - } - - return StringUtils.trim(TextUtils.removeLastOr(sql)) + ")"; + return StringUtils.trim(TextUtils.removeLastOr(sql.toString())) + ")"; } /** - * Returns a period data start date clause. + * Wraps the provided interface around a common exception handling strategy. * - * @param dataStartDate the data start date. - * @return a period data start date clause. + * @param supplier the {@link Supplier} containing the code block to execute and wrap around the + * exception handling. + * @return the {@link Optional} wrapping th result of the supplier execution. */ - public static String getDataStartDateClause(Date dataStartDate) { - return dataStartDate != null ? "and pe.startdate >= :data_start_date " : StringUtils.EMPTY; - } + public static Optional withExceptionHandling(Supplier supplier) { + try { + return Optional.ofNullable(supplier.get()); + } catch (BadSqlGrammarException ex) { + if (relationDoesNotExist(ex.getSQLException())) { + log.info(SqlExceptionUtils.ERR_MSG_TABLE_NOT_EXISTING, ex); + throw ex; + } + log.info(SqlExceptionUtils.ERR_MSG_SILENT_FALLBACK, ex); + } catch (QueryRuntimeException ex) { + log.error("Internal runtime exception", ex); + throw ex; + } catch (DataIntegrityViolationException ex) { + log.error(E2208.getMessage(), ex); + throw new IllegalQueryException(ErrorCode.E2208); + } catch (DataAccessResourceFailureException ex) { + log.error(E7131.getMessage(), ex); + throw new QueryRuntimeException(E7131); + } - /** - * Returns a period data end date clause. - * - * @param dataStartDate the data start date. - * @return a period data end date clause. - */ - public static String getDataEndDateClause(Date dataStartDate) { - return dataStartDate != null ? "and pe.enddate <= :data_end_date " : StringUtils.EMPTY; + return Optional.empty(); } } diff --git a/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/validation/outlierdetection/ValidationOutlierDetectionRequest.java b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/validation/outlierdetection/ValidationOutlierDetectionRequest.java new file mode 100644 index 000000000000..d5d79b86dad5 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/main/java/org/hisp/dhis/validation/outlierdetection/ValidationOutlierDetectionRequest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.validation.outlierdetection; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.common.IllegalQueryException; +import org.hisp.dhis.feedback.ErrorCode; +import org.hisp.dhis.feedback.ErrorMessage; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingManager; +import org.springframework.stereotype.Component; + +/** OutlierDetectionRequest validator. */ +@Component +@AllArgsConstructor +@Slf4j +public class ValidationOutlierDetectionRequest { + + private final SystemSettingManager systemSettingManager; + + /** + * Validates the given request. + * + * @param request the {@link OutlierDetectionRequest}. + * @throws IllegalQueryException if request is invalid. + */ + public void validate(OutlierDetectionRequest request, boolean isAnalytics) + throws IllegalQueryException { + ErrorMessage errorMessage = validateForErrorMessage(request, isAnalytics); + + if (errorMessage != null) { + log.warn( + String.format( + "Outlier detection request validation failed, code: '%s', message: '%s'", + errorMessage.getErrorCode(), errorMessage.getMessage())); + + throw new IllegalQueryException(errorMessage); + } + } + + private ErrorMessage validateForErrorMessage( + OutlierDetectionRequest request, boolean isAnalytics) { + + int maxLimit = + isAnalytics + ? systemSettingManager.getSystemSetting(SettingKey.ANALYTICS_MAX_LIMIT, Integer.class) + : 500; + ErrorMessage errorMessage = getErrorMessage(request, maxLimit); + + if (errorMessage != null) { + return errorMessage; + } + + if (isAnalytics) { + if (request.getDataStartDate() != null) { + return new ErrorMessage(ErrorCode.E2209); + } + if (request.getDataEndDate() != null) { + return new ErrorMessage(ErrorCode.E2210); + } + if (request.getAlgorithm() == OutlierDetectionAlgorithm.MIN_MAX) { + return new ErrorMessage(ErrorCode.E2211); + } + } + + if (request.hasDataStartEndDate() + && request.getDataStartDate().after(request.getDataEndDate())) { + return new ErrorMessage(ErrorCode.E2207); + } + + return null; + } + + private ErrorMessage getErrorMessage(OutlierDetectionRequest request, int maxLimit) { + ErrorMessage error = null; + + if (request.getDataElements().isEmpty()) { + error = new ErrorMessage(ErrorCode.E2200); + } else if (request.getStartDate() == null || request.getEndDate() == null) { + error = new ErrorMessage(ErrorCode.E2201); + } else if (request.getStartDate().after(request.getEndDate())) { + error = new ErrorMessage(ErrorCode.E2202); + } else if (request.getOrgUnits().isEmpty()) { + error = new ErrorMessage(ErrorCode.E2203); + } else if (request.getThreshold() <= 0) { + error = new ErrorMessage(ErrorCode.E2204); + } else if (request.getMaxResults() <= 0) { + error = new ErrorMessage(ErrorCode.E2205); + } else if (request.getMaxResults() > maxLimit) { + error = new ErrorMessage(ErrorCode.E2206, maxLimit); + } + + return error; + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZscoreSqlStatementProcessorTest.java b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZscoreSqlStatementProcessorTest.java new file mode 100644 index 000000000000..5f685a6e1d72 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/AnalyticsZscoreSqlStatementProcessorTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.DhisConvenienceTest.createDataElement; +import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; +import static org.hisp.dhis.DhisConvenienceTest.getDate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AnalyticsZscoreSqlStatementProcessorTest { + private OutlierSqlStatementProcessor subject; + + // ------------------------------------------------------------------------- + // Fixture + // ------------------------------------------------------------------------- + + private DataElement deA; + + private DataElement deB; + + private DataElement deC; + + private OrganisationUnit ouA; + + private OrganisationUnit ouB; + + @BeforeEach + public void setUp() { + subject = new AnalyticsZScoreSqlStatementProcessor(); + + deA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); + deB = createDataElement('B', ValueType.INTEGER, AggregationType.SUM); + deC = createDataElement('C', ValueType.NUMBER, AggregationType.SUM); + + ouA = createOrganisationUnit('A'); + ouB = createOrganisationUnit('B'); + } + + @Test + void testGetSqlStatement() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .build(); + String sql = subject.getSqlStatement(request); + String expected = + "select * from (select ax.dataelementid, ax.de_uid, ax.ou_uid, ax.coc_uid, ax.aoc_uid, ax.de_name, ax.ou_name, ax.coc_name, ax.aoc_name, ax.value, ax.pestartdate as pe_start_date, ax.pt_name, ax.avg_middle_value as middle_value, ax.std_dev as std_dev, ax.mad as mad, abs(ax.value::double precision - ax.avg_middle_value) as middle_value_abs_dev, (case when ax.std_dev = 0 then 0 else abs(ax.value::double precision - ax.avg_middle_value ) / ax.std_dev end) as z_score, ax.avg_middle_value - (ax.std_dev * :threshold) as lower_bound, ax.avg_middle_value + (ax.std_dev * :threshold) as upper_bound from analytics ax where dataelementid in (:data_element_ids) and (ax.\"path\" like '/ouabcdefghA%' or ax.\"path\" like '/ouabcdefghB%') and ax.pestartdate >= :start_date and ax.peenddate <= :end_date) t1 where t1.z_score > :threshold order by middle_value_abs_dev desc limit :max_results "; + assertEquals(expected, sql); + } + + @Test + void testGetSqlStatementWithZScore() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withAlgorithm(OutlierDetectionAlgorithm.Z_SCORE) + .build(); + String sql = subject.getSqlStatement(request); + assertTrue(sql.contains("avg_middle_value")); + } + + @Test + void testGetSqlStatementWithModifiedZScore() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withAlgorithm(OutlierDetectionAlgorithm.MOD_Z_SCORE) + .build(); + String sql = subject.getSqlStatement(request); + assertTrue(sql.contains("percentile_middle_value")); + } + + @Test + void testGetSqlStatementWithNullRequest() { + assertEquals(StringUtils.EMPTY, subject.getSqlStatement(null)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessorTest.java b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessorTest.java new file mode 100644 index 000000000000..9b195925076c --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/MinMaxSqlStatementProcessorTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.DhisConvenienceTest.createDataElement; +import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; +import static org.hisp.dhis.DhisConvenienceTest.getDate; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MinMaxSqlStatementProcessorTest { + private OutlierSqlStatementProcessor subject; + + // ------------------------------------------------------------------------- + // Fixture + // ------------------------------------------------------------------------- + + private DataElement deA; + + private DataElement deB; + + private DataElement deC; + + private OrganisationUnit ouA; + + private OrganisationUnit ouB; + + @BeforeEach + public void setUp() { + subject = new MinMaxSqlStatementProcessor(); + + deA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); + deB = createDataElement('B', ValueType.INTEGER, AggregationType.SUM); + deC = createDataElement('C', ValueType.NUMBER, AggregationType.SUM); + + ouA = createOrganisationUnit('A'); + ouB = createOrganisationUnit('B'); + } + + @Test + void testGetSqlStatement() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .build(); + String sql = subject.getSqlStatement(request); + String expected = + "select de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, pe.startdate as pe_start_date, pt.name as pt_name, dv.value::double precision as value, dv.followup as follow_up, least(abs(dv.value::double precision - mm.minimumvalue), abs(dv.value::double precision - mm.maximumvalue)) as bound_abs_dev, mm.minimumvalue as lower_bound, mm.maximumvalue as upper_bound from datavalue dv inner join dataelement de on dv.dataelementid = de.dataelementid inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid inner join period pe on dv.periodid = pe.periodid inner join periodtype pt on pe.periodtypeid = pt.periodtypeid inner join organisationunit ou on dv.sourceid = ou.organisationunitid inner join minmaxdataelement mm on (dv.dataelementid = mm.dataelementid and dv.sourceid = mm.sourceid and dv.categoryoptioncomboid = mm.categoryoptioncomboid) where dv.dataelementid in (:data_element_ids) and pe.startdate >= :start_date and pe.enddate <= :end_date and (ou.\"path\" like '/ouabcdefghA%' or ou.\"path\" like '/ouabcdefghB%') and dv.deleted is false and (dv.value::double precision < mm.minimumvalue or dv.value::double precision > mm.maximumvalue) order by bound_abs_dev desc limit :max_results;"; + assertEquals(expected, sql); + } + + @Test + void testGetSqlStatementWithNullRequest() { + assertEquals(StringUtils.EMPTY, subject.getSqlStatement(null)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessorTest.java b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessorTest.java new file mode 100644 index 000000000000..9710ad498244 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/processor/ZscoreSqlStatementProcessorTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.outlierdetection.processor; + +import static org.hisp.dhis.DhisConvenienceTest.createDataElement; +import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; +import static org.hisp.dhis.DhisConvenienceTest.getDate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; +import org.hisp.dhis.analytics.AggregationType; +import org.hisp.dhis.common.ValueType; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ZscoreSqlStatementProcessorTest { + private OutlierSqlStatementProcessor subject; + + // ------------------------------------------------------------------------- + // Fixture + // ------------------------------------------------------------------------- + + private DataElement deA; + + private DataElement deB; + + private DataElement deC; + + private OrganisationUnit ouA; + + private OrganisationUnit ouB; + + @BeforeEach + public void setUp() { + subject = new ZscoreSqlStatementProcessor(); + + deA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); + deB = createDataElement('B', ValueType.INTEGER, AggregationType.SUM); + deC = createDataElement('C', ValueType.NUMBER, AggregationType.SUM); + + ouA = createOrganisationUnit('A'); + ouB = createOrganisationUnit('B'); + } + + @Test + void testGetSqlStatement() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .build(); + String sql = subject.getSqlStatement(request); + String expected = + "select dvs.de_uid, dvs.ou_uid, dvs.coc_uid, dvs.aoc_uid, dvs.de_name, dvs.ou_name, dvs.coc_name, dvs.aoc_name, dvs.value, dvs.follow_up, dvs.pe_start_date, dvs.pt_name, stats.middle_value as middle_value, stats.std_dev as std_dev, abs(dvs.value::double precision - stats.middle_value) as middle_value_abs_dev, abs(dvs.value::double precision - stats.middle_value) / stats.std_dev as z_score, stats.middle_value - (stats.std_dev * :threshold) as lower_bound, stats.middle_value + (stats.std_dev * :threshold) as upper_bound from (select dv.dataelementid, dv.sourceid, dv.periodid, dv.categoryoptioncomboid, dv.attributeoptioncomboid, de.uid as de_uid, ou.uid as ou_uid, coc.uid as coc_uid, aoc.uid as aoc_uid, de.name as de_name, ou.name as ou_name, coc.name as coc_name, aoc.name as aoc_name, pe.startdate as pe_start_date, pt.name as pt_name, dv.value as value, dv.followup as follow_up from datavalue dv inner join dataelement de on dv.dataelementid = de.dataelementid inner join categoryoptioncombo coc on dv.categoryoptioncomboid = coc.categoryoptioncomboid inner join categoryoptioncombo aoc on dv.attributeoptioncomboid = aoc.categoryoptioncomboid inner join period pe on dv.periodid = pe.periodid inner join periodtype pt on pe.periodtypeid = pt.periodtypeid inner join organisationunit ou on dv.sourceid = ou.organisationunitid where dv.dataelementid in (:data_element_ids) and pe.startdate >= :start_date and pe.enddate <= :end_date and (ou.\"path\" like '/ouabcdefghA%' or ou.\"path\" like '/ouabcdefghB%') and dv.deleted is false) as dvs inner join (select dv.dataelementid as dataelementid, dv.sourceid as sourceid, dv.categoryoptioncomboid as categoryoptioncomboid, dv.attributeoptioncomboid as attributeoptioncomboid, avg(dv.value::double precision) as middle_value, stddev_pop(dv.value::double precision) as std_dev from datavalue dv inner join period pe on dv.periodid = pe.periodid inner join organisationunit ou on dv.sourceid = ou.organisationunitid where dv.dataelementid in (:data_element_ids) and (ou.\"path\" like '/ouabcdefghA%' or ou.\"path\" like '/ouabcdefghB%') and dv.deleted is false group by dv.dataelementid, dv.sourceid, dv.categoryoptioncomboid, dv.attributeoptioncomboid) as stats on dvs.dataelementid = stats.dataelementid and dvs.sourceid = stats.sourceid and dvs.categoryoptioncomboid = stats.categoryoptioncomboid and dvs.attributeoptioncomboid = stats.attributeoptioncomboid where stats.std_dev != 0.0 and (abs(dvs.value::double precision - stats.middle_value) / stats.std_dev) >= :threshold order by middle_value_abs_dev desc limit :max_results;"; + assertEquals(expected, sql); + } + + @Test + void testGetSqlStatementWithZScore() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withAlgorithm(OutlierDetectionAlgorithm.Z_SCORE) + .build(); + String sql = subject.getSqlStatement(request); + assertTrue(sql.contains("avg(dv.value::double precision)")); + } + + @Test + void testGetSqlStatementWithModifiedZScore() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 3, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withAlgorithm(OutlierDetectionAlgorithm.MOD_Z_SCORE) + .build(); + String sql = subject.getSqlStatement(request); + assertTrue( + sql.contains("percentile_cont(0.5) within group(order by dv.value::double precision)")); + } + + @Test + void testGetSqlStatementWithNullRequest() { + assertEquals(StringUtils.EMPTY, subject.getSqlStatement(null)); + } +} diff --git a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtilsTest.java b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtilsTest.java index 83dfcc371674..e58321e3203d 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtilsTest.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/util/OutlierDetectionUtilsTest.java @@ -28,12 +28,21 @@ package org.hisp.dhis.outlierdetection.util; import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; +import static org.hisp.dhis.feedback.ErrorCode.E2208; +import static org.hisp.dhis.feedback.ErrorCode.E7131; +import static org.hisp.dhis.outlierdetection.util.OutlierDetectionUtils.withExceptionHandling; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.Lists; import java.util.List; +import org.hisp.dhis.common.IllegalQueryException; +import org.hisp.dhis.common.QueryRuntimeException; +import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.junit.jupiter.api.Test; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; /** * @author Lars Helge Overland @@ -48,6 +57,33 @@ void testGetOrgUnitPathClause() { List orgUnits = Lists.newArrayList(ouA, ouB, ouC); String expected = "(ou.\"path\" like '/ouabcdefghA%' or ou.\"path\" like '/ouabcdefghB%' or ou.\"path\" like '/ouabcdefghC%')"; - assertEquals(expected, OutlierDetectionUtils.getOrgUnitPathClause(orgUnits)); + assertEquals(expected, OutlierDetectionUtils.getOrgUnitPathClause(orgUnits, "ou")); + } + + @Test + void testWithIllegalQueryExceptionHandling() { + IllegalQueryException illegalQueryException = + assertThrows( + IllegalQueryException.class, + () -> withExceptionHandling(() -> supplyWithErrorCode(E2208))); + assertEquals(E2208.getMessage(), illegalQueryException.getMessage()); + } + + @Test + void testWithQueryRuntimeExceptionHandling() { + QueryRuntimeException queryRuntimeException = + assertThrows( + QueryRuntimeException.class, + () -> withExceptionHandling(() -> supplyWithErrorCode(E7131))); + assertEquals(E7131.getMessage(), queryRuntimeException.getMessage()); + } + + private Object supplyWithErrorCode(ErrorCode errorCode) { + switch (errorCode) { + case E2208 -> throw new DataIntegrityViolationException(errorCode.getMessage()); + case E7131 -> throw new DataAccessResourceFailureException(errorCode.getMessage()); + } + + return null; } } diff --git a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceValidationTest.java b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/validation/ValidationOutlierDetectionRequestTest.java similarity index 67% rename from dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceValidationTest.java rename to dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/validation/ValidationOutlierDetectionRequestTest.java index e2bedd3cdd6b..8aa9bc8f275a 100644 --- a/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceValidationTest.java +++ b/dhis-2/dhis-services/dhis-service-validation/src/test/java/org/hisp/dhis/validation/ValidationOutlierDetectionRequestTest.java @@ -25,44 +25,45 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.hisp.dhis.outlierdetection.service; +package org.hisp.dhis.validation; import static org.hisp.dhis.DhisConvenienceTest.createDataElement; import static org.hisp.dhis.DhisConvenienceTest.createOrganisationUnit; import static org.hisp.dhis.DhisConvenienceTest.getDate; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; import com.google.common.collect.Lists; import org.hisp.dhis.analytics.AggregationType; -import org.hisp.dhis.common.IdentifiableObjectManager; import org.hisp.dhis.common.IllegalQueryException; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.feedback.ErrorCode; import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.outlierdetection.OutlierDetectionAlgorithm; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.validation.outlierdetection.ValidationOutlierDetectionRequest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; -/** - * @author Lars Helge Overland - */ @ExtendWith(MockitoExtension.class) -class OutlierDetectionServiceValidationTest { - - @Mock private IdentifiableObjectManager idObjectManager; +@MockitoSettings(strictness = Strictness.LENIENT) +class ValidationOutlierDetectionRequestTest { - @Mock private ZScoreOutlierDetectionManager zScoreOutlierManager; + @Mock private SystemSettingManager systemSettingManager; - @Mock private MinMaxOutlierDetectionManager minMaxOutlierManager; - - private OutlierDetectionService subject; + private ValidationOutlierDetectionRequest subject; // ------------------------------------------------------------------------- // Fixture @@ -80,10 +81,9 @@ class OutlierDetectionServiceValidationTest { @BeforeEach public void setUp() { - subject = - new DefaultOutlierDetectionService( - idObjectManager, zScoreOutlierManager, minMaxOutlierManager); - + subject = new ValidationOutlierDetectionRequest(systemSettingManager); + when(systemSettingManager.getSystemSetting(SettingKey.ANALYTICS_MAX_LIMIT, Integer.class)) + .thenReturn(500); deA = createDataElement('A', ValueType.INTEGER, AggregationType.SUM); deB = createDataElement('B', ValueType.INTEGER, AggregationType.SUM); deC = createDataElement('C', ValueType.NUMBER, AggregationType.SUM); @@ -92,8 +92,9 @@ public void setUp() { ouB = createOrganisationUnit('B'); } - @Test - void testSuccessfulValidation() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSuccessfulValidation(boolean isAnalytics) { OutlierDetectionRequest request = new OutlierDetectionRequest.Builder() .withDataElements(Lists.newArrayList(deA, deB, deC)) @@ -101,10 +102,11 @@ void testSuccessfulValidation() { .withOrgUnits(Lists.newArrayList(ouA, ouB)) .build(); - assertNull(subject.validateForErrorMessage(request)); + assertDoesNotThrow(() -> subject.validate(request, isAnalytics)); } - @Test + @ParameterizedTest + @ValueSource(booleans = {true, false}) void testErrorValidation() { OutlierDetectionRequest request = new OutlierDetectionRequest.Builder() @@ -113,7 +115,7 @@ void testErrorValidation() { .build(); IllegalQueryException ex = - assertThrows(IllegalQueryException.class, () -> subject.validate(request)); + assertThrows(IllegalQueryException.class, () -> subject.validate(request, true)); assertEquals(ErrorCode.E2203, ex.getErrorCode()); } @@ -124,8 +126,7 @@ void testErrorNoDataElements() { .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 7, 1)) .withOrgUnits(Lists.newArrayList(ouA, ouB)) .build(); - - assertEquals(ErrorCode.E2200, subject.validateForErrorMessage(request).getErrorCode()); + assertRequest(request, ErrorCode.E2200); } @Test @@ -137,7 +138,7 @@ void testErrorStartAfterEndDates() { .withOrgUnits(Lists.newArrayList(ouA, ouB)) .build(); - assertEquals(ErrorCode.E2202, subject.validateForErrorMessage(request).getErrorCode()); + assertRequest(request, ErrorCode.E2202); } @Test @@ -149,8 +150,7 @@ void testErrorNegativeThreshold() { .withOrgUnits(Lists.newArrayList(ouA, ouB)) .withThreshold(-23.4) .build(); - - assertEquals(ErrorCode.E2204, subject.validateForErrorMessage(request).getErrorCode()); + assertRequest(request, ErrorCode.E2204); } @Test @@ -163,20 +163,55 @@ void testErrorNegativeMaxResults() { .withMaxResults(-100) .build(); - assertEquals(ErrorCode.E2205, subject.validateForErrorMessage(request).getErrorCode()); + assertRequest(request, ErrorCode.E2205); + } + + @Test + void testErrorDataStartDateIsNotAllowed() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 6, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withDataStartDate(getDate(2020, 5, 1)) + .withDataEndDate(getDate(2020, 6, 1)) + .build(); + + assertRequest(request, ErrorCode.E2209); } @Test - void testErrorDataStartDateAfterDataEndDate() { + void testErrorDataEndDateIsNotAllowed() { OutlierDetectionRequest request = new OutlierDetectionRequest.Builder() .withDataElements(Lists.newArrayList(deA, deB, deC)) .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 6, 1)) .withOrgUnits(Lists.newArrayList(ouA, ouB)) - .withDataStartDate(getDate(2020, 6, 1)) .withDataEndDate(getDate(2020, 5, 1)) .build(); - assertEquals(ErrorCode.E2207, subject.validateForErrorMessage(request).getErrorCode()); + assertRequest(request, ErrorCode.E2210); + } + + @Test + void testMinMaxAlgorithmIsNotAllowed() { + OutlierDetectionRequest request = + new OutlierDetectionRequest.Builder() + .withDataElements(Lists.newArrayList(deA, deB, deC)) + .withStartEndDate(getDate(2020, 1, 1), getDate(2020, 6, 1)) + .withOrgUnits(Lists.newArrayList(ouA, ouB)) + .withAlgorithm(OutlierDetectionAlgorithm.MIN_MAX) + .build(); + + assertRequest(request, ErrorCode.E2211); + } + + private void assertRequest(OutlierDetectionRequest request, ErrorCode errorCode) { + assertThrows(IllegalQueryException.class, () -> subject.validate(request, true)); + try { + subject.validate(request, false); + } catch (IllegalQueryException e) { + assertEquals(errorCode, e.getErrorCode()); + } } } diff --git a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/collection/CachingMapTest.java b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/collection/CachingMapTest.java index ab52e1083abf..c0c006eacc4a 100644 --- a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/collection/CachingMapTest.java +++ b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/collection/CachingMapTest.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.commons.collection; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashSet; import java.util.Set; diff --git a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/CronUtilsTest.java b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/CronUtilsTest.java index 958c9fa63477..c16539282fb6 100644 --- a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/CronUtilsTest.java +++ b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/CronUtilsTest.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.commons.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/ExpressionFunctionsTest.java b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/ExpressionFunctionsTest.java index f83e2e70be7e..df474d25c994 100644 --- a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/ExpressionFunctionsTest.java +++ b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/ExpressionFunctionsTest.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.commons.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/PageRangeTest.java b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/PageRangeTest.java index df3b33d7dfd4..4911440f8649 100644 --- a/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/PageRangeTest.java +++ b/dhis-2/dhis-support/dhis-support-commons/src/test/java/org/hisp/dhis/commons/util/PageRangeTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.commons.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/java/org/hisp/dhis/db/migration/helper/JdbcSqlFileExecutor.java b/dhis-2/dhis-support/dhis-support-db-migration/src/main/java/org/hisp/dhis/db/migration/helper/JdbcSqlFileExecutor.java index 34e014d6b6de..2577b4e10d48 100644 --- a/dhis-2/dhis-support/dhis-support-db-migration/src/main/java/org/hisp/dhis/db/migration/helper/JdbcSqlFileExecutor.java +++ b/dhis-2/dhis-support/dhis-support-db-migration/src/main/java/org/hisp/dhis/db/migration/helper/JdbcSqlFileExecutor.java @@ -27,8 +27,14 @@ */ package org.hisp.dhis.db.migration.helper; -import java.io.*; -import java.sql.*; +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Reader; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; diff --git a/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_39__Create_jsonb_function_search_translations_token.sql b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_39__Create_jsonb_function_search_translations_token.sql new file mode 100644 index 000000000000..fb7ec3e846da --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-db-migration/src/main/resources/org/hisp/dhis/db/migration/2.41/V2_41_39__Create_jsonb_function_search_translations_token.sql @@ -0,0 +1,19 @@ +/** + Find translated value that matches given token and given locale. + @param $1 the translations column name + @param $2 the properties to search for (array of strings such as '{NAME,SHORT_NAME}') + @param $3 the locale language to search for + @param $4 the token to search (example : '(?=.*année)') + */ +CREATE OR replace FUNCTION jsonb_search_translated_token(jsonb, text, text, text) +RETURNS bool +AS $$ +SELECT exists( + SELECT 1 + FROM jsonb_array_elements($1) trans + WHERE trans->>'property' = ANY ($2::text[]) + AND trans->>'locale' = $3 + AND trans->>'value' ~* $4 +); +$$ +LANGUAGE SQL IMMUTABLE PARALLEL SAFE; \ No newline at end of file diff --git a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/config/ServiceConfig.java b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/config/ServiceConfig.java index 6e6e492190b4..8baeb25c1f3c 100644 --- a/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/config/ServiceConfig.java +++ b/dhis-2/dhis-support/dhis-support-external/src/main/java/org/hisp/dhis/external/config/ServiceConfig.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.external.config; -import static org.hisp.dhis.external.conf.ConfigurationKey.*; +import static org.hisp.dhis.external.conf.ConfigurationKey.META_DATA_SYNC_RETRY; +import static org.hisp.dhis.external.conf.ConfigurationKey.META_DATA_SYNC_RETRY_TIME_FREQUENCY_MILLISEC; +import static org.hisp.dhis.external.conf.ConfigurationKey.SYSTEM_SESSION_TIMEOUT; import org.hisp.dhis.external.conf.ConfigurationPropertyFactoryBean; import org.hisp.dhis.external.location.DefaultLocationManager; diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonProgramRuleEvaluationEnvironmentSetBinaryType.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonProgramRuleEvaluationEnvironmentSetBinaryType.java index fdece8872e44..db7e4a4a7698 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonProgramRuleEvaluationEnvironmentSetBinaryType.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonProgramRuleEvaluationEnvironmentSetBinaryType.java @@ -29,7 +29,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.Set; import org.hibernate.HibernateException; import org.hisp.dhis.programrule.ProgramRuleActionEvaluationEnvironment; diff --git a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonbFunctions.java b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonbFunctions.java index 6606f0eeed1c..121b7b68fd78 100644 --- a/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonbFunctions.java +++ b/dhis-2/dhis-support/dhis-support-hibernate/src/main/java/org/hisp/dhis/hibernate/jsonb/type/JsonbFunctions.java @@ -79,4 +79,6 @@ public class JsonbFunctions { * to search $2 Regular expression for matching */ public static final String REGEXP_SEARCH = "regexp_search"; + + public static final String SEARCH_TRANSLATION_TOKEN = "jsonb_search_translated_token"; } diff --git a/dhis-2/dhis-support/dhis-support-system/pom.xml b/dhis-2/dhis-support/dhis-support-system/pom.xml index c089cbf8d070..be4bc309f4ff 100644 --- a/dhis-2/dhis-support/dhis-support-system/pom.xml +++ b/dhis-2/dhis-support/dhis-support-system/pom.xml @@ -205,10 +205,6 @@ javax.annotation javax.annotation-api - - commons-beanutils - commons-beanutils - com.fasterxml.jackson.core jackson-core diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/HibernateDatabaseInfoProvider.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/HibernateDatabaseInfoProvider.java index 0ace43ccb460..94e8f9a913bb 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/HibernateDatabaseInfoProvider.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/database/HibernateDatabaseInfoProvider.java @@ -29,10 +29,13 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.util.Date; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.PostConstruct; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.commons.util.SystemUtils; @@ -126,14 +129,16 @@ public void init() { String url = config.getProperty(ConfigurationKey.CONNECTION_URL); String user = config.getProperty(ConfigurationKey.CONNECTION_USERNAME); - InternalDatabaseInfo internalDatabaseInfo = getInternalDatabaseInfo(); - - info = new DatabaseInfo(); - info.setName(internalDatabaseInfo.getDatabase()); - info.setUser(StringUtils.defaultIfEmpty(internalDatabaseInfo.getUser(), user)); - info.setUrl(url); - info.setSpatialSupport(spatialSupport); - info.setDatabaseVersion(internalDatabaseInfo.getVersion()); + InternalDatabaseInfo internalInfo = getInternalDatabaseInfo(); + + info = + DatabaseInfo.builder() + .name(internalInfo.getDatabase()) + .user(StringUtils.defaultIfEmpty(internalInfo.getUser(), user)) + .url(url) + .spatialSupport(spatialSupport) + .databaseVersion(internalInfo.getVersion()) + .build(); } // ------------------------------------------------------------------------- @@ -142,9 +147,7 @@ public void init() { @Override public DatabaseInfo getDatabaseInfo() { - // parts of returned object may be reset due to security reasons - // (clone must be created to preserve original values) - return info == null ? null : info.instance(); + return info.toBuilder().time(now()).build(); } @Override @@ -160,28 +163,27 @@ public boolean isInMemory() { private InternalDatabaseInfo getInternalDatabaseInfo() { if (SystemUtils.isH2(environment.getActiveProfiles())) { return new InternalDatabaseInfo(); - } else { - try { - final InternalDatabaseInfo internalDatabaseInfo = - jdbcTemplate.queryForObject( - "select version(),current_catalog,current_user", - (rs, rowNum) -> { - String version = rs.getString(1); - final Matcher versionMatcher = POSTGRES_VERSION_PATTERN.matcher(version); - - if (versionMatcher.find()) { - version = versionMatcher.group(1); - } - - return new InternalDatabaseInfo(version, rs.getString(2), rs.getString(3)); - }); - - return internalDatabaseInfo == null ? new InternalDatabaseInfo() : internalDatabaseInfo; - } catch (Exception ex) { - log.error("An error occurred when retrieving database info.", ex); - - return new InternalDatabaseInfo(); - } + } + try { + final InternalDatabaseInfo internalDatabaseInfo = + jdbcTemplate.queryForObject( + "select version(),current_catalog,current_user", + (rs, rowNum) -> { + String version = rs.getString(1); + final Matcher versionMatcher = POSTGRES_VERSION_PATTERN.matcher(version); + + if (versionMatcher.find()) { + version = versionMatcher.group(1); + } + + return new InternalDatabaseInfo(version, rs.getString(2), rs.getString(3)); + }); + + return internalDatabaseInfo == null ? new InternalDatabaseInfo() : internalDatabaseInfo; + } catch (Exception ex) { + log.error("An error occurred when retrieving database info.", ex); + + return new InternalDatabaseInfo(); } } @@ -189,6 +191,10 @@ private void checkDatabaseConnectivity() { jdbcTemplate.queryForObject("select 'checking db connection';", String.class); } + private Date now() { + return jdbcTemplate.queryForObject("select now()", Date.class); + } + /** * Attempts to create a spatial database extension. Checks if spatial operations are supported. */ @@ -253,33 +259,16 @@ private boolean isBtreeGinExtensionCreated() { } } + @Getter + @RequiredArgsConstructor protected static class InternalDatabaseInfo { - private final String version; + private final String version; private final String database; - private final String user; public InternalDatabaseInfo() { - this(StringUtils.EMPTY, StringUtils.EMPTY, StringUtils.EMPTY); - } - - public InternalDatabaseInfo(String version, String database, String user) { - this.version = version; - this.database = database; - this.user = user; - } - - public String getVersion() { - return version; - } - - public String getDatabase() { - return database; - } - - public String getUser() { - return user; + this("", "", ""); } } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/JacksonCsvUtils.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/JacksonCsvUtils.java index 31a586b31add..276f76c11f1a 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/JacksonCsvUtils.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/JacksonCsvUtils.java @@ -31,7 +31,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import java.io.IOException; -import java.io.OutputStream; +import java.io.Writer; import org.hisp.dhis.commons.jackson.config.JacksonObjectMapperConfig; /** @@ -43,13 +43,13 @@ public class JacksonCsvUtils { * is inferred from the given type using {@CsvSchema}. A header line is included. * * @param value the value to write. - * @param out the {@link OutputStream} to write to. + * @param writer the {@link Writer} to write to. * @throws IOException if the write operation fails. */ - public static void toCsv(Object value, Class type, OutputStream out) throws IOException { + public static void toCsv(Object value, Class type, Writer writer) throws IOException { CsvMapper csvMapper = JacksonObjectMapperConfig.csvMapper; CsvSchema schema = csvMapper.schemaFor(type).withHeader(); - ObjectWriter writer = csvMapper.writer(schema); - writer.writeValue(out, value); + ObjectWriter objectWriter = csvMapper.writer(schema); + objectWriter.writeValue(writer, value); } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/database/DatabaseInfoTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/database/DatabaseInfoTest.java index c026d5fc934b..7cb9b56d1b97 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/database/DatabaseInfoTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/database/DatabaseInfoTest.java @@ -27,47 +27,37 @@ */ package org.hisp.dhis.system.database; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.apache.commons.beanutils.BeanUtils; +import java.util.Date; import org.junit.jupiter.api.Test; /** * @author Volker Schmidt */ class DatabaseInfoTest { - @Test - void cloneDatabaseInfo() throws Exception { - DatabaseInfo databaseInfo = getDataBaseInfo(); - DatabaseInfo cloned = databaseInfo.instance(); - BeanUtils.copyProperties(cloned, databaseInfo); - assertNotSame(databaseInfo, cloned); - assertEquals(databaseInfo.getName(), cloned.getName()); - assertEquals(databaseInfo.getUser(), cloned.getUser()); - assertEquals(databaseInfo.getUrl(), cloned.getUrl()); - assertEquals(databaseInfo.getDatabaseVersion(), cloned.getDatabaseVersion()); - assertEquals(databaseInfo.isSpatialSupport(), cloned.isSpatialSupport()); - } @Test void clearSensitiveInfo() { - DatabaseInfo databaseInfo = getDataBaseInfo(); - databaseInfo.clearSensitiveInfo(); - assertNull(databaseInfo.getName()); - assertNull(databaseInfo.getUser()); - assertNull(databaseInfo.getUrl()); - assertNull(databaseInfo.getDatabaseVersion()); + DatabaseInfo info = getDataBaseInfo().withoutSensitiveInfo(); + assertNull(info.getName()); + assertNull(info.getUser()); + assertNull(info.getUrl()); + assertNull(info.getDatabaseVersion()); + assertTrue(info.isSpatialSupport()); + assertNotNull(info.getTime()); } private DatabaseInfo getDataBaseInfo() { - DatabaseInfo databaseInfo = new DatabaseInfo(); - databaseInfo.setName("testDatabase"); - databaseInfo.setUser("testUser"); - databaseInfo.setUrl("theUrl"); - databaseInfo.setDatabaseVersion("xzy 10.7"); - databaseInfo.setSpatialSupport(true); - return databaseInfo; + return DatabaseInfo.builder() + .name("testDatabase") + .user("testUser") + .url("theUrl") + .databaseVersion("xzy 10.7") + .spatialSupport(true) + .time(new Date()) + .build(); } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/grid/GridUtilsTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/grid/GridUtilsTest.java index 454ce21a7289..82c11d7c4443 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/grid/GridUtilsTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/grid/GridUtilsTest.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.system.grid; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import com.google.common.collect.Lists; import java.nio.charset.StandardCharsets; diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/CodeUtilsTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/CodeUtilsTest.java index 6f326b9cc46b..789821a6e1c4 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/CodeUtilsTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/CodeUtilsTest.java @@ -27,7 +27,7 @@ */ package org.hisp.dhis.system.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test; diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/LocaleUtilsTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/LocaleUtilsTest.java index 1d337f07c69e..8db2c14168e0 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/LocaleUtilsTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/LocaleUtilsTest.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.system.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Locale; diff --git a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java index b2faadcacc5b..a0d4e56ccb83 100644 --- a/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java +++ b/dhis-2/dhis-support/dhis-support-test/src/main/java/org/hisp/dhis/DhisConvenienceTest.java @@ -2277,16 +2277,19 @@ public static DataSetNotificationTemplate createDataSetNotificationTemplate( DataSetNotificationTrigger dataSetNotificationTrigger, Integer relativeScheduledDays, SendStrategy sendStrategy) { - return new DataSetNotificationTemplate( - Sets.newHashSet(), - Sets.newHashSet(), - "Message", - notificationRecipient, - dataSetNotificationTrigger, - "Subject", - null, - relativeScheduledDays, - sendStrategy); + DataSetNotificationTemplate dst = + new DataSetNotificationTemplate( + newHashSet(), + newHashSet(), + "Message", + notificationRecipient, + dataSetNotificationTrigger, + "Subject", + null, + relativeScheduledDays, + sendStrategy); + dst.setName(name); + return dst; } public static ValidationNotificationTemplate createValidationNotificationTemplate(String name) { diff --git a/dhis-2/dhis-test-e2e/pom.xml b/dhis-2/dhis-test-e2e/pom.xml index 06bea5e36cfc..dcd875da8233 100644 --- a/dhis-2/dhis-test-e2e/pom.xml +++ b/dhis-2/dhis-test-e2e/pom.xml @@ -15,17 +15,17 @@ 1.2.1 5.10.1 2.10.1 - 2.21.1 + 2.22.0 5.3.2 2.16.0 32.1.3-jre 0.10.3 5.1.8 1.0.2 - 5.8 + 5.9 1.0.12 4.4 - 3.13.0 + 3.14.0 2.8.0 1.5.1 4.2.0 @@ -34,7 +34,7 @@ 4.15.0 2.0.9 4.5.14 - 5.2.1 + 5.2.2 2.15.0 diff --git a/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/dto/TrackerApiResponse.java b/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/dto/TrackerApiResponse.java index 400626b89c50..03abfccf2668 100644 --- a/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/dto/TrackerApiResponse.java +++ b/dhis-2/dhis-test-e2e/src/main/java/org/hisp/dhis/dto/TrackerApiResponse.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.dto; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.number.OrderingComparison.greaterThanOrEqualTo; import io.restassured.response.ValidatableResponse; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/LoginTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/LoginTests.java index be17387300c1..ffc3db7becdf 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/LoginTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/LoginTests.java @@ -27,8 +27,6 @@ */ package org.hisp.dhis; -import static org.hamcrest.Matchers.*; - import org.hisp.dhis.actions.LoginActions; import org.hisp.dhis.actions.UserActions; import org.hisp.dhis.utils.DataGenerator; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java index 19fbcb133b6f..5853d7ca0b5d 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java @@ -28,7 +28,15 @@ package org.hisp.dhis.aggregate; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import com.google.gson.JsonArray; import com.google.gson.JsonElement; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/TrackedEntityInstanceAclReadTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/TrackedEntityInstanceAclReadTests.java index 2d01110e04e4..7f443d4556b6 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/TrackedEntityInstanceAclReadTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/TrackedEntityInstanceAclReadTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.deprecated.tracker; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Lists; import com.google.gson.JsonArray; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventExportTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventExportTests.java index 2839cc2f3b22..7dbcf3188ae8 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventExportTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventExportTests.java @@ -30,10 +30,11 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.*; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.in; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportIdSchemeTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportIdSchemeTests.java index ae0955f7b431..e56b65ba66bd 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportIdSchemeTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportIdSchemeTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.deprecated.tracker.event; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.number.OrderingComparison.greaterThan; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportValidationTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportValidationTests.java index f4f3b52be4d1..4adaa34443d4 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportValidationTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/deprecated/tracker/event/EventImportValidationTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.deprecated.tracker.event; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; +import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.jupiter.api.Assertions.assertNotNull; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/helpers/TestCleanUp.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/helpers/TestCleanUp.java index 0a300f7f91e8..24fa79971dc7 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/helpers/TestCleanUp.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/helpers/TestCleanUp.java @@ -27,7 +27,12 @@ */ package org.hisp.dhis.helpers; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hisp.dhis.TestRunStorage; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/MetadataPatchTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/MetadataPatchTests.java index 2bfe88bd62e1..f0cb30416439 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/MetadataPatchTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/MetadataPatchTests.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.metadata; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import com.google.gson.JsonArray; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/MetadataImportTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/MetadataImportTest.java index d23b91c17527..6e49973f5b68 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/MetadataImportTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/metadata_import/MetadataImportTest.java @@ -28,8 +28,23 @@ package org.hisp.dhis.metadata.metadata_import; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import com.google.gson.JsonArray; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitProfileTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitProfileTests.java index 4ac4b7eabc44..785883da1c78 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitProfileTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitProfileTests.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.metadata.orgunits; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; import com.google.gson.JsonArray; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitsRemovalTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitsRemovalTest.java index 3d6741c96019..580f62a31ba2 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitsRemovalTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/orgunits/OrgUnitsRemovalTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.metadata.orgunits; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import java.io.File; import org.hamcrest.Matchers; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/programs/ProgramStagesTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/programs/ProgramStagesTest.java index 2852834166ca..b9fa0ae52dcd 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/programs/ProgramStagesTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/programs/ProgramStagesTest.java @@ -27,7 +27,11 @@ */ package org.hisp.dhis.metadata.programs; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import com.google.gson.JsonObject; import org.hisp.dhis.ApiTest; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/users/UserLookupTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/users/UserLookupTests.java index e5880814781c..36db1e29807d 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/users/UserLookupTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/metadata/users/UserLookupTests.java @@ -28,7 +28,10 @@ package org.hisp.dhis.metadata.users; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import com.google.gson.JsonArray; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/PotentialDuplicatesTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/PotentialDuplicatesTests.java index 6aaf5b897651..bdf12a893e1c 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/PotentialDuplicatesTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/PotentialDuplicatesTests.java @@ -29,7 +29,12 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hisp.dhis.helpers.matchers.MatchesJson.matchesJSON; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesAttributeMergeTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesAttributeMergeTests.java index a949d91dca5a..a87f0a416156 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesAttributeMergeTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesAttributeMergeTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.tracker.deduplication.merge; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; import com.google.gson.JsonObject; import java.util.Arrays; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesEnrollmentsTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesEnrollmentsTests.java index 10e6523c9e7e..cfc4ca728f62 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesEnrollmentsTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesEnrollmentsTests.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.tracker.deduplication.merge; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesMergeTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesMergeTests.java index 29cf65b5ec74..f55164634ba6 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesMergeTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/deduplication/merge/PotentialDuplicatesMergeTests.java @@ -27,7 +27,10 @@ */ package org.hisp.dhis.tracker.deduplication.merge; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.both; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.everyItem; import java.util.Arrays; import org.hamcrest.Matchers; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/RuleEngineTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/RuleEngineTests.java index ee4706283d4e..2de66cc64970 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/RuleEngineTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/RuleEngineTests.java @@ -27,7 +27,15 @@ */ package org.hisp.dhis.tracker.imports; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.stringContainsInOrder; import com.google.gson.JsonObject; import java.io.File; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/TrackerImporterImportModeTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/TrackerImporterImportModeTests.java index 45b36bd3b0cf..5ae006a10a72 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/TrackerImporterImportModeTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/TrackerImporterImportModeTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.tracker.imports; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; import java.io.File; import org.hisp.dhis.dto.ApiResponse; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/enrollments/EnrollmentAttributeTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/enrollments/EnrollmentAttributeTests.java index a35cbee5ed05..54be3ae86165 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/enrollments/EnrollmentAttributeTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/enrollments/EnrollmentAttributeTests.java @@ -28,7 +28,10 @@ package org.hisp.dhis.tracker.imports.enrollments; import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; import com.google.gson.JsonObject; import java.util.Arrays; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventNotesTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventNotesTests.java index 654e2987c7b6..d563dcbd7234 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventNotesTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventNotesTests.java @@ -28,7 +28,10 @@ package org.hisp.dhis.tracker.imports.events; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; import com.google.gson.JsonObject; import org.hisp.dhis.Constants; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventUpdateTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventUpdateTests.java index 3ba7582925ea..e5e39e193e5c 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventUpdateTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventUpdateTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.tracker.imports.events; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.Matchers.hasSize; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventsDataValueValidationTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventsDataValueValidationTests.java index 5a12e1024b14..a3985a7a9f7d 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventsDataValueValidationTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/events/EventsDataValueValidationTests.java @@ -29,7 +29,11 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertNotNull; import com.google.gson.JsonObject; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiUpdateTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiUpdateTests.java index 23373d3bbc41..d1f360aa7989 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiUpdateTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiUpdateTests.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.tracker.imports.tei; -import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; import com.google.gson.JsonObject; import java.io.File; diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiValidationTests.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiValidationTests.java index 51826031de2f..02b2224c8c63 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiValidationTests.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/tracker/imports/tei/TeiValidationTests.java @@ -27,7 +27,13 @@ */ package org.hisp.dhis.tracker.imports.tei; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsStringIgnoringCase; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; import com.google.gson.JsonObject; import org.hisp.dhis.Constants; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java index 87e3304a1d0f..ca10b84684ce 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/analytics/data/AnalyticsServiceTest.java @@ -253,8 +253,8 @@ public void setUpTest() throws IOException, InterruptedException { // We need to make sure that table generation start time is greater than // lastUpdated on tables populated in the setup - Date oneSecondFromNow = - Date.from(LocalDateTime.now().plusSeconds(1).atZone(ZoneId.systemDefault()).toInstant()); + Date tenSecondsFromNow = + Date.from(LocalDateTime.now().plusSeconds(10).atZone(ZoneId.systemDefault()).toInstant()); assertNull( systemSettingManager.getSystemSetting( @@ -265,7 +265,7 @@ public void setUpTest() throws IOException, InterruptedException { processStartTime = new Date(); // Generate analytics tables analyticsTableGenerator.generateTables( - AnalyticsTableUpdateParams.newBuilder().withStartTime(oneSecondFromNow).build(), + AnalyticsTableUpdateParams.newBuilder().withStartTime(tenSecondsFromNow).build(), NoopJobProgress.INSTANCE); } @@ -1539,7 +1539,11 @@ void resourceTablesTimestampUpdated() { systemSettingManager.getSystemSetting( SettingKey.LAST_SUCCESSFUL_RESOURCE_TABLES_UPDATE, Date.class); assertNotEquals(null, resourceTablesUpdated); - assertTrue(tableLastUpdated.compareTo(processStartTime) > 0); - assertTrue(resourceTablesUpdated.compareTo(processStartTime) > 0); + assertTrue( + tableLastUpdated.compareTo(processStartTime) > 0, + String.format("%s > %s", tableLastUpdated, processStartTime)); + assertTrue( + resourceTablesUpdated.compareTo(processStartTime) > 0, + String.format("%s > %s", resourceTablesUpdated, processStartTime)); } } diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java index eea4b2d672d0..936afc0e71c4 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataanalysis/DataAnalysisStoreTest.java @@ -37,7 +37,8 @@ import org.hisp.dhis.category.CategoryCombo; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.category.CategoryService; -import org.hisp.dhis.dataelement.*; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.datavalue.DataValueService; import org.hisp.dhis.organisationunit.OrganisationUnit; import org.hisp.dhis.organisationunit.OrganisationUnitService; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataapproval/DataApprovalStoreIntegrationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataapproval/DataApprovalStoreIntegrationTest.java index 2c94b622ac56..9e4eceafdcc4 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataapproval/DataApprovalStoreIntegrationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataapproval/DataApprovalStoreIntegrationTest.java @@ -168,8 +168,7 @@ public void setUpTest() throws Exception { currentUserService, categoryService, systemSettingManager, - new PostgreSQLStatementBuilder(), - organisationUnitService); + new PostgreSQLStatementBuilder()); // --------------------------------------------------------------------- // Add supporting data diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStoreTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStoreTest.java new file mode 100644 index 000000000000..8255feee1bc7 --- /dev/null +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dataset/notifications/DataSetNotificationTemplateStoreTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.dataset.notifications; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Set; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.dataset.DataSetStore; +import org.hisp.dhis.notification.SendStrategy; +import org.hisp.dhis.test.integration.SingleSetupIntegrationTestBase; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +class DataSetNotificationTemplateStoreTest extends SingleSetupIntegrationTestBase { + + @Autowired private DataSetNotificationTemplateStore store; + @Autowired private DataSetStore dataSetStore; + + @Test + void testGetNotificationsByTriggerTypeAndDataSet() { + DataSet dataSet = createDataSet('A'); + dataSetStore.save(dataSet); + + DataSetNotificationTemplate template = + createDataSetNotificationTemplate( + "A", + DataSetNotificationRecipient.USER_GROUP, + DataSetNotificationTrigger.DATA_SET_COMPLETION, + 1, + SendStrategy.SINGLE_NOTIFICATION); + template.setDataSets(Set.of(dataSet)); + store.save(template); + + assertEquals( + 1, + store + .getNotificationsByTriggerType(dataSet, DataSetNotificationTrigger.DATA_SET_COMPLETION) + .size()); + } + + @Test + void testGetScheduledNotificationsValid() { + DataSetNotificationTemplate template = + createDataSetNotificationTemplate( + "A", + DataSetNotificationRecipient.USER_GROUP, + DataSetNotificationTrigger.DATA_SET_COMPLETION, + 1, + SendStrategy.SINGLE_NOTIFICATION); + store.save(template); + + Assertions.assertEquals( + 1, store.getScheduledNotifications(DataSetNotificationTrigger.DATA_SET_COMPLETION).size()); + } + + @Test + void testGetScheduledNotificationsNotExist() { + DataSetNotificationTemplate template = + createDataSetNotificationTemplate( + "A", + DataSetNotificationRecipient.USER_GROUP, + DataSetNotificationTrigger.DATA_SET_COMPLETION, + 1, + SendStrategy.SINGLE_NOTIFICATION); + store.save(template); + + Assertions.assertEquals( + 0, store.getScheduledNotifications(DataSetNotificationTrigger.SCHEDULED_DAYS).size()); + } +} diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/HandleRelationshipsTrackedEntityServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/HandleRelationshipsTrackedEntityServiceTest.java index eca0d966f56d..d06f047288b1 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/HandleRelationshipsTrackedEntityServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/HandleRelationshipsTrackedEntityServiceTest.java @@ -27,10 +27,11 @@ */ package org.hisp.dhis.dxf2.deprecated.tracker; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.Lists; -import java.util.*; +import java.util.HashSet; +import java.util.Set; import org.hisp.dhis.category.CategoryCombo; import org.hisp.dhis.category.CategoryOptionCombo; import org.hisp.dhis.common.IdentifiableObjectManager; @@ -41,7 +42,13 @@ import org.hisp.dhis.dxf2.importsummary.ImportStatus; import org.hisp.dhis.dxf2.importsummary.ImportSummary; import org.hisp.dhis.organisationunit.OrganisationUnit; -import org.hisp.dhis.program.*; +import org.hisp.dhis.program.Enrollment; +import org.hisp.dhis.program.EnrollmentService; +import org.hisp.dhis.program.Event; +import org.hisp.dhis.program.EventService; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramStage; +import org.hisp.dhis.program.ProgramType; import org.hisp.dhis.relationship.RelationshipType; import org.hisp.dhis.relationship.RelationshipTypeService; import org.hisp.dhis.test.integration.SingleSetupIntegrationTestBase; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackedEntityServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackedEntityServiceTest.java index c51ab031fd3f..87d375c7c42b 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackedEntityServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackedEntityServiceTest.java @@ -162,7 +162,7 @@ class TrackedEntityServiceTest extends TransactionalIntegrationTest { protected void setUpTest() throws Exception { fileResource = createFileResource('F', "fileResource".getBytes()); - fileResourceService.saveFileResource(fileResource, "fileResource".getBytes()); + fileResourceService.asyncSaveFileResource(fileResource, "fileResource".getBytes()); userService = _userService; user = createAndAddAdminUser(AUTHORITY_ALL); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/MergeServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/MergeServiceTest.java index a1f40a2aa05e..221b800c5baa 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/MergeServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/MergeServiceTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.dxf2.metadata; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.util.Date; import org.hisp.dhis.common.MergeMode; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/validation/TrackedEntityTypeValidationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/validation/TrackedEntityTypeValidationTest.java index e0f5643fe06b..3617c25b802a 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/validation/TrackedEntityTypeValidationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/dxf2/metadata/validation/TrackedEntityTypeValidationTest.java @@ -37,7 +37,7 @@ import org.hisp.dhis.dxf2.metadata.MetadataImportService; import org.hisp.dhis.dxf2.metadata.MetadataObjects; import org.hisp.dhis.dxf2.metadata.feedback.ImportReport; -import org.hisp.dhis.dxf2.metadata.objectbundle.*; +import org.hisp.dhis.dxf2.metadata.objectbundle.ObjectBundleMode; import org.hisp.dhis.feedback.Status; import org.hisp.dhis.importexport.ImportStrategy; import org.hisp.dhis.render.RenderFormat; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/fileresource/FileResourceCleanUpJobTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/fileresource/FileResourceCleanUpJobTest.java index ae35abac4c22..207a2a506760 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/fileresource/FileResourceCleanUpJobTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/fileresource/FileResourceCleanUpJobTest.java @@ -182,12 +182,12 @@ void testOrphan() { content = "filecontentA".getBytes(StandardCharsets.UTF_8); FileResource fileResourceA = createFileResource('A', content); fileResourceA.setCreated(DateTime.now().minus(Days.ONE).toDate()); - String uidA = fileResourceService.saveFileResource(fileResourceA, content); + String uidA = fileResourceService.asyncSaveFileResource(fileResourceA, content); content = "filecontentB".getBytes(StandardCharsets.UTF_8); FileResource fileResourceB = createFileResource('A', content); fileResourceB.setCreated(DateTime.now().minus(Days.ONE).toDate()); - String uidB = fileResourceService.saveFileResource(fileResourceB, content); + String uidB = fileResourceService.asyncSaveFileResource(fileResourceB, content); User userB = makeUser("B"); userB.setAvatar(fileResourceB); @@ -259,7 +259,7 @@ private DataValue createFileResourceDataValue(char uniqueChar, byte[] content) { organisationUnitService.addOrganisationUnit(orgUnit); FileResource fileResource = createFileResource(uniqueChar, content); - String uid = fileResourceService.saveFileResource(fileResource, content); + String uid = fileResourceService.asyncSaveFileResource(fileResource, content); DataValue dataValue = createDataValue(fileElement, period, orgUnit, uid, null); fileResource.setAssigned(true); @@ -275,7 +275,7 @@ private DataValue createFileResourceDataValue(char uniqueChar, byte[] content) { private ExternalFileResource createExternal(char uniqueChar, byte[] content) { ExternalFileResource externalFileResource = createExternalFileResource(uniqueChar, content); - fileResourceService.saveFileResource(externalFileResource.getFileResource(), content); + fileResourceService.asyncSaveFileResource(externalFileResource.getFileResource(), content); externalFileResourceService.saveExternalFileResource(externalFileResource); FileResource fileResource = externalFileResource.getFileResource(); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/icon/IconTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/icon/IconTest.java index 02433f00e707..b6c00dd795bd 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/icon/IconTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/icon/IconTest.java @@ -226,7 +226,7 @@ public FileResource createAndPersistFileResource(char uniqueChar) { fileResource.setCreated(new Date()); fileResource.setAutoFields(); - String fileResourceUid = fileResourceService.saveFileResource(fileResource, content); + String fileResourceUid = fileResourceService.asyncSaveFileResource(fileResource, content); return fileResourceService.getFileResource(fileResourceUid); } diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java index 9b093418cd7d..10b3ac47edba 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/organisationunit/OrganisationUnitServiceTest.java @@ -532,12 +532,12 @@ void testIsDescendantSet() { organisationUnitService.addOrganisationUnit(unit3); OrganisationUnit unit4 = createOrganisationUnit('4'); organisationUnitService.addOrganisationUnit(unit4); - assertTrue(organisationUnitService.isDescendant(unit1, Sets.newHashSet(unit1))); - assertTrue(organisationUnitService.isDescendant(unit2, Sets.newHashSet(unit1))); - assertTrue(organisationUnitService.isDescendant(unit3, Sets.newHashSet(unit1))); - assertTrue(organisationUnitService.isDescendant(unit2, Sets.newHashSet(unit1, unit3))); - assertFalse(organisationUnitService.isDescendant(unit2, Sets.newHashSet(unit3))); - assertFalse(organisationUnitService.isDescendant(unit4, Sets.newHashSet(unit1))); + assertTrue(unit1.isDescendant(Sets.newHashSet(unit1))); + assertTrue(unit2.isDescendant(Sets.newHashSet(unit1))); + assertTrue(unit3.isDescendant(Sets.newHashSet(unit1))); + assertTrue(unit2.isDescendant(Sets.newHashSet(unit1, unit3))); + assertFalse(unit2.isDescendant(Sets.newHashSet(unit3))); + assertFalse(unit4.isDescendant(Sets.newHashSet(unit1))); } @Test @@ -572,11 +572,15 @@ void testIsDescendantObject() { organisationUnitService.addOrganisationUnit(unit3); OrganisationUnit unit4 = createOrganisationUnit('4'); organisationUnitService.addOrganisationUnit(unit4); - assertTrue(organisationUnitService.isDescendant(unit1, unit1)); - assertTrue(organisationUnitService.isDescendant(unit2, unit1)); - assertTrue(organisationUnitService.isDescendant(unit3, unit1)); - assertFalse(organisationUnitService.isDescendant(unit2, unit3)); - assertFalse(organisationUnitService.isDescendant(unit4, unit1)); + assertTrue(unit1.isDescendant(unit1)); + assertTrue(unit2.isDescendant(unit1)); + assertTrue(unit3.isDescendant(unit1)); + assertTrue(unit3.isDescendant(unit2)); + assertFalse(unit2.isDescendant(unit3)); + assertFalse(unit1.isDescendant(unit2)); + assertFalse(unit1.isDescendant(unit3)); + assertFalse(unit4.isDescendant(unit1)); + assertFalse(unit1.isDescendant(unit4)); } @Test @@ -1016,7 +1020,7 @@ void testSaveImage() { fileResource.setAssigned(false); fileResource.setCreated(new Date()); fileResource.setAutoFields(); - fileResourceService.saveFileResource(fileResource, content); + fileResourceService.asyncSaveFileResource(fileResource, content); OrganisationUnit orgUnit = createOrganisationUnit('A'); orgUnit.setImage(fileResource); organisationUnitService.addOrganisationUnit(orgUnit); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceMinMaxTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceMinMaxTest.java index 9327d23c10e9..9d0c978f3d4c 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceMinMaxTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceMinMaxTest.java @@ -47,8 +47,8 @@ import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.outlierdetection.parser.OutlierDetectionQueryParser; import org.hisp.dhis.period.MonthlyPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; @@ -71,7 +71,9 @@ class OutlierDetectionServiceMinMaxTest extends IntegrationTestBase { @Autowired private DataValueService dataValueService; - @Autowired private OutlierDetectionService subject; + @Autowired private DefaultOutlierDetectionService subject; + + @Autowired private OutlierDetectionQueryParser parser; private DataElement deA; @@ -121,7 +123,7 @@ void testGetFromQuery() { query.setOu(Lists.newArrayList("ouabcdefghA", "ouabcdefghB")); query.setAlgorithm(OutlierDetectionAlgorithm.MIN_MAX); query.setMaxResults(200); - OutlierDetectionRequest request = subject.getFromQuery(query); + OutlierDetectionRequest request = parser.getFromQuery(query); assertEquals(2, request.getDataElements().size()); assertEquals(2, request.getOrgUnits().size()); assertEquals(getDate(2020, 1, 1), request.getStartDate()); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceModifiedZScoreTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceModifiedZScoreTest.java index e48b53df9f90..70ea1c810204 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceModifiedZScoreTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceModifiedZScoreTest.java @@ -35,6 +35,7 @@ import com.google.common.math.StatsAccumulator; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -53,8 +54,8 @@ import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.outlierdetection.parser.OutlierDetectionQueryParser; import org.hisp.dhis.period.MonthlyPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; @@ -73,7 +74,9 @@ class OutlierDetectionServiceModifiedZScoreTest extends IntegrationTestBase { @Autowired private DataValueService dataValueService; - @Autowired private OutlierDetectionService subject; + @Autowired private DefaultOutlierDetectionService subject; + + @Autowired private OutlierDetectionQueryParser parser; private DataElement deA; @@ -124,7 +127,7 @@ void testGetFromQuery() { query.setAlgorithm(OutlierDetectionAlgorithm.MOD_Z_SCORE); query.setThreshold(2.5); query.setMaxResults(100); - OutlierDetectionRequest request = subject.getFromQuery(query); + OutlierDetectionRequest request = parser.getFromQuery(query); assertEquals(2, request.getDataElements().size()); assertEquals(2, request.getOrgUnits().size()); assertEquals(getDate(2020, 1, 1), request.getStartDate()); @@ -247,7 +250,7 @@ void testGetOutlierValuesAsCsv() throws IOException { .withThreshold(2.0) .build(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - subject.getOutlierValuesAsCsv(request, out); + subject.getOutlierValuesAsCsv(request, new PrintWriter(out)); List csvLines = TextUtils.toLines(new String(out.toByteArray(), StandardCharsets.UTF_8)); final int endIndex = 61; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceZScoreTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceZScoreTest.java index 605f6edc61ff..5b99f36ffaf3 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceZScoreTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/outlierdetection/service/OutlierDetectionServiceZScoreTest.java @@ -34,6 +34,7 @@ import com.google.common.math.StatsAccumulator; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Stream; @@ -51,8 +52,8 @@ import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; import org.hisp.dhis.outlierdetection.OutlierValue; +import org.hisp.dhis.outlierdetection.parser.OutlierDetectionQueryParser; import org.hisp.dhis.period.MonthlyPeriodType; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; @@ -74,7 +75,9 @@ class OutlierDetectionServiceZScoreTest extends IntegrationTestBase { @Autowired private DataValueService dataValueService; - @Autowired private OutlierDetectionService subject; + @Autowired private DefaultOutlierDetectionService subject; + + @Autowired private OutlierDetectionQueryParser parser; private DataElement deA; @@ -125,7 +128,7 @@ void testGetFromQuery() { query.setAlgorithm(OutlierDetectionAlgorithm.Z_SCORE); query.setThreshold(2.5); query.setMaxResults(100); - OutlierDetectionRequest request = subject.getFromQuery(query); + OutlierDetectionRequest request = parser.getFromQuery(query); assertEquals(2, request.getDataElements().size()); assertEquals(2, request.getOrgUnits().size()); assertEquals(getDate(2020, 1, 1), request.getStartDate()); @@ -248,7 +251,7 @@ void testGetOutlierValuesAsCsv() throws IOException { .withThreshold(2.0) .build(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - subject.getOutlierValuesAsCsv(request, out); + subject.getOutlierValuesAsCsv(request, new PrintWriter(out)); List csvLines = TextUtils.toLines(new String(out.toByteArray(), StandardCharsets.UTF_8)); final int endIndex = 61; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/programrule/engine/ProgramRuleEngineTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/programrule/engine/ProgramRuleEngineTest.java index 2c91af1944d9..2a51afefdc83 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/programrule/engine/ProgramRuleEngineTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/programrule/engine/ProgramRuleEngineTest.java @@ -28,13 +28,20 @@ package org.hisp.dhis.programrule.engine; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.HashSet; +import java.util.List; import org.hamcrest.Matchers; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.common.DeliveryChannel; @@ -78,7 +85,13 @@ import org.hisp.dhis.programrule.ProgramRuleVariable; import org.hisp.dhis.programrule.ProgramRuleVariableService; import org.hisp.dhis.programrule.ProgramRuleVariableSourceType; -import org.hisp.dhis.rules.models.*; +import org.hisp.dhis.rules.models.RuleAction; +import org.hisp.dhis.rules.models.RuleActionScheduleMessage; +import org.hisp.dhis.rules.models.RuleActionSendMessage; +import org.hisp.dhis.rules.models.RuleActionShowError; +import org.hisp.dhis.rules.models.RuleActionShowWarning; +import org.hisp.dhis.rules.models.RuleEffect; +import org.hisp.dhis.rules.models.RuleEffects; import org.hisp.dhis.test.integration.TransactionalIntegrationTest; import org.hisp.dhis.trackedentity.TrackedEntity; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/schema/SchemaServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/schema/SchemaServiceTest.java index e0583e8da44c..ea3773effed1 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/schema/SchemaServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/schema/SchemaServiceTest.java @@ -103,19 +103,19 @@ void testProgramSchema() { @Test void testCanScanJobParameters() { JobParameters parameters = - new AnalyticsJobParameters(10, new HashSet<>(), new HashSet<>(), true); + new AnalyticsJobParameters(10, new HashSet<>(), new HashSet<>(), true, true); Schema schema = schemaService.getDynamicSchema(parameters.getClass()); assertNotNull(schema); assertFalse(schema.getProperties().isEmpty()); - assertEquals(4, schema.getProperties().size()); + assertEquals(5, schema.getProperties().size()); } @Test void testCanScanJobConfigurationWithJobParameters() { JobConfiguration configuration = new JobConfiguration(); configuration.setJobParameters( - new AnalyticsJobParameters(10, new HashSet<>(), new HashSet<>(), true)); + new AnalyticsJobParameters(10, new HashSet<>(), new HashSet<>(), true, true)); Schema schema = schemaService.getDynamicSchema(configuration.getClass()); assertNotNull(schema); @@ -131,6 +131,6 @@ void testCanScanJobConfigurationWithJobParameters() { assertNotNull(schema); assertFalse(schema.getProperties().isEmpty()); - assertEquals(4, schema.getProperties().size()); + assertEquals(5, schema.getProperties().size()); } } diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/setting/SystemSettingStoreTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/setting/SystemSettingStoreTest.java index e04528086afc..23dffa4bd78e 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/setting/SystemSettingStoreTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/setting/SystemSettingStoreTest.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.setting; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.util.List; import org.hisp.dhis.test.integration.TransactionalIntegrationTest; diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/trackedentityattributevalue/TrackedEntityAttributeValueServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/trackedentityattributevalue/TrackedEntityAttributeValueServiceTest.java index a3505e52fd6d..707b92765f73 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/trackedentityattributevalue/TrackedEntityAttributeValueServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/trackedentityattributevalue/TrackedEntityAttributeValueServiceTest.java @@ -198,10 +198,10 @@ void testFileAttributeValues() { content = "filecontentA".getBytes(); fileResourceA = createFileResource('A', content); fileResourceA.setContentType("image/jpg"); - fileResourceService.saveFileResource(fileResourceA, content); + fileResourceService.asyncSaveFileResource(fileResourceA, content); content = "filecontentB".getBytes(); fileResourceB = createFileResource('B', content); - fileResourceService.saveFileResource(fileResourceB, content); + fileResourceService.asyncSaveFileResource(fileResourceB, content); attributeValueA = createTrackedEntityAttributeValue('A', entityInstanceA, attributeA); attributeValueB = createTrackedEntityAttributeValue('B', entityInstanceB, attributeB); attributeValueA.setValue(fileResourceA.getUid()); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/TrackedEntityProgramAttributeFileResourceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/TrackedEntityProgramAttributeFileResourceTest.java index 5338a1ddaee6..efda3371695b 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/TrackedEntityProgramAttributeFileResourceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/TrackedEntityProgramAttributeFileResourceTest.java @@ -79,7 +79,7 @@ void testTrackedEntityProgramAttributeFileResourceValue() throws IOException { FileResourceDomain.DOCUMENT); fileResource.setUid("Jzf6hHNP7jx"); File file = File.createTempFile("file-resource", "test"); - fileResourceService.saveFileResource(fileResource, file); + fileResourceService.asyncSaveFileResource(fileResource, file); assertFalse(fileResource.isAssigned()); ImportReport importReport = trackerImportService.importTracker( diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TeTaValidationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TeTaValidationTest.java index f13c725ba1a7..171ceb52e277 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TeTaValidationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TeTaValidationTest.java @@ -79,7 +79,7 @@ void testTrackedEntityProgramAttributeFileResourceValue() throws IOException { FileResourceDomain.DOCUMENT); fileResource.setUid("Jzf6hHNP7jx"); File file = File.createTempFile("file-resource", "test"); - fileResourceService.saveFileResource(fileResource, file); + fileResourceService.asyncSaveFileResource(fileResource, file); assertFalse(fileResource.isAssigned()); TrackerObjects trackerObjects = fromJson("tracker/validations/te-program_with_tea_fileresource_data.json"); @@ -105,7 +105,7 @@ void testFileAlreadyAssign() throws IOException { FileResourceDomain.DOCUMENT); fileResource.setUid("Jzf6hHNP7jx"); File file = File.createTempFile("file-resource", "test"); - fileResourceService.saveFileResource(fileResource, file); + fileResourceService.asyncSaveFileResource(fileResource, file); assertFalse(fileResource.isAssigned()); TrackerObjects trackerObjects = fromJson("tracker/validations/te-program_with_tea_fileresource_data.json"); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TrackedEntityImportValidationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TrackedEntityImportValidationTest.java index d5f62d263553..9a477bc9fcaa 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TrackedEntityImportValidationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/TrackedEntityImportValidationTest.java @@ -38,7 +38,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; -import java.util.HashSet; import org.hisp.dhis.trackedentity.TrackedEntityService; import org.hisp.dhis.tracker.TrackerTest; import org.hisp.dhis.tracker.imports.AtomicMode; @@ -126,15 +125,9 @@ void testUpdateAccessInSearchScopeOu() throws IOException { ImportReport importReport = trackerImportService.importTracker(params, trackerObjects); assertNoErrors(importReport); assertEquals(3, importReport.getStats().getCreated()); - // For some reason teiSearchOrgunits is not created properly from - // metadata - // Redoing the update here for the time being. - User user = userService.getUser(USER_8); - user.setTeiSearchOrganisationUnits(new HashSet<>(user.getDataViewOrganisationUnits())); - userService.updateUser(user); - dbmsManager.clearSession(); + trackerObjects = fromJson("tracker/validations/te-data_with_different_ou.json"); - user = userService.getUser(USER_8); + User user = userService.getUser(USER_8); params.setUserId(user.getUid()); params.setImportStrategy(TrackerImportStrategy.CREATE_AND_UPDATE); params.setAtomicMode(AtomicMode.OBJECT); diff --git a/dhis-2/dhis-test-web-api/pom.xml b/dhis-2/dhis-test-web-api/pom.xml index d66860ac7a8e..8f3b498d730e 100644 --- a/dhis-2/dhis-test-web-api/pom.xml +++ b/dhis-2/dhis-test-web-api/pom.xml @@ -351,7 +351,7 @@ org.openapitools openapi-generator - 7.0.1 + 7.1.0 test diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/DhisControllerTestBase.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/DhisControllerTestBase.java index 2628f0ef07f1..fc195b2abd50 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/DhisControllerTestBase.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/DhisControllerTestBase.java @@ -32,7 +32,7 @@ import static org.hisp.dhis.web.WebClientUtils.assertStatus; import static org.hisp.dhis.web.WebClientUtils.failOnException; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import org.hisp.dhis.common.ValueType; import org.hisp.dhis.user.User; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/MvcTestConfig.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/MvcTestConfig.java index 115b55d93814..8ac78f354e63 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/MvcTestConfig.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/MvcTestConfig.java @@ -44,9 +44,8 @@ import org.hisp.dhis.message.MessageSender; import org.hisp.dhis.node.DefaultNodeService; import org.hisp.dhis.node.NodeService; -import org.hisp.dhis.system.SystemInfo; -import org.hisp.dhis.system.SystemService; import org.hisp.dhis.system.database.DatabaseInfo; +import org.hisp.dhis.system.database.DatabaseInfoProvider; import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.UserSettingService; import org.hisp.dhis.webapi.mvc.CurrentUserHandlerMethodArgumentResolver; @@ -61,7 +60,6 @@ import org.hisp.dhis.webapi.mvc.messageconverter.XmlMessageConverter; import org.hisp.dhis.webapi.mvc.messageconverter.XmlPathMappingJackson2XmlHttpMessageConverter; import org.hisp.dhis.webapi.view.CustomPathExtensionContentNegotiationStrategy; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; @@ -198,17 +196,9 @@ public NodeService nodeService() { return new DefaultNodeService(); } - @Bean("databaseInfo") - public DatabaseInfo databaseInfo() { - return new DatabaseInfo(); - } - - @Primary - @Bean("systemService") - public SystemService systemService() { - SystemService systemService = Mockito.mock(SystemService.class); - Mockito.when(systemService.getSystemInfo()).thenReturn(new SystemInfo()); - return systemService; + @Bean + public DatabaseInfoProvider databaseInfoProvider() { + return () -> DatabaseInfo.builder().build(); } @Override diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CrudControllerIntegrationTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CrudControllerIntegrationTest.java index fe3a8a502385..e546156635b6 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CrudControllerIntegrationTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/CrudControllerIntegrationTest.java @@ -29,14 +29,29 @@ import static org.hisp.dhis.web.WebClientUtils.assertStatus; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Locale; import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.jsontree.JsonArray; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.user.User; +import org.hisp.dhis.user.UserSettingKey; +import org.hisp.dhis.user.UserSettingService; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerIntegrationTest; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; class CrudControllerIntegrationTest extends DhisControllerIntegrationTest { + + @Autowired private UserSettingService userSettingService; + + @Autowired private SystemSettingManager systemSettingManager; + @Test void testGetNonAccessibleObject() { User admin = getCurrentUser(); @@ -73,4 +88,88 @@ void testGetAccessibleObject() { GET("/dataSets/{id}", id).content(HttpStatus.OK); } + + @Test + @DisplayName("Search by token should use translations column instead of default columns") + void testSearchByToken() { + setUpTranslation(); + User userA = createAndAddUser("userA", null, "ALL"); + userSettingService.saveUserSetting(UserSettingKey.DB_LOCALE, Locale.FRENCH, userA); + injectSecurityContext(userA); + assertTrue( + GET("/dataSets?filter=identifiable:token:bb").content().getArray("dataSets").isEmpty()); + assertFalse( + GET("/dataSets?filter=identifiable:token:fr").content().getArray("dataSets").isEmpty()); + assertFalse( + GET("/dataSets?filter=identifiable:token:dataSet") + .content() + .getArray("dataSets") + .isEmpty()); + } + + @Test + @DisplayName("Search by token should use default properties instead of translations column") + void testSearchTokenDefaultLocale() { + setUpTranslation(); + User userA = createAndAddUser("userA", null, "ALL"); + userSettingService.saveUserSetting(UserSettingKey.DB_LOCALE, Locale.ENGLISH, userA); + injectSecurityContext(userA); + + systemSettingManager.saveSystemSetting(SettingKey.DB_LOCALE, Locale.ENGLISH); + assertTrue( + GET("/dataSets?filter=identifiable:token:bb").content().getArray("dataSets").isEmpty()); + assertTrue( + GET("/dataSets?filter=identifiable:token:fr").content().getArray("dataSets").isEmpty()); + assertTrue( + GET("/dataSets?filter=identifiable:token:dataSet") + .content() + .getArray("dataSets") + .isEmpty()); + + assertFalse( + GET("/dataSets?filter=identifiable:token:my").content().getArray("dataSets").isEmpty()); + } + + @Test + @DisplayName("Search by token should use default properties instead of translations column") + void testSearchTokenWithNullLocale() { + setUpTranslation(); + User userA = createAndAddUser("userA", null, "ALL"); + injectSecurityContext(userA); + + systemSettingManager.saveSystemSetting(SettingKey.DB_LOCALE, Locale.ENGLISH); + assertTrue( + GET("/dataSets?filter=identifiable:token:bb").content().getArray("dataSets").isEmpty()); + assertTrue( + GET("/dataSets?filter=identifiable:token:fr").content().getArray("dataSets").isEmpty()); + assertTrue( + GET("/dataSets?filter=identifiable:token:dataSet") + .content() + .getArray("dataSets") + .isEmpty()); + + assertFalse( + GET("/dataSets?filter=identifiable:token:my").content().getArray("dataSets").isEmpty()); + } + + private void setUpTranslation() { + String id = + assertStatus( + HttpStatus.CREATED, + POST( + "/dataSets/", + "{'name':'My data set', 'shortName': 'MDS', 'periodType':'Monthly'}")); + + PUT( + "/dataSets/" + id + "/translations", + "{'translations': [{'locale':'fr', 'property':'NAME', 'value':'fr dataSet'}]}") + .content(HttpStatus.NO_CONTENT); + + JsonArray translations = + GET("/dataSets/{id}/translations", id).content().getArray("translations"); + assertEquals(1, translations.size()); + + assertTrue( + GET("/dataSets?filter=identifiable:token:fr", id).content().getArray("dataSets").isEmpty()); + } } diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobConfigurationRunErrorsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobConfigurationRunErrorsControllerTest.java index 0a4971250222..4c4efb2e14d3 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobConfigurationRunErrorsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobConfigurationRunErrorsControllerTest.java @@ -116,19 +116,55 @@ void testGetJobRunErrors_ObjectProgressErrors() { assertEquals(1, errors.size()); } + @Test + void testGetJobRunErrors_ListIncludeInput() throws InterruptedException { + // language=JSON + String json = + """ + { + "trackedEntities": [ + { + "trackedEntity":"sHH8mh1Fn0z", + "trackedEntityType": "nEenWmSyUEp", + "orgUnit": "DiszpKrYNg7" + } + ] + } + """; + JsonWebMessage msg = + POST("/tracker?async=true", json).content(HttpStatus.OK).as(JsonWebMessage.class); + String jobId = msg.getString("response.id").string(); + waitUntilJobIsComplete(jobId); + + JsonArray errors = GET("/jobConfigurations/errors?includeInput=true").content(); + assertEquals(2, errors.size()); + JsonObject trackerImportError = + errors.asList(JsonObject.class).stream() + .filter(obj -> obj.getString("id").string().equals(jobId)) + .findFirst() + .orElseThrow(); + + // language=JSON + String expected = + """ + {"trackedEntities":[{"trackedEntity":"sHH8mh1Fn0z","trackedEntityType":{"idScheme":"UID","identifier":"nEenWmSyUEp"},"orgUnit":{"idScheme":"UID","identifier":"DiszpKrYNg7"},"inactive":false,"deleted":false,"potentialDuplicate":false,"relationships":[],"attributes":[],"enrollments":[]}],"enrollments":[],"events":[],"relationships":[]}"""; + assertEquals(expected, trackerImportError.getObject("input").node().getDeclaration()); + } + private String createAndRunImportWithErrors() throws InterruptedException { - JsonWebMessage message = - POST( - "/metadata?async=true", - "{'organisationUnits':[{'name':'My Unit', 'shortName':'OU1'}]}") - .content(HttpStatus.OK) - .as(JsonWebMessage.class); - String jobId = message.getString("response.id").string(); + String json = "{'organisationUnits':[{'name':'My Unit', 'shortName':'OU1'}]}"; + JsonWebMessage msg = + POST("/metadata?async=true", json).content(HttpStatus.OK).as(JsonWebMessage.class); + String jobId = msg.getString("response.id").string(); + waitUntilJobIsComplete(jobId); + return jobId; + } + + private void waitUntilJobIsComplete(String jobId) throws InterruptedException { BooleanSupplier jobCompleted = () -> isDone(GET("/jobConfigurations/{id}/gist?fields=id,jobStatus", jobId).content()); assertTrue(await(ofSeconds(10), jobCompleted), "import did not run"); - return jobId; } private static boolean isDone(JsonMixed config) { diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobSchedulerControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobSchedulerControllerTest.java index dd25429b473b..512ce62981f9 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobSchedulerControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/JobSchedulerControllerTest.java @@ -29,7 +29,9 @@ import static java.lang.String.format; import static org.hisp.dhis.web.WebClientUtils.assertStatus; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import java.util.Set; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/LocaleControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/LocaleControllerTest.java index 397b9eea7e33..4e44d6f3511d 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/LocaleControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/LocaleControllerTest.java @@ -30,7 +30,7 @@ import static org.hisp.dhis.web.WebClientUtils.assertStatus; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.hisp.dhis.jsontree.*; +import org.hisp.dhis.jsontree.JsonArray; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerConvenienceTest; import org.hisp.dhis.webapi.json.domain.JsonWebLocale; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java index 9413405725af..8b9d968d5609 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java @@ -28,6 +28,8 @@ package org.hisp.dhis.webapi.controller; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import org.hisp.dhis.jsontree.JsonArray; @@ -98,6 +100,23 @@ void testGetTaskSummaryJson() { assertEquals(0, summary.size()); } + @Test + void testGetSystemInfo() { + JsonObject info = GET("/system/info").content(); + // testing one sensitive and one non-sensitive property + assertNotNull(info.getString("javaVersion").string()); + assertNotNull(info.getString("serverDate").string()); + } + + @Test + void testGetSystemInfo_NonSuperUser() { + switchToNewUser("guest"); + JsonObject info = GET("/system/info").content(); + // testing one sensitive and one non-sensitive property + assertNull(info.getString("javaVersion").string()); + assertNotNull(info.getString("serverDate").string()); + } + private static void assertObjectMembers(JsonObject root, String... members) { for (String member : members) { JsonObject memberObj = root.getObject(member); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/TwoFactorControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/TwoFactorControllerTest.java index 9b48d47d14b6..be1541c82fc0 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/TwoFactorControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/TwoFactorControllerTest.java @@ -29,7 +29,9 @@ import static org.hisp.dhis.user.UserService.TWO_FACTOR_CODE_APPROVAL_PREFIX; import static org.hisp.dhis.web.WebClientUtils.assertStatus; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.util.List; import org.hisp.dhis.user.CurrentUserUtil; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java index b44f5011f2cc..d197cee39d27 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/AbstractDataIntegrityIntegrationTest.java @@ -28,11 +28,15 @@ package org.hisp.dhis.webapi.controller.dataintegrity; import static org.hisp.dhis.web.WebClientUtils.assertStatus; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Set; import java.util.stream.Collectors; -import org.hisp.dhis.jsontree.*; +import org.hisp.dhis.jsontree.JsonArray; +import org.hisp.dhis.jsontree.JsonList; +import org.hisp.dhis.jsontree.JsonObject; import org.hisp.dhis.web.HttpStatus; import org.hisp.dhis.webapi.DhisControllerIntegrationTest; import org.hisp.dhis.webapi.json.domain.JsonDataIntegrityDetails; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java index 589828ba93d7..dcee9c2874bd 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityGroupSizeProgramIndicatorGroupControllerTest.java @@ -28,7 +28,12 @@ package org.hisp.dhis.webapi.controller.dataintegrity; import java.util.Set; -import org.hisp.dhis.program.*; +import org.hisp.dhis.program.Program; +import org.hisp.dhis.program.ProgramIndicator; +import org.hisp.dhis.program.ProgramIndicatorGroup; +import org.hisp.dhis.program.ProgramIndicatorService; +import org.hisp.dhis.program.ProgramService; +import org.hisp.dhis.program.ProgramType; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java index 0a343727701e..6ebd9ff3f62f 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataintegrity/DataIntegrityPeriodsSameStartEndDateControllerTest.java @@ -29,7 +29,10 @@ import java.time.ZonedDateTime; import java.util.Date; -import org.hisp.dhis.period.*; +import org.hisp.dhis.period.MonthlyPeriodType; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AppController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AppController.java index 11255dc1b264..aec9c68819ad 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AppController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/AppController.java @@ -43,7 +43,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; -import org.hisp.dhis.appmanager.*; +import org.hisp.dhis.appmanager.App; +import org.hisp.dhis.appmanager.AppManager; +import org.hisp.dhis.appmanager.AppMenuManager; +import org.hisp.dhis.appmanager.AppStatus; +import org.hisp.dhis.appmanager.AppType; import org.hisp.dhis.appmanager.webmodules.WebModule; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.OpenApi; @@ -68,7 +72,15 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.core.context.SecurityContextImpl; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.context.request.ServletWebRequest; import org.springframework.web.multipart.MultipartFile; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/LocaleController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/LocaleController.java index 219604a07ec7..8b00d2cf3769 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/LocaleController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/LocaleController.java @@ -27,7 +27,9 @@ */ package org.hisp.dhis.webapi.controller; -import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.*; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.conflict; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.created; +import static org.hisp.dhis.dxf2.webmessage.WebMessageUtils.notFound; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import java.util.Comparator; @@ -53,7 +55,14 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; @OpenApi.Tags("ui") @Controller diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/ResourceTableController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/ResourceTableController.java index 4a5046960285..74d6ff8866cf 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/ResourceTableController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/ResourceTableController.java @@ -96,7 +96,8 @@ public WebMessage analytics( // STILL EXPERIMENTAL: to export TEIs, FE needs to set this to "false" explicitly @RequestParam(defaultValue = "true") Boolean skipTrackedEntities, @RequestParam(defaultValue = "false") Boolean skipOrgUnitOwnership, - @RequestParam(required = false) Integer lastYears) + @RequestParam(required = false) Integer lastYears, + @RequestParam(defaultValue = "false") Boolean skipOutliers) throws ConflictException, @OpenApi.Ignore NotFoundException { Set skipTableTypes = new HashSet<>(); Set skipPrograms = new HashSet<>(); @@ -128,7 +129,8 @@ public WebMessage analytics( JobConfiguration config = new JobConfiguration(ANALYTICS_TABLE); config.setExecutedBy(currentUserService.getCurrentUser().getUid()); config.setJobParameters( - new AnalyticsJobParameters(lastYears, skipTableTypes, skipPrograms, skipResourceTables)); + new AnalyticsJobParameters( + lastYears, skipTableTypes, skipPrograms, skipResourceTables, skipOutliers)); return execute(config); } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SvgConversionController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SvgConversionController.java index d22c4e13814f..dccb80b8906d 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SvgConversionController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SvgConversionController.java @@ -30,7 +30,7 @@ import static org.hisp.dhis.system.util.GeoUtils.replaceUnsafeSvgText; import static org.hisp.dhis.system.util.SvgUtils.replaceUnicodeZeroWidthSpace; -import java.awt.*; +import java.awt.Color; import java.io.OutputStream; import java.io.StringReader; import javax.servlet.http.HttpServletResponse; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java index 31f7a2e244b7..7c4b8bcfd81b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java @@ -233,13 +233,14 @@ public ResponseEntity getTaskSummaryJson( @RequestParam(defaultValue = "*") List fields, HttpServletRequest request, HttpServletResponse response) { - SystemInfo info = systemService.getSystemInfo(); - - info.setContextPath(ContextUtils.getContextPath(request)); - info.setUserAgent(request.getHeader(ContextUtils.HEADER_USER_AGENT)); + SystemInfo info = + systemService.getSystemInfo().toBuilder() + .contextPath(ContextUtils.getContextPath(request)) + .userAgent(request.getHeader(ContextUtils.HEADER_USER_AGENT)) + .build(); if (!currentUserService.currentUserIsSuper()) { - info.clearSensitiveInfo(); + info = info.withoutSensitiveInfo(); } setNoStore(response); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramIndicatorController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramIndicatorController.java index f86c5a52a93f..9c1a8b0a2f18 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramIndicatorController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/event/ProgramIndicatorController.java @@ -27,7 +27,8 @@ */ package org.hisp.dhis.webapi.controller.event; -import static org.springframework.http.MediaType.*; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/metadata/MetadataImportJob.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/metadata/MetadataImportJob.java index 4af6ddd09ca5..1eeb25641f41 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/metadata/MetadataImportJob.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/metadata/MetadataImportJob.java @@ -103,12 +103,12 @@ public void execute(JobConfiguration config, JobProgress progress) { if (report.hasErrorReports()) { report.forEachErrorReport( - r -> + e -> progress.addError( - r.getErrorCode(), - r.getMainId(), - r.getMainKlass().getSimpleName(), - r.getArgs())); + e.getErrorCode(), + e.getMainId(), + e.getMainKlass().getSimpleName(), + e.getArgs())); } notifier.addJobSummary(config, report, ImportReport.class); diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/AnalyticsOutlierDetectionController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/AnalyticsOutlierDetectionController.java new file mode 100644 index 000000000000..fa858238eee2 --- /dev/null +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/AnalyticsOutlierDetectionController.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2004-2023, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.webapi.controller.outlierdetection; + +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_CSV; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_EXCEL; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_HTML; +import static org.hisp.dhis.webapi.utils.ContextUtils.CONTENT_TYPE_XML; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +import java.io.IOException; +import javax.servlet.http.HttpServletResponse; +import lombok.AllArgsConstructor; +import org.hisp.dhis.common.DhisApiVersion; +import org.hisp.dhis.common.Grid; +import org.hisp.dhis.common.OpenApi; +import org.hisp.dhis.common.cache.CacheStrategy; +import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; +import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; +import org.hisp.dhis.outlierdetection.parser.OutlierDetectionQueryParser; +import org.hisp.dhis.outlierdetection.service.AnalyticsOutlierDetectionService; +import org.hisp.dhis.validation.outlierdetection.ValidationOutlierDetectionRequest; +import org.hisp.dhis.webapi.mvc.annotation.ApiVersion; +import org.hisp.dhis.webapi.utils.ContextUtils; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Analytics Outlier detection API controller. + * + * @author Dusan Bernat + */ +@OpenApi.Tags("analytics") +@RestController +@AllArgsConstructor +@ApiVersion({DhisApiVersion.DEFAULT, DhisApiVersion.ALL}) +@PreAuthorize("hasRole('ALL') or hasRole('F_RUN_VALIDATION')") +public class AnalyticsOutlierDetectionController { + private static final String RESOURCE_PATH = "/analytics/outlierDetection"; + private final AnalyticsOutlierDetectionService outlierService; + private final ContextUtils contextUtils; + private final OutlierDetectionQueryParser queryParser; + private final ValidationOutlierDetectionRequest validator; + + @GetMapping(value = RESOURCE_PATH, produces = APPLICATION_JSON_VALUE) + public Grid getOutliersJson(OutlierDetectionQuery query) { + OutlierDetectionRequest request = getFromQuery(query); + + return outlierService.getOutlierValues(request); + } + + @GetMapping(value = RESOURCE_PATH + ".csv") + public void getOutliersCsv(OutlierDetectionQuery query, HttpServletResponse response) + throws IOException { + OutlierDetectionRequest request = getFromQuery(query); + contextUtils.configureResponse( + response, CONTENT_TYPE_CSV, CacheStrategy.NO_CACHE, "outlierdata.csv", true); + + outlierService.getOutlierValuesAsCsv(request, response.getWriter()); + } + + @GetMapping(value = RESOURCE_PATH + ".xml") + public void getOutliersXml(OutlierDetectionQuery query, HttpServletResponse response) + throws IOException { + OutlierDetectionRequest request = getFromQuery(query); + contextUtils.configureResponse(response, CONTENT_TYPE_XML, CacheStrategy.NO_CACHE); + + outlierService.getOutlierValuesAsXml(request, response.getOutputStream()); + } + + @GetMapping(value = RESOURCE_PATH + ".xls") + public void getOutliersXls(OutlierDetectionQuery query, HttpServletResponse response) + throws IOException { + OutlierDetectionRequest request = getFromQuery(query); + contextUtils.configureResponse( + response, CONTENT_TYPE_EXCEL, CacheStrategy.NO_CACHE, "outlierdata.xls", true); + + outlierService.getOutlierValuesAsXls(request, response.getOutputStream()); + } + + @GetMapping(value = RESOURCE_PATH + ".html") + public void getOutliersHtml(OutlierDetectionQuery query, HttpServletResponse response) + throws IOException { + OutlierDetectionRequest request = getFromQuery(query); + + contextUtils.configureResponse(response, CONTENT_TYPE_HTML, CacheStrategy.NO_CACHE); + + outlierService.getOutlierValuesAsHtml(request, response.getWriter()); + } + + @GetMapping(value = RESOURCE_PATH + ".html+css") + public void getOutliersHtmlCss(OutlierDetectionQuery query, HttpServletResponse response) + throws IOException { + OutlierDetectionRequest request = getFromQuery(query); + contextUtils.configureResponse(response, CONTENT_TYPE_HTML, CacheStrategy.NO_CACHE); + + outlierService.getOutlierValuesAsHtmlCss(request, response.getWriter()); + } + + private OutlierDetectionRequest getFromQuery(OutlierDetectionQuery query) { + OutlierDetectionRequest request = queryParser.getFromQuery(query); + validator.validate(request, true); + + return request; + } +} diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/OutlierDetectionController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/OutlierDetectionController.java index 0115d57fd09a..f7c755aa9a45 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/OutlierDetectionController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/outlierdetection/OutlierDetectionController.java @@ -39,11 +39,14 @@ import org.hisp.dhis.outlierdetection.OutlierDetectionQuery; import org.hisp.dhis.outlierdetection.OutlierDetectionRequest; import org.hisp.dhis.outlierdetection.OutlierDetectionResponse; -import org.hisp.dhis.outlierdetection.OutlierDetectionService; +import org.hisp.dhis.outlierdetection.parser.OutlierDetectionQueryParser; +import org.hisp.dhis.outlierdetection.service.DefaultOutlierDetectionService; +import org.hisp.dhis.validation.outlierdetection.ValidationOutlierDetectionRequest; import org.hisp.dhis.webapi.mvc.annotation.ApiVersion; import org.hisp.dhis.webapi.utils.ContextUtils; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; /** @@ -57,25 +60,33 @@ @ApiVersion({DhisApiVersion.DEFAULT, DhisApiVersion.ALL}) @PreAuthorize("hasRole('ALL') or hasRole('F_RUN_VALIDATION')") public class OutlierDetectionController { - private final OutlierDetectionService outlierService; + private final DefaultOutlierDetectionService outlierService; private final ContextUtils contextUtils; + private final OutlierDetectionQueryParser queryParser; + private final ValidationOutlierDetectionRequest validator; @GetMapping(value = "/outlierDetection", produces = APPLICATION_JSON_VALUE) - public OutlierDetectionResponse getOutliersJson(OutlierDetectionQuery query) { - OutlierDetectionRequest request = outlierService.getFromQuery(query); + public @ResponseBody OutlierDetectionResponse getOutliersJson(OutlierDetectionQuery query) { + OutlierDetectionRequest request = getFromQuery(query); return outlierService.getOutlierValues(request); } - @GetMapping(value = "/outlierDetection", produces = CONTENT_TYPE_CSV) + @GetMapping(value = "/outlierDetection.csv") public void getOutliersCsv(OutlierDetectionQuery query, HttpServletResponse response) throws IOException { - OutlierDetectionRequest request = outlierService.getFromQuery(query); - + OutlierDetectionRequest request = getFromQuery(query); contextUtils.configureResponse( response, CONTENT_TYPE_CSV, CacheStrategy.NO_CACHE, "outlierdata.csv", true); - outlierService.getOutlierValuesAsCsv(request, response.getOutputStream()); + outlierService.getOutlierValuesAsCsv(request, response.getWriter()); + } + + private OutlierDetectionRequest getFromQuery(OutlierDetectionQuery query) { + OutlierDetectionRequest request = queryParser.getFromQuery(query); + validator.validate(request, false); + + return request; } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java index 11aae6ce13d6..349489e2158c 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/scheduling/JobConfigurationController.java @@ -35,11 +35,18 @@ import org.hisp.dhis.common.IdentifiableObjects; import org.hisp.dhis.common.OpenApi; import org.hisp.dhis.common.UID; -import org.hisp.dhis.feedback.*; +import org.hisp.dhis.feedback.ConflictException; +import org.hisp.dhis.feedback.ForbiddenException; +import org.hisp.dhis.feedback.NotFoundException; +import org.hisp.dhis.feedback.ObjectReport; import org.hisp.dhis.jsontree.JsonMixed; import org.hisp.dhis.jsontree.JsonObject; -import org.hisp.dhis.scheduling.*; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobConfigurationService; +import org.hisp.dhis.scheduling.JobProgress; import org.hisp.dhis.scheduling.JobProgress.Progress; +import org.hisp.dhis.scheduling.JobRunErrorsParams; +import org.hisp.dhis.scheduling.JobSchedulerService; import org.hisp.dhis.schema.Property; import org.hisp.dhis.schema.descriptors.JobConfigurationSchemaDescriptor; import org.hisp.dhis.user.CurrentUser; @@ -71,6 +78,7 @@ public class JobConfigurationController extends AbstractCrudController getJobRunErrors(JobRunErrorsParams params) { return jobConfigurationService.findJobRunErrors(params); @@ -78,16 +86,13 @@ public List getJobRunErrors(JobRunErrorsParams params) { @GetMapping("{uid}/errors") public JsonObject getJobRunErrors( - @PathVariable("uid") @OpenApi.Param({UID.class, JobConfiguration.class}) UID uid) - throws NotFoundException { + @PathVariable("uid") @OpenApi.Param({UID.class, JobConfiguration.class}) UID uid, + @CurrentUser User currentUser) + throws NotFoundException, ForbiddenException { + checkExecutingUserOrAdmin(uid, currentUser); List errors = jobConfigurationService.findJobRunErrors(new JobRunErrorsParams().setJob(uid)); - if (errors.isEmpty()) { - JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid.getValue()); - if (obj == null) throw new NotFoundException(JobConfiguration.class, uid.getValue()); - return JsonMixed.of("{}"); - } - return errors.get(0); + return errors.isEmpty() ? JsonMixed.of("{}") : errors.get(0); } @GetMapping("/due") @@ -128,25 +133,19 @@ public ObjectReport executeNow(@PathVariable("uid") String uid) @PostMapping("{uid}/cancel") @ResponseStatus(HttpStatus.NO_CONTENT) - public void cancelExecution(@PathVariable("uid") String uid, @CurrentUser User currentUser) + public void cancelExecution(@PathVariable("uid") UID uid, @CurrentUser User currentUser) throws NotFoundException, ForbiddenException { - JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid); - if (obj == null) throw new NotFoundException(JobConfiguration.class, uid); - boolean canCancel = - currentUser.isSuper() - || currentUser.isAuthorized("F_PERFORM_MAINTENANCE") - || currentUser.getUid().equals(obj.getExecutedBy()); - if (!canCancel) throw new ForbiddenException(JobConfiguration.class, obj.getUid()); - jobSchedulerService.requestCancel(uid); + checkExecutingUserOrAdmin(uid, currentUser); + jobSchedulerService.requestCancel(uid.getValue()); } - @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @GetMapping("{uid}/progress") - public Progress getProgress(@PathVariable("uid") String uid) { - return jobSchedulerService.getProgress(uid); + public Progress getProgress(@PathVariable("uid") UID uid, @CurrentUser User currentUser) + throws ForbiddenException, NotFoundException { + checkExecutingUserOrAdmin(uid, currentUser); + return jobSchedulerService.getProgress(uid.getValue()); } - @PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')") @GetMapping("{uid}/progress/errors") public List getErrors(@PathVariable("uid") String uid) { return jobSchedulerService.getErrors(uid); @@ -161,9 +160,9 @@ public void deleteDoneJobs(@RequestParam int minutes) { @PostMapping("{uid}/enable") @ResponseStatus(HttpStatus.NO_CONTENT) - public void enable(@PathVariable("uid") String uid) throws NotFoundException, ConflictException { - JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid); - if (obj == null) throw new NotFoundException(JobConfiguration.class, uid); + public void enable(@PathVariable("uid") UID uid) throws NotFoundException, ConflictException { + JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid.getValue()); + if (obj == null) throw new NotFoundException(JobConfiguration.class, uid.getValue()); checkModifiable(obj, "Job %s is a system job that cannot be modified."); if (!obj.isEnabled()) { obj.setEnabled(true); @@ -173,9 +172,9 @@ public void enable(@PathVariable("uid") String uid) throws NotFoundException, Co @PostMapping("{uid}/disable") @ResponseStatus(HttpStatus.NO_CONTENT) - public void disable(@PathVariable("uid") String uid) throws NotFoundException, ConflictException { - JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid); - if (obj == null) throw new NotFoundException(JobConfiguration.class, uid); + public void disable(@PathVariable("uid") UID uid) throws NotFoundException, ConflictException { + JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid.getValue()); + if (obj == null) throw new NotFoundException(JobConfiguration.class, uid.getValue()); checkModifiable(obj, "Job %s is a system job that cannot be modified."); if (obj.isEnabled()) { obj.setEnabled(false); @@ -216,4 +215,15 @@ private void checkModifiable(JobConfiguration configuration, String message) throw new ConflictException(String.format(message, identifier)); } } + + private void checkExecutingUserOrAdmin(UID uid, User currentUser) + throws NotFoundException, ForbiddenException { + JobConfiguration obj = jobConfigurationService.getJobConfigurationByUid(uid.getValue()); + if (obj == null) throw new NotFoundException(JobConfiguration.class, uid.getValue()); + boolean canCancel = + currentUser.isSuper() + || currentUser.isAuthorized("F_PERFORM_MAINTENANCE") + || currentUser.getUid().equals(obj.getExecutedBy()); + if (!canCancel) throw new ForbiddenException(JobConfiguration.class, obj.getUid()); + } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java index 6c73596488cb..1e8566f6c14f 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventMapper.java @@ -64,6 +64,7 @@ public interface EventMapper entry("completedAt", "completedDate"), entry("completedBy", "completedBy"), entry("createdAt", "created"), + entry("createdAtClient", "createdAtClient"), entry("createdBy", "createdBy"), entry("deleted", "deleted"), entry("enrolledAt", "enrollment.enrollmentDate"), @@ -81,6 +82,7 @@ public interface EventMapper entry("storedBy", "storedBy"), entry("trackedEntity", "enrollment.trackedEntity.uid"), entry("updatedAt", "lastUpdated"), + entry("updatedAtClient", "lastUpdatedAtClient"), entry("updatedBy", "lastUpdatedBy")); @Mapping(target = "event", source = "uid") diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/RequestParams.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/ImportRequestParams.java similarity index 99% rename from dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/RequestParams.java rename to dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/ImportRequestParams.java index 216368e1a7be..c47b83697852 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/RequestParams.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/ImportRequestParams.java @@ -47,7 +47,7 @@ @NoArgsConstructor @AllArgsConstructor @Builder -class RequestParams { +public class ImportRequestParams { /** Should import be imported or just validated. */ @JsonProperty @Builder.Default private TrackerBundleMode importMode = TrackerBundleMode.COMMIT; diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java index e882355a4df9..36f503c25302 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java @@ -32,11 +32,10 @@ import static org.hisp.dhis.webapi.utils.ContextUtils.setNoStore; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.ObjectOutputStream; import java.util.Deque; import java.util.List; import java.util.Optional; @@ -102,16 +101,18 @@ public class TrackerImportController { private final JobConfigurationService jobConfigurationService; + private final ObjectMapper jsonMapper; + @PostMapping(value = "", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @ResponseBody public WebMessage asyncPostJsonTracker( HttpServletRequest request, - RequestParams requestParams, + ImportRequestParams importRequestParams, @CurrentUser User currentUser, @RequestBody Body body) throws ConflictException, NotFoundException, IOException { TrackerImportParams trackerImportParams = - TrackerImportParamsMapper.trackerImportParams(currentUser.getUid(), requestParams); + TrackerImportParamsMapper.trackerImportParams(currentUser.getUid(), importRequestParams); TrackerObjects trackerObjects = TrackerImportParamsMapper.trackerObjects(body, trackerImportParams.getIdSchemes()); @@ -133,17 +134,11 @@ private WebMessage startAsyncTracker( JobConfiguration config = new JobConfiguration(JobType.TRACKER_IMPORT_JOB); config.setExecutedBy(user.getUid()); config.setJobParameters(params); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - - oos.writeObject(trackerObjects); - oos.flush(); - oos.close(); + byte[] jsonInput = jsonMapper.writeValueAsBytes(trackerObjects); - InputStream is = new ByteArrayInputStream(baos.toByteArray()); - - jobSchedulerService.executeNow(jobConfigurationService.create(config, contentType, is)); + jobSchedulerService.executeNow( + jobConfigurationService.create(config, contentType, new ByteArrayInputStream(jsonInput))); String jobId = config.getUid(); String location = ContextUtils.getRootPath(request) + "/tracker/jobs/" + jobId; return ok(TRACKER_JOB_ADDED) @@ -156,9 +151,11 @@ private WebMessage startAsyncTracker( consumes = APPLICATION_JSON_VALUE, params = {"async=false"}) public ResponseEntity syncPostJsonTracker( - RequestParams requestParams, @CurrentUser User currentUser, @RequestBody Body body) { + ImportRequestParams importRequestParams, + @CurrentUser User currentUser, + @RequestBody Body body) { TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams(currentUser.getUid(), requestParams); + TrackerImportParamsMapper.trackerImportParams(currentUser.getUid(), importRequestParams); TrackerObjects trackerObjects = TrackerImportParamsMapper.trackerObjects(body, params.getIdSchemes()); ImportReport importReport = @@ -180,7 +177,7 @@ public ResponseEntity syncPostJsonTracker( @ResponseBody public WebMessage asyncPostCsvTracker( HttpServletRequest request, - RequestParams importRequest, + ImportRequestParams importRequest, @CurrentUser User currentUser, @RequestParam(required = false, defaultValue = "true") boolean skipFirst) throws IOException, ParseException, ConflictException, NotFoundException { @@ -211,7 +208,7 @@ public WebMessage asyncPostCsvTracker( params = {"async=false"}) public ResponseEntity syncPostCsvTracker( HttpServletRequest request, - RequestParams importRequest, + ImportRequestParams importRequest, @RequestParam(required = false, defaultValue = "true") boolean skipFirst, @RequestParam(defaultValue = "errors", required = false) TrackerBundleReportMode reportMode, @CurrentUser User currentUser) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportJob.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportJob.java index d144c60e0970..2969adee5894 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportJob.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportJob.java @@ -27,9 +27,9 @@ */ package org.hisp.dhis.webapi.controller.tracker.imports; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.io.ObjectInputStream; import java.util.function.Consumer; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -46,6 +46,7 @@ import org.hisp.dhis.tracker.imports.report.ImportReport; import org.hisp.dhis.tracker.imports.report.Stats; import org.hisp.dhis.tracker.imports.report.Status; +import org.hisp.dhis.tracker.imports.validation.ValidationCode; import org.springframework.stereotype.Component; @Slf4j @@ -55,6 +56,7 @@ public class TrackerImportJob implements Job { private final TrackerImportService trackerImportService; private final FileResourceService fileResourceService; private final Notifier notifier; + private final ObjectMapper jsonMapper; @Override public JobType getJobType() { @@ -78,6 +80,20 @@ public void execute(JobConfiguration config, JobProgress progress) { return; } notifier.addJobSummary(config, report, ImportReport.class); + + if (report.getValidationReport().hasErrors()) { + report + .getValidationReport() + .getErrors() + .forEach( + e -> + progress.addError( + ValidationCode.valueOf(e.getErrorCode()), + e.getUid(), + e.getTrackerType(), + e.getArgs())); + } + Stats stats = report.getStats(); Consumer endProcess = report.getStatus() == Status.ERROR ? progress::failedProcess : progress::completedProcess; @@ -94,9 +110,7 @@ public void execute(JobConfiguration config, JobProgress progress) { } } - private TrackerObjects toTrackerObjects(InputStream input) - throws IOException, ClassNotFoundException { - ObjectInputStream ois = new ObjectInputStream(input); - return (TrackerObjects) ois.readObject(); + private TrackerObjects toTrackerObjects(InputStream input) throws IOException { + return jsonMapper.readValue(input, TrackerObjects.class); } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapper.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapper.java index e1de076a1d4b..5cde8caffe1f 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapper.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapper.java @@ -57,7 +57,8 @@ public static TrackerObjects trackerObjects(Body body, TrackerIdSchemeParams idS private TrackerImportParamsMapper() {} - public static TrackerImportParams trackerImportParams(String userId, RequestParams request) { + public static TrackerImportParams trackerImportParams( + String userId, ImportRequestParams request) { TrackerIdSchemeParam defaultIdSchemeParam = request.getIdScheme(); TrackerIdSchemeParams idSchemeParams = TrackerIdSchemeParams.builder() diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/dimension/TeiAnalyticsPrefixStrategy.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/dimension/TeiAnalyticsPrefixStrategy.java index c9a437250768..2b167db63efd 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/dimension/TeiAnalyticsPrefixStrategy.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/dimension/TeiAnalyticsPrefixStrategy.java @@ -29,15 +29,22 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.common.PrefixedDimension; +import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.program.ProgramStageDataElement; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class TeiAnalyticsPrefixStrategy implements PrefixStrategy { - public static final PrefixStrategy INSTANCE = EnrollmentAnalyticsPrefixStrategy.INSTANCE; + public static final PrefixStrategy INSTANCE = new TeiAnalyticsPrefixStrategy(); @Override public String apply(PrefixedDimension pDimension) { - throw new UnsupportedOperationException("This method should not be called for TEI analytics"); + if (pDimension.getItem() instanceof DataElement + || pDimension.getItem() instanceof ProgramStageDataElement) { + return pDimension.getPrefix(); + } + return StringUtils.EMPTY; } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/FileResourceUtils.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/FileResourceUtils.java index 7f55a8707517..91e03eafc38b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/FileResourceUtils.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/utils/FileResourceUtils.java @@ -210,7 +210,7 @@ public FileResource saveFileResource(String uid, MultipartFile file, FileResourc throw new WebMessageException( conflict(ErrorCode.E1119, FileResource.class.getSimpleName(), uid)); } - fileResourceService.saveFileResource(fileResource, tmpFile); + fileResourceService.asyncSaveFileResource(fileResource, tmpFile); return fileResource; } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/analytics/dimension/DimensionFilteringAndPagingServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/analytics/dimension/DimensionFilteringAndPagingServiceTest.java index c9cacebf120c..7881d88eb205 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/analytics/dimension/DimensionFilteringAndPagingServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/analytics/dimension/DimensionFilteringAndPagingServiceTest.java @@ -30,7 +30,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.Collection; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandControllerTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandControllerTest.java index 5ca7c6a1ddef..57dcaba9c997 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandControllerTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/dataelement/DataElementOperandControllerTest.java @@ -72,6 +72,7 @@ import org.hisp.dhis.schema.Schema; import org.hisp.dhis.schema.SchemaService; import org.hisp.dhis.security.acl.AclService; +import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.user.CurrentUserService; import org.hisp.dhis.user.User; import org.hisp.dhis.webapi.mvc.messageconverter.JsonMessageConverter; @@ -109,6 +110,8 @@ class DataElementOperandControllerTest { @Mock private CategoryService dataElementCategoryService; + @Mock private SystemSettingManager systemSettingManager; + private QueryService queryService; @Mock private CurrentUserService currentUserService; @@ -124,7 +127,7 @@ public void setUp() { QueryService _queryService = new DefaultQueryService( new DefaultJpaQueryParser(schemaService), - new DefaultQueryPlanner(schemaService), + new DefaultQueryPlanner(schemaService, systemSettingManager), mock(JpaCriteriaQueryEngine.class), new InMemoryQueryEngine<>(schemaService, mock(AclService.class), currentUserService)); // Use "spy" on queryService, because we want a partial mock: we only diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/ImportRequestParamsValidatorTest.java similarity index 99% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/ImportRequestParamsValidatorTest.java index f6ca0c66d740..f0bd80f5cc68 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/RequestParamsValidatorTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/ImportRequestParamsValidatorTest.java @@ -65,7 +65,7 @@ import org.junit.jupiter.params.provider.ValueSource; /** Tests {@link RequestParamsValidator}. */ -class RequestParamsValidatorTest { +class ImportRequestParamsValidatorTest { private static final String TEA_1_UID = "TvjwTPToKHO"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageRequestParamsTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageImportRequestParamsTest.java similarity index 94% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageRequestParamsTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageImportRequestParamsTest.java index ee81faac4297..e91aa9135d5c 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageRequestParamsTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/PageImportRequestParamsTest.java @@ -27,12 +27,13 @@ */ package org.hisp.dhis.webapi.controller.tracker.export; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import lombok.Data; import org.junit.jupiter.api.Test; -class PageRequestParamsTest { +class PageImportRequestParamsTest { @Data private static class PaginationParameters implements PageRequestParams { diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentImportRequestParamsMapperTest.java similarity index 99% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapperTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentImportRequestParamsMapperTest.java index fb63914d7128..b4864df9f61b 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentRequestParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/enrollment/EnrollmentImportRequestParamsMapperTest.java @@ -62,7 +62,7 @@ @MockitoSettings(strictness = Strictness.LENIENT) // common setup @ExtendWith(MockitoExtension.class) -class EnrollmentRequestParamsMapperTest { +class EnrollmentImportRequestParamsMapperTest { private static final String ORG_UNIT_1_UID = "lW0T2U7gZUi"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventImportRequestParamsMapperTest.java similarity index 99% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventImportRequestParamsMapperTest.java index 47fc14eb58db..63c994fe2da8 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventRequestParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventImportRequestParamsMapperTest.java @@ -89,7 +89,7 @@ @MockitoSettings(strictness = Strictness.LENIENT) // common setup @ExtendWith(MockitoExtension.class) -class EventRequestParamsMapperTest { +class EventImportRequestParamsMapperTest { private static final String DE_1_UID = "OBzmpRP6YUh"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipImportRequestParamsMapperTest.java similarity index 99% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapperTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipImportRequestParamsMapperTest.java index bd6b8e5666cd..c96eec658f96 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipRequestParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/relationship/RelationshipImportRequestParamsMapperTest.java @@ -46,7 +46,7 @@ import org.hisp.dhis.webapi.controller.event.webrequest.OrderCriteria; import org.junit.jupiter.api.Test; -class RelationshipRequestParamsMapperTest { +class RelationshipImportRequestParamsMapperTest { private final RelationshipRequestParamsMapper mapper = new RelationshipRequestParamsMapper(); diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityImportRequestParamsMapperTest.java similarity index 99% rename from dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapperTest.java rename to dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityImportRequestParamsMapperTest.java index b596f2101d62..ce84ae7ffd9e 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityRequestParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackedentity/TrackedEntityImportRequestParamsMapperTest.java @@ -66,7 +66,7 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -class TrackedEntityRequestParamsMapperTest { +class TrackedEntityImportRequestParamsMapperTest { public static final String TEA_1_UID = "TvjwTPToKHO"; public static final String TEA_2_UID = "cy2oRh2sNr6"; diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java index 67610dbe6e05..b0855f0121cb 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java @@ -44,6 +44,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.LinkedList; import org.hisp.dhis.common.CodeGenerator; @@ -111,7 +112,8 @@ public void setUp() { csvEventService, notifier, jobSchedulerService, - jobConfigurationService); + jobConfigurationService, + new ObjectMapper()); mockMvc = MockMvcBuilders.standaloneSetup(controller) diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapperTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapperTest.java index 8cb802fe993d..8ec096c76351 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapperTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportParamsMapperTest.java @@ -61,9 +61,10 @@ void testValidationMode() { Arrays.stream(ValidationMode.values()) .forEach( e -> { - RequestParams requestParams = RequestParams.builder().validationMode(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().validationMode(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat(params.getValidationMode(), is(e)); }); } @@ -73,9 +74,10 @@ void testImportMode() { Arrays.stream(TrackerBundleMode.values()) .forEach( e -> { - RequestParams requestParams = RequestParams.builder().importMode(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().importMode(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat(params.getImportMode(), is(e)); }); } @@ -85,9 +87,10 @@ void testAtomicMode() { Arrays.stream(AtomicMode.values()) .forEach( e -> { - RequestParams requestParams = RequestParams.builder().atomicMode(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().atomicMode(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat(params.getAtomicMode(), is(e)); }); } @@ -97,9 +100,10 @@ void testFlushMode() { Arrays.stream(FlushMode.values()) .forEach( e -> { - RequestParams requestParams = RequestParams.builder().flushMode(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().flushMode(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat(params.getFlushMode(), is(e)); }); } @@ -109,19 +113,20 @@ void testImportStrategy() { Arrays.stream(TrackerImportStrategy.values()) .forEach( e -> { - RequestParams requestParams = RequestParams.builder().importStrategy(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().importStrategy(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat(params.getImportStrategy(), is(e)); }); } @Test void testIdSchemeUsingIdSchemeName() { - RequestParams requestParams = - RequestParams.builder().idScheme(TrackerIdSchemeParam.NAME).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().idScheme(TrackerIdSchemeParam.NAME).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); TrackerIdSchemeParam expected = TrackerIdSchemeParam.NAME; assertEquals(expected, params.getIdSchemes().getIdScheme()); @@ -135,10 +140,12 @@ void testIdSchemeUsingIdSchemeName() { @Test void testIdSchemeUsingIdSchemeAttribute() { - RequestParams requestParams = - RequestParams.builder().idScheme(TrackerIdSchemeParam.ofAttribute("WSiOAALYocA")).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder() + .idScheme(TrackerIdSchemeParam.ofAttribute("WSiOAALYocA")) + .build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); TrackerIdSchemeParam expected = TrackerIdSchemeParam.ofAttribute("WSiOAALYocA"); assertEquals(expected, params.getIdSchemes().getIdScheme()); @@ -155,9 +162,10 @@ void testOrgUnitIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = RequestParams.builder().orgUnitIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().orgUnitIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getOrgUnitIdScheme().getIdScheme(), is(e.getIdScheme())); }); @@ -168,9 +176,10 @@ void testProgramIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = RequestParams.builder().programIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().programIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getProgramIdScheme().getIdScheme(), is(e.getIdScheme())); }); @@ -178,12 +187,12 @@ void testProgramIdentifier() { @Test void testProgramIdentifierUsingIdSchemeAttribute() { - RequestParams requestParams = - RequestParams.builder() + ImportRequestParams importRequestParams = + ImportRequestParams.builder() .programIdScheme(TrackerIdSchemeParam.ofAttribute("WSiOAALYocA")) .build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertEquals( TrackerIdSchemeParam.ofAttribute("WSiOAALYocA"), @@ -195,9 +204,10 @@ void testProgramStageIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = RequestParams.builder().programStageIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().programStageIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getProgramStageIdScheme().getIdScheme(), is(e.getIdScheme())); @@ -209,9 +219,10 @@ void testDataElementIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = RequestParams.builder().dataElementIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().dataElementIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getDataElementIdScheme().getIdScheme(), is(e.getIdScheme())); @@ -223,10 +234,10 @@ void testCategoryOptionComboIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = - RequestParams.builder().categoryOptionComboIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().categoryOptionComboIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getCategoryOptionComboIdScheme().getIdScheme(), is(e.getIdScheme())); @@ -238,10 +249,10 @@ void testCategoryOptionIdentifier() { idSchemeParams() .forEach( e -> { - RequestParams requestParams = - RequestParams.builder().categoryOptionIdScheme(e).build(); + ImportRequestParams importRequestParams = + ImportRequestParams.builder().categoryOptionIdScheme(e).build(); TrackerImportParams params = - TrackerImportParamsMapper.trackerImportParams("userId", requestParams); + TrackerImportParamsMapper.trackerImportParams("userId", importRequestParams); assertThat( params.getIdSchemes().getCategoryOptionIdScheme().getIdScheme(), is(e.getIdScheme())); diff --git a/dhis-2/dhis-web-embedded-jetty/src/main/java/org/hisp/dhis/web/embeddedjetty/EmbeddedJettyBase.java b/dhis-2/dhis-web-embedded-jetty/src/main/java/org/hisp/dhis/web/embeddedjetty/EmbeddedJettyBase.java index ca91769b1cb9..6672fecc4aba 100644 --- a/dhis-2/dhis-web-embedded-jetty/src/main/java/org/hisp/dhis/web/embeddedjetty/EmbeddedJettyBase.java +++ b/dhis-2/dhis-web-embedded-jetty/src/main/java/org/hisp/dhis/web/embeddedjetty/EmbeddedJettyBase.java @@ -39,7 +39,12 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.jetty.rewrite.handler.RedirectPatternRule; import org.eclipse.jetty.rewrite.handler.RewriteHandler; -import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; diff --git a/dhis-2/dhis-web/dhis-web-dataentry/src/main/resources/org/hisp/dhis/de/i18n_module_uz_UZ_Cyrl.properties b/dhis-2/dhis-web/dhis-web-dataentry/src/main/resources/org/hisp/dhis/de/i18n_module_uz_UZ_Cyrl.properties index 47cb11b920c0..6d85fce0996a 100644 --- a/dhis-2/dhis-web/dhis-web-dataentry/src/main/resources/org/hisp/dhis/de/i18n_module_uz_UZ_Cyrl.properties +++ b/dhis-2/dhis-web/dhis-web-dataentry/src/main/resources/org/hisp/dhis/de/i18n_module_uz_UZ_Cyrl.properties @@ -52,9 +52,9 @@ operator=\u041e\u043f\u0435\u0440\u0430\u0442\u043e\u0440 no_organisationunit_selected=\u0422\u0430\u0448\u043a\u0438\u043b\u0438 no_period_selected=\u0414\u0430\u0432\u0440 \u0442\u0430\u043d\u043b\u0430\u043d\u043c\u0430\u0434\u0438 no_dataelement_selected=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u044d\u043b\u0435\u043c\u0435\u043d\u0442\u043b\u0430\u0440 \u0442\u0430\u043d\u043b\u0430\u043d\u043c\u0430\u0434\u0438 -dataset_is_approved=Data has been approved and cannot be modified -dataset_is_closed=Data input period is closed -dataset_is_concluded=Data entry has concluded +dataset_is_approved=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0430\u0432\u0432\u0430\u043b \u0442\u0430\u0441\u0434\u0438\u049b\u043b\u0430\u043d\u0433\u0430\u043d \u0432\u0430 \u045e\u0437\u0433\u0430\u0440\u0442\u0438\u0440\u0438\u0448 \u0438\u043c\u043a\u043e\u043d\u0441\u0438\u0437 +dataset_is_closed=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u043a\u0438\u0440\u0438\u0442\u0438\u0448 \u0434\u0430\u0432\u0440\u0438 \u0451\u043f\u0438\u043b\u0433\u0430\u043d +dataset_is_concluded=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440\u043d\u0438 \u043a\u0438\u0440\u0438\u0442\u0438\u0448 \u044f\u043a\u0443\u043d\u0438\u0433\u0430 \u0435\u0442\u0433\u0430\u043d dataset_is_locked=\u041c\u0430\u044a\u043b\u0443\u043c\u043e\u0442\u043b\u0430\u0440 \u0442\u045e\u043f\u043b\u0430\u043c\u0438 \u0431\u043b\u043e\u043a\u043b\u0430\u043d\u0433\u0430\u043d complete=\u0422\u045e\u043b\u0434\u0438\u0440\u0438\u0448 incomplete=\u0422\u0443\u0433\u0430\u0442\u0438\u043b\u043c\u0430\u0433\u0430\u043d diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 8de8794e8348..8d9d2f60db69 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -104,12 +104,12 @@ 3.5.2 - 5.3.30 - 2.7.17 + 5.3.31 + 2.7.18 2.7.4 1.1.5.RELEASE 1.3.4 - 2.0.7.RELEASE + 2.0.8.RELEASE 2.4.1 1.6.0.Final 6.3.0.RELEASE @@ -121,7 +121,7 @@ 0.9.5.5 5.1.0 - 42.6.0 + 42.7.0 2.5.1 1.9 @@ -166,7 +166,7 @@ 2.31.2 4.1.101.Final - 4.8.164 + 4.8.165 0.2.1.1 @@ -206,7 +206,7 @@ 1.12.0 - 2.21.1 + 2.22.0 1.7.36 @@ -215,10 +215,10 @@ 2.0.9 2.2 2.0.8 - 1.19.2 + 1.19.3 1.5.1 4.2.0 - 2.1.18 + 2.1.19 5.0.0 0.2.5 2.2.224 @@ -234,8 +234,8 @@ 3.4.1 3.5.0 2.4.0 - 2.16.1 - 8.4.2 + 2.16.2 + 8.4.3 4.8.1.0 2.40.0 0.8.11 diff --git a/jenkinsfiles/stable b/jenkinsfiles/stable index b5ca7d8c9cc5..c05bf145a719 100644 --- a/jenkinsfiles/stable +++ b/jenkinsfiles/stable @@ -1,5 +1,22 @@ #!/usr/bin/env groovy +/* +# This pipeline can be tested by executing the following steps +* Create a branch with a non-existing version name such as "patch/4.1.2.3" +* Update all the pom files with the same version (4.1.2.3) +* Commit the pom files +* Tag the commit from above with "4.1.2.3" +* Push +* Update the "Filter by name (with regular expression)" field under "Branch Sources" (here)[https://ci.dhis2.org/view/dhis2-core/job/dhis2-core-stable/configure] to include your version. + An example of such regular expression could be: ^2[.]\d+[.]\d+$|^2[.]\d+[.]\d+-(?i)embargoed$|^patch\/2[.]\d+[.]\d+$|^patch\/(2|4)[.]\d+[.]\d+[.]\d+$|^2[.]\d+[.]\d+[.]\d+$ +* Schedule a build for your branch here... https://ci.dhis2.org/view/dhis2-core/job/dhis2-core-stable/ + +# Clean up +* Restore the regular expression previously updated to include your branch +* Delete the war file from S3: aws s3 rm s3://releases.dhis2.org/... +* Delete the docker image from the repository +*/ + @Library('pipeline-library') _ pipeline {