diff --git a/dqops/src/main/java/com/dqops/core/scheduler/JobSchedulerServiceImpl.java b/dqops/src/main/java/com/dqops/core/scheduler/JobSchedulerServiceImpl.java index 2321768027..6adfd0aa55 100644 --- a/dqops/src/main/java/com/dqops/core/scheduler/JobSchedulerServiceImpl.java +++ b/dqops/src/main/java/com/dqops/core/scheduler/JobSchedulerServiceImpl.java @@ -582,7 +582,8 @@ public void applyScheduleDeltaToJob(JobSchedulesDelta schedulesDelta, JobKey job try { Trigger triggerToAdd = this.triggerFactory.createTrigger(scheduleToAdd, jobKey, dataDomainName); - if (!this.scheduler.checkExists(triggerToAdd.getKey())) { + TriggerKey triggerKey = triggerToAdd.getKey(); + if (!this.scheduler.checkExists(triggerKey)) { this.scheduler.scheduleJob(triggerToAdd); } } diff --git a/dqops/src/main/java/com/dqops/core/scheduler/quartz/TriggerFactoryImpl.java b/dqops/src/main/java/com/dqops/core/scheduler/quartz/TriggerFactoryImpl.java index 811add6785..767e39bcce 100644 --- a/dqops/src/main/java/com/dqops/core/scheduler/quartz/TriggerFactoryImpl.java +++ b/dqops/src/main/java/com/dqops/core/scheduler/quartz/TriggerFactoryImpl.java @@ -87,8 +87,9 @@ public Trigger createTrigger(CronScheduleSpec schedule, JobKey jobKey, String da jobDataMapAdapter.setSchedule(triggerJobData, schedule); jobDataMapAdapter.setDataDomain(triggerJobData, dataDomainName); + String key = "job=" + jobKey.getName() + ",cron=" + schedule.getCronExpression() + ",domain=" + dataDomainName; TriggerBuilder triggerBuilder = newTrigger() - .withIdentity(schedule.toString() + "/" + dataDomainName) + .withIdentity(key) .usingJobData(triggerJobData); ScheduleBuilder scheduleBuilder = null; diff --git a/dqops/src/main/java/com/dqops/execution/statistics/StatisticsCollectorsExecutionServiceImpl.java b/dqops/src/main/java/com/dqops/execution/statistics/StatisticsCollectorsExecutionServiceImpl.java index d05124a0aa..89797dc216 100644 --- a/dqops/src/main/java/com/dqops/execution/statistics/StatisticsCollectorsExecutionServiceImpl.java +++ b/dqops/src/main/java/com/dqops/execution/statistics/StatisticsCollectorsExecutionServiceImpl.java @@ -200,7 +200,7 @@ public StatisticsCollectionExecutionSummary executeStatisticsCollectorsOnTable(E * @return Collection of table wrappers. */ public Collection listTargetTables(UserHome userHome, StatisticsCollectorSearchFilters statisticsCollectorSearchFilters) { - Collection tables = this.hierarchyNodeTreeSearcher.findTables(userHome.getConnections(), statisticsCollectorSearchFilters); + Collection tables = this.hierarchyNodeTreeSearcher.findTablesForStatisticsCollection(userHome.getConnections(), statisticsCollectorSearchFilters); return tables; } } diff --git a/dqops/src/main/java/com/dqops/execution/statistics/TableStatisticsCollectorsExecutionServiceImpl.java b/dqops/src/main/java/com/dqops/execution/statistics/TableStatisticsCollectorsExecutionServiceImpl.java index d908c4fa8d..171e31c279 100644 --- a/dqops/src/main/java/com/dqops/execution/statistics/TableStatisticsCollectorsExecutionServiceImpl.java +++ b/dqops/src/main/java/com/dqops/execution/statistics/TableStatisticsCollectorsExecutionServiceImpl.java @@ -159,7 +159,7 @@ public StatisticsCollectionExecutionSummary executeCollectorsOnTable(ExecutionCo StatisticsCollectionExecutionSummary statisticsCollectionExecutionSummary = new StatisticsCollectionExecutionSummary(); CollectorExecutionStatistics executionStatistics = new CollectorExecutionStatistics(); - Collection> collectors = this.hierarchyNodeTreeSearcher.findStatisticsCollectors(targetTable, statisticsCollectorSearchFilters); + Collection> collectors = this.hierarchyNodeTreeSearcher.findStatisticsCollectors(targetTable.getSpec(), statisticsCollectorSearchFilters); if (collectors.size() == 0) { statisticsCollectionExecutionSummary.reportTableStats(connectionWrapper, targetTable.getSpec(), executionStatistics); return statisticsCollectionExecutionSummary; // no checks for this table diff --git a/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcher.java b/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcher.java index 3e1ca63fd9..6749c15245 100644 --- a/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcher.java +++ b/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcher.java @@ -55,7 +55,7 @@ public interface HierarchyNodeTreeSearcher { * @param statisticsCollectorSearchFilters Search filters. * @return Collection of tables that passed the filter and will be analyzed to collect statistics. */ - Collection findTablesForStatisticsCollection(HierarchyNode startNode, StatisticsCollectorSearchFilters statisticsCollectorSearchFilters); + Collection findTablesForStatisticsCollection(HierarchyNode startNode, StatisticsCollectorSearchFilters statisticsCollectorSearchFilters); /** * Search for connection in the tree. diff --git a/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcherImpl.java b/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcherImpl.java index 0cf5835066..cee3632699 100644 --- a/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcherImpl.java +++ b/dqops/src/main/java/com/dqops/metadata/search/HierarchyNodeTreeSearcherImpl.java @@ -98,8 +98,8 @@ public Collection> findStatisticsCollectors(H * @return Collection of tables that passed the filter and will be analyzed to collect statistics. */ @Override - public Collection findTablesForStatisticsCollection(HierarchyNode startNode, StatisticsCollectorSearchFilters statisticsCollectorSearchFilters) { - StatisticsCollectorTableSearchFiltersVisitor searchFilterVisitor = statisticsCollectorSearchFilters.createTableCollectorSearchFilterVisitor(); + public Collection findTablesForStatisticsCollection(HierarchyNode startNode, StatisticsCollectorSearchFilters statisticsCollectorSearchFilters) { + StatisticsCollectorTargetTableSearchFiltersVisitor searchFilterVisitor = statisticsCollectorSearchFilters.createTargetTableCollectorSearchFilterVisitor(); ArrayList matchingNodes = new ArrayList<>(); LabelsSearcherObject labelsSearcherObject = new LabelsSearcherObject(); DataGroupingConfigurationSearcherObject dataGroupingConfigurationSearcherObject = new DataGroupingConfigurationSearcherObject(); @@ -107,7 +107,7 @@ public Collection findTablesForStatisticsCollection(HierarchyNode sta this.hierarchyNodeTreeWalker.traverseHierarchyNodeTree(startNode, node -> node.visit(searchFilterVisitor, searchParameterObject)); - return (List)(ArrayList)matchingNodes; + return (List)(ArrayList)matchingNodes; } /** diff --git a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFilters.java b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFilters.java index 693bc15ad8..114868fd2d 100644 --- a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFilters.java +++ b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFilters.java @@ -85,8 +85,8 @@ public StatisticsCollectorSearchFiltersVisitor createCollectorSearchFilterVisito * Create a hierarchy tree node traversal visitor that will search for tables matching the filters. * @return Search visitor. */ - public StatisticsCollectorTableSearchFiltersVisitor createTableCollectorSearchFilterVisitor() { - return new StatisticsCollectorTableSearchFiltersVisitor(this); + public StatisticsCollectorTargetTableSearchFiltersVisitor createTargetTableCollectorSearchFilterVisitor() { + return new StatisticsCollectorTargetTableSearchFiltersVisitor(this); } /** diff --git a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFiltersVisitor.java b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFiltersVisitor.java index 0a7d226009..4b2396ab8f 100644 --- a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFiltersVisitor.java +++ b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorSearchFiltersVisitor.java @@ -18,6 +18,8 @@ import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; import com.dqops.metadata.id.HierarchyId; import com.dqops.metadata.labels.LabelSetSpec; +import com.dqops.metadata.scheduling.CronScheduleSpec; +import com.dqops.metadata.scheduling.CronSchedulesSpec; import com.dqops.metadata.sources.*; import com.dqops.metadata.traversal.TreeNodeTraversalResult; import com.dqops.sensors.AbstractSensorParametersSpec; @@ -36,13 +38,174 @@ /** * Visitor for {@link StatisticsCollectorSearchFilters} that finds statistics collectors to execute. */ -public class StatisticsCollectorSearchFiltersVisitor extends StatisticsCollectorTableSearchFiltersVisitor { +public class StatisticsCollectorSearchFiltersVisitor extends AbstractSearchVisitor { + private final StatisticsCollectorSearchFilters filters; + /** * Creates a visitor for the given filters. * @param filters Check search filters. */ public StatisticsCollectorSearchFiltersVisitor(StatisticsCollectorSearchFilters filters) { - super(filters); + this.filters = filters; + } + + /** + * Accepts a list of connections. + * + * @param connectionList List of connections. + * @param parameter Target object where found hierarchy nodes, dimensions and labels should be added. + * @return Accept's result. + */ + @Override + public TreeNodeTraversalResult accept(ConnectionList connectionList, SearchParameterObject parameter) { + String connectionNameFilter = this.filters.getConnection(); + if (Strings.isNullOrEmpty(connectionNameFilter)) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; + } + + if (StringPatternComparer.isSearchPattern(connectionNameFilter)) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; // we need to iterate anyway + } + + // exact connection name given, let's find it + ConnectionWrapper connectionWrapper = connectionList.getByObjectName(connectionNameFilter, true); + if (connectionWrapper == null) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; // another try, maybe the name is case-sensitive + } + + return TreeNodeTraversalResult.traverseSelectedChildNodes(connectionWrapper); + } + + /** + * Accepts a connection wrapper (lazy loader). + * + * @param connectionWrapper Connection wrapper. + * @param parameter Target object where found hierarchy nodes, dimensions and labels should be added. + * @return Accept's result. + */ + @Override + public TreeNodeTraversalResult accept(ConnectionWrapper connectionWrapper, SearchParameterObject parameter) { + if (!Strings.isNullOrEmpty(this.filters.getEnabledCronScheduleExpression())) { + CronSchedulesSpec schedules = connectionWrapper.getSpec().getSchedules(); + CronScheduleSpec profilingSchedule = schedules != null ? schedules.getProfiling() : null; + this.filters.setIgnoreTablesWithoutSchedule( + profilingSchedule == null || + profilingSchedule.isDisabled() || + !Objects.equals(profilingSchedule.getCronExpression(), this.filters.getEnabledCronScheduleExpression())); + } + + String connectionNameFilter = this.filters.getConnection(); + LabelsSearcherObject labelsSearcherObject = parameter.getLabelsSearcherObject(); + labelsSearcherObject.setConnectionLabels(connectionWrapper.getSpec().getLabels()); + + if (Strings.isNullOrEmpty(connectionNameFilter)) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; + } + + if (StringPatternComparer.matchSearchPattern(connectionWrapper.getName(), connectionNameFilter)) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; + } + + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + + /** + * Accepts a collection of tables inside a connection. + * + * @param tableList Table list. + * @param parameter Target object where found hierarchy nodes, dimensions and labels should be added. + * @return Accept's result. + */ + @Override + public TreeNodeTraversalResult accept(TableList tableList, SearchParameterObject parameter) { + String schemaTableName = this.filters.getFullTableName(); + if (Strings.isNullOrEmpty(schemaTableName)) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; + } + + PhysicalTableName physicalTableName = PhysicalTableName.fromSchemaTableFilter(schemaTableName); + if (physicalTableName.isSearchPattern()) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; // we need to iterate anyway + } + + TableWrapper tableWrapper = tableList.getByObjectName(physicalTableName, true); + if (tableWrapper == null) { + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; // another try, maybe the name is case-sensitive + } + + return TreeNodeTraversalResult.traverseSelectedChildNodes(tableWrapper); + } + + /** + * Accepts a table wrapper (lazy loader). + * + * @param tableWrapper Table wrapper. + * @param parameter Target object where found hierarchy nodes, dimensions and labels should be added. + * @return Accept's result. + */ + @Override + public TreeNodeTraversalResult accept(TableWrapper tableWrapper, SearchParameterObject parameter) { + TableSpec tableSpec = tableWrapper.getSpec(); + Boolean enabledFilter = this.filters.getEnabled(); + + if (this.filters.isIgnoreTablesWithoutSchedule()) { + if (tableSpec.getSchedulesOverride() == null || tableSpec.getSchedulesOverride().getProfiling() == null || + tableSpec.getSchedulesOverride().getProfiling().isDisabled() || + !Objects.equals(tableSpec.getSchedulesOverride().getProfiling().getCronExpression(), this.filters.getEnabledCronScheduleExpression())) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + } + + if (!Strings.isNullOrEmpty(this.filters.getEnabledCronScheduleExpression())) { + if (tableSpec.getSchedulesOverride() != null && tableSpec.getSchedulesOverride().getProfiling() != null && + !tableSpec.getSchedulesOverride().getProfiling().isDisabled() && + !Objects.equals(tableSpec.getSchedulesOverride().getProfiling().getCronExpression(), this.filters.getEnabledCronScheduleExpression())) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + } + + LabelsSearcherObject labelsSearcherObject = parameter.getLabelsSearcherObject(); + labelsSearcherObject.setTableLabels(tableSpec.getLabels()); + + if (labelsSearcherObject != null) { + labelsSearcherObject.setTableLabels(tableWrapper.getSpec().getLabels()); + } + + LabelSetSpec overriddenLabels = new LabelSetSpec(); + + if (labelsSearcherObject.getTableLabels() != null) { + overriddenLabels.addAll(labelsSearcherObject.getTableLabels()); + } + + if (labelsSearcherObject.getConnectionLabels() != null) { + overriddenLabels.addAll(labelsSearcherObject.getConnectionLabels()); + } + + if (!LabelsSearchMatcher.matchTableLabels(this.filters, overriddenLabels)) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + + PhysicalTableName physicalTableName = this.filters.getPhysicalTableName(); + if (physicalTableName != null) { + if (!tableWrapper.getPhysicalTableName().matchPattern(physicalTableName)) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + } + + if (tableSpec.isDisabled()) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + + if (enabledFilter != null) { + if (enabledFilter && tableSpec.isDisabled()) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + if (!enabledFilter && !tableSpec.isDisabled()) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + } + + return TreeNodeTraversalResult.TRAVERSE_CHILDREN; } /** diff --git a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTableSearchFiltersVisitor.java b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTargetTableSearchFiltersVisitor.java similarity index 81% rename from dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTableSearchFiltersVisitor.java rename to dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTargetTableSearchFiltersVisitor.java index 13fc8cf5a6..468ed5955d 100644 --- a/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTableSearchFiltersVisitor.java +++ b/dqops/src/main/java/com/dqops/metadata/search/StatisticsCollectorTargetTableSearchFiltersVisitor.java @@ -15,8 +15,6 @@ */ package com.dqops.metadata.search; -import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; -import com.dqops.metadata.id.HierarchyId; import com.dqops.metadata.labels.LabelSetSpec; import com.dqops.metadata.policies.column.ColumnQualityPolicyList; import com.dqops.metadata.policies.table.TableQualityPolicyList; @@ -24,30 +22,21 @@ import com.dqops.metadata.scheduling.CronSchedulesSpec; import com.dqops.metadata.sources.*; import com.dqops.metadata.traversal.TreeNodeTraversalResult; -import com.dqops.sensors.AbstractSensorParametersSpec; -import com.dqops.statistics.AbstractRootStatisticsCollectorsContainerSpec; -import com.dqops.statistics.AbstractStatisticsCollectorCategorySpec; -import com.dqops.statistics.AbstractStatisticsCollectorSpec; -import com.dqops.statistics.StatisticsCollectorTarget; -import com.dqops.statistics.column.ColumnStatisticsCollectorsRootCategoriesSpec; -import com.dqops.statistics.table.TableStatisticsCollectorsRootCategoriesSpec; import com.google.common.base.Strings; -import java.util.Collection; import java.util.Objects; -import java.util.Set; /** - * Visitor for {@link StatisticsCollectorSearchFilters} that finds statistics collectors to execute. + * Visitor for {@link StatisticsCollectorSearchFilters} that finds target tables on which statistics collectors are run. */ -public class StatisticsCollectorTableSearchFiltersVisitor extends AbstractSearchVisitor { +public class StatisticsCollectorTargetTableSearchFiltersVisitor extends AbstractSearchVisitor { protected final StatisticsCollectorSearchFilters filters; /** * Creates a visitor for the given filters. * @param filters Check search filters. */ - public StatisticsCollectorTableSearchFiltersVisitor(StatisticsCollectorSearchFilters filters) { + public StatisticsCollectorTargetTableSearchFiltersVisitor(StatisticsCollectorSearchFilters filters) { this.filters = filters; } @@ -147,33 +136,7 @@ public TreeNodeTraversalResult accept(TableList tableList, SearchParameterObject */ @Override public TreeNodeTraversalResult accept(TableWrapper tableWrapper, SearchParameterObject parameter) { - String schemaTableName = this.filters.getFullTableName(); - - if (Strings.isNullOrEmpty(schemaTableName)) { - return TreeNodeTraversalResult.TRAVERSE_CHILDREN; - } - - PhysicalTableName physicalTableName = PhysicalTableName.fromSchemaTableFilter(schemaTableName); - if (physicalTableName.isSearchPattern()) { - return TreeNodeTraversalResult.TRAVERSE_CHILDREN; // we need to iterate anyway - } - - if (tableWrapper.getPhysicalTableName().matchPattern(physicalTableName)) { - return TreeNodeTraversalResult.TRAVERSE_CHILDREN; - } - - return TreeNodeTraversalResult.SKIP_CHILDREN; - } - - /** - * Accepts a table specification. - * - * @param tableSpec Table specification. - * @param parameter Target object where found hierarchy nodes, dimensions and labels should be added. - * @return Accept's result. - */ - @Override - public TreeNodeTraversalResult accept(TableSpec tableSpec, SearchParameterObject parameter) { + TableSpec tableSpec = tableWrapper.getSpec(); Boolean enabledFilter = this.filters.getEnabled(); if (this.filters.isIgnoreTablesWithoutSchedule()) { @@ -195,6 +158,35 @@ public TreeNodeTraversalResult accept(TableSpec tableSpec, SearchParameterObject LabelsSearcherObject labelsSearcherObject = parameter.getLabelsSearcherObject(); labelsSearcherObject.setTableLabels(tableSpec.getLabels()); + if (labelsSearcherObject != null) { + labelsSearcherObject.setTableLabels(tableWrapper.getSpec().getLabels()); + } + + LabelSetSpec overriddenLabels = new LabelSetSpec(); + + if (labelsSearcherObject.getTableLabels() != null) { + overriddenLabels.addAll(labelsSearcherObject.getTableLabels()); + } + + if (labelsSearcherObject.getConnectionLabels() != null) { + overriddenLabels.addAll(labelsSearcherObject.getConnectionLabels()); + } + + if (!LabelsSearchMatcher.matchTableLabels(this.filters, overriddenLabels)) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + + PhysicalTableName physicalTableName = this.filters.getPhysicalTableName(); + if (physicalTableName != null) { + if (!tableWrapper.getPhysicalTableName().matchPattern(physicalTableName)) { + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + } + else { + parameter.getNodes().add(tableWrapper); + return TreeNodeTraversalResult.SKIP_CHILDREN; + } + if (tableSpec.isDisabled()) { return TreeNodeTraversalResult.SKIP_CHILDREN; } @@ -208,7 +200,7 @@ public TreeNodeTraversalResult accept(TableSpec tableSpec, SearchParameterObject } } - parameter.getNodes().add(tableSpec); + parameter.getNodes().add(tableWrapper); return TreeNodeTraversalResult.SKIP_CHILDREN; }