Skip to content

Commit

Permalink
Updated UI postgres indexer test refactor and workbench version selec…
Browse files Browse the repository at this point in the history 
…tion

Fixed executions paging on postgres
Updated version selection on workbench to match definitions editor and display more than 11 entries.
Updated postgres indexer to show total record count on searches.
  • Loading branch information
@danmiller192
danmiller192 committed Jul 22, 2024
1 parent 7230de1 commit 9902415
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 43 deletions.
30 changes: 22 additions & 8 deletions postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresIndexDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public void indexWorkflow(WorkflowSummary workflow) {
+ "VALUES (?, ?, ?, ?, ?, ?, ?::JSONB) ON CONFLICT (workflow_id) \n"
+ "DO UPDATE SET correlation_id = EXCLUDED.correlation_id, workflow_type = EXCLUDED.workflow_type, "
+ "start_time = EXCLUDED.start_time, status = EXCLUDED.status, json_data = EXCLUDED.json_data "
+ "WHERE EXCLUDED.update_time > workflow_index.update_time";
+ "WHERE EXCLUDED.update_time >= workflow_index.update_time";

if (onlyIndexOnStatusChange) {
INSERT_WORKFLOW_INDEX_SQL += " AND workflow_index.status != EXCLUDED.status";
Expand Down Expand Up @@ -125,12 +125,19 @@ public SearchResult<WorkflowSummary> searchWorkflowSummary(
queryBuilder.getQuery(),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeAndFetch(WorkflowSummary.class);
});

// To avoid making a second potentially expensive query to postgres say we've
// got enough results for another page so the pagination works
int totalHits = results.size() == count ? start + count + 1 : start + results.size();
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});

int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}

Expand All @@ -141,7 +148,7 @@ public void indexTask(TaskSummary task) {
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?::JSONB) ON CONFLICT (task_id) "
+ "DO UPDATE SET task_type = EXCLUDED.task_type, task_def_name = EXCLUDED.task_def_name, "
+ "status = EXCLUDED.status, update_time = EXCLUDED.update_time, json_data = EXCLUDED.json_data "
+ "WHERE EXCLUDED.update_time > task_index.update_time";
+ "WHERE EXCLUDED.update_time >= task_index.update_time";

if (onlyIndexOnStatusChange) {
INSERT_TASK_INDEX_SQL += " AND task_index.status != EXCLUDED.status";
Expand Down Expand Up @@ -181,12 +188,19 @@ public SearchResult<TaskSummary> searchTaskSummary(
queryBuilder.getQuery(),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeAndFetch(TaskSummary.class);
});

// To avoid making a second potentially expensive query to postgres say we've
// got enough results for another page so the pagination works
int totalHits = results.size() == count ? start + count + 1 : start + results.size();
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});

int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}

Expand Down
19 changes: 19 additions & 0 deletions ...sistence/src/main/java/com/netflix/conductor/postgres/util/PostgresIndexQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,29 @@ public String getQuery() {
+ " LIMIT ? OFFSET ?";
}

public String getCountQuery() {
String queryString = "";
List<Condition> validConditions =
conditions.stream().filter(c -> c.isValid()).collect(Collectors.toList());
if (validConditions.size() > 0) {
queryString =
" WHERE "
+ String.join(
" AND ",
validConditions.stream()
.map(c -> c.getQueryFragment())
.collect(Collectors.toList()));
}
return "SELECT COUNT(json_data::TEXT) FROM " + table + queryString;
}

public void addParameters(Query q) throws SQLException {
for (Condition condition : conditions) {
condition.addParameter(q);
}
}

public void addPagingParameters(Query q) throws SQLException {
q.addParameter(count);
q.addParameter(start);
}
Expand Down
94 changes: 65 additions & 29 deletions ...es-persistence/src/test/java/com/netflix/conductor/postgres/dao/PostgresIndexDAOTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,24 +198,43 @@ public void testIndexExistingWorkflow() throws SQLException {
}

@Test
public void testIndexExistingWorkflowWithOlderUpdateToEnsureItsNotIndexed()
throws SQLException {
public void testWhenWorkflowIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
WorkflowSummary firstWorkflowUpdate =
getMockWorkflowSummary("workflow-id-existing-no-index");
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");

WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-existing-no-index");
WorkflowSummary secondWorkflowUpdateSummary =
getMockWorkflowSummary("workflow-id-existing-no-index");
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:43:45Z");
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);

indexDAO.indexWorkflow(wfs);
indexDAO.indexWorkflow(secondWorkflowUpdateSummary);

compareWorkflowSummary(wfs);
compareWorkflowSummary(secondWorkflowUpdateSummary);

// Set the update time to the past
wfs.setUpdateTime("2023-02-07T08:42:45Z");
wfs.setStatus(Workflow.WorkflowStatus.FAILED);
indexDAO.indexWorkflow(firstWorkflowUpdate);

indexDAO.indexWorkflow(wfs);
compareWorkflowSummary(secondWorkflowUpdateSummary);
}

// Reset the workflow to check it's not been updated
wfs = getMockWorkflowSummary("workflow-id-existing-no-index");
compareWorkflowSummary(wfs);
@Test
public void testWhenWorkflowUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
WorkflowSummary firstWorkflowUpdate =
getMockWorkflowSummary("workflow-id-existing-same-time-index");
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");

WorkflowSummary secondWorkflowUpdateSummary =
getMockWorkflowSummary("workflow-id-existing-same-time-index");
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:42:45Z");
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);

indexDAO.indexWorkflow(firstWorkflowUpdate);

compareWorkflowSummary(firstWorkflowUpdate);

indexDAO.indexWorkflow(secondWorkflowUpdateSummary);

compareWorkflowSummary(secondWorkflowUpdateSummary);
}

@Test
Expand Down Expand Up @@ -244,22 +263,39 @@ public void testIndexExistingTask() throws SQLException {
}

@Test
public void testIndexExistingTaskWithOlderUpdateToEnsureItsNotIndexed() throws SQLException {
TaskSummary ts = getMockTaskSummary("task-id-exiting-no-update");
public void testWhenTaskIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-no-update");
firstTaskState.setUpdateTime("2023-02-07T09:41:45Z");
firstTaskState.setStatus(Task.Status.FAILED);

indexDAO.indexTask(ts);
TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-no-update");
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");

compareTaskSummary(ts);
indexDAO.indexTask(secondTaskState);

// Set the update time to the past
ts.setUpdateTime("2023-02-07T09:41:45Z");
ts.setStatus(Task.Status.FAILED);
compareTaskSummary(secondTaskState);

indexDAO.indexTask(ts);
indexDAO.indexTask(firstTaskState);

// Reset the task to check it's not been updated
ts = getMockTaskSummary("task-id-exiting-no-update");
compareTaskSummary(ts);
compareTaskSummary(secondTaskState);
}

@Test
public void testWhenTaskUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
firstTaskState.setUpdateTime("2023-02-07T09:42:45Z");
firstTaskState.setStatus(Task.Status.FAILED);

TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");

indexDAO.indexTask(firstTaskState);

compareTaskSummary(firstTaskState);

indexDAO.indexTask(secondTaskState);

compareTaskSummary(secondTaskState);
}

@Test
Expand Down Expand Up @@ -347,7 +383,7 @@ public void testSearchWorkflowSummaryPagination() {
List<String> orderBy = Arrays.asList(new String[] {"workflowId:DESC"});
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary("", "workflow-id-pagination*", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 3, results.getTotalHits());
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand All @@ -358,7 +394,7 @@ public void testSearchWorkflowSummaryPagination() {
"workflow-id-pagination-3",
results.getResults().get(1).getWorkflowId());
results = indexDAO.searchWorkflowSummary("", "*", 2, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong totalHits returned", 8, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand All @@ -369,7 +405,7 @@ public void testSearchWorkflowSummaryPagination() {
"workflow-id-pagination-1",
results.getResults().get(1).getWorkflowId());
results = indexDAO.searchWorkflowSummary("", "*", 4, 2, orderBy);
assertEquals("Wrong totalHits returned", 7, results.getTotalHits());
assertEquals("Wrong totalHits returned", 8, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand Down Expand Up @@ -400,7 +436,7 @@ public void testSearchTaskSummaryPagination() {

List<String> orderBy = Arrays.asList(new String[] {"taskId:DESC"});
SearchResult<TaskSummary> results = indexDAO.searchTaskSummary("", "*", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 3, results.getTotalHits());
assertEquals("Wrong totalHits returned", 10, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand All @@ -411,7 +447,7 @@ public void testSearchTaskSummaryPagination() {
"task-id-pagination-3",
results.getResults().get(1).getTaskId());
results = indexDAO.searchTaskSummary("", "*", 2, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong totalHits returned", 10, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand All @@ -422,7 +458,7 @@ public void testSearchTaskSummaryPagination() {
"task-id-pagination-1",
results.getResults().get(1).getTaskId());
results = indexDAO.searchTaskSummary("", "*", 4, 2, orderBy);
assertEquals("Wrong totalHits returned", 7, results.getTotalHits());
assertEquals("Wrong totalHits returned", 10, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
Expand Down
10 changes: 10 additions & 0 deletions ...ence/src/test/java/com/netflix/conductor/postgres/util/PostgresIndexQueryBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ void shouldGenerateQueryForEmptyString() throws SQLException {
assertEquals("SELECT json_data::TEXT FROM table_name LIMIT ? OFFSET ?", generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
Expand All @@ -57,6 +58,7 @@ void shouldGenerateQueryForNull() throws SQLException {
assertEquals("SELECT json_data::TEXT FROM table_name LIMIT ? OFFSET ?", generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
Expand All @@ -75,6 +77,7 @@ void shouldGenerateQueryForWorkflowId() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("abc123");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -94,6 +97,7 @@ void shouldGenerateQueryForMultipleInClause() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter(new ArrayList<>(List.of("COMPLETED", "RUNNING")));
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -113,6 +117,7 @@ void shouldGenerateQueryForSingleInClause() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("COMPLETED");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -132,6 +137,7 @@ void shouldGenerateQueryForStartTimeGt() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("2023-02-06T16:54:58Z");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -151,6 +157,7 @@ void shouldGenerateQueryForStartTimeLt() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("2023-02-06T16:54:58Z");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -170,6 +177,7 @@ void shouldGenerateQueryForUpdateTimeGt() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("2023-02-06T16:54:58Z");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -189,6 +197,7 @@ void shouldGenerateQueryForUpdateTimeLt() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("2023-02-06T16:54:58Z");
inOrder.verify(mockQuery).addParameter(15);
Expand All @@ -209,6 +218,7 @@ void shouldGenerateQueryForMultipleConditions() throws SQLException {
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("2023-02-06T16:54:58Z");
inOrder.verify(mockQuery).addParameter("2023-02-06T16:38:18Z");
Expand Down
2 changes: 1 addition & 1 deletion ui/cypress/e2e/spec.cy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ describe("Landing Page", () => {
it("Homepage preloads with default query", () => {
cy.visit("/");
cy.contains("Search Execution");
cy.contains("Page 1 of 5");
cy.contains("Page 1 of 1");
cy.get(".rdt_TableCell").contains("feature_value_compute_workflow");
});

Expand Down
Loading

0 comments on commit 9902415

Please sign in to comment.