Skip to content

Commit

Permalink
uuid for query
Browse files Browse the repository at this point in the history
  • Loading branch information
Luke Sikina committed Dec 24, 2023
1 parent 9dd6ef2 commit 7301781
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public Query(Query query) {
this.fields = new ArrayList<String>(query.fields);
this.requiredFields = new ArrayList<String>(query.requiredFields);
this.anyRecordOf = new ArrayList<String>(query.anyRecordOf);
this.numericFilters = new TreeMap<String, DoubleFilter>(query.numericFilters);
this.numericFilters = new TreeMap<String, Filter.DoubleFilter>(query.numericFilters);
this.categoryFilters = new TreeMap<String, String[]>(query.categoryFilters);
this.variantInfoFilters = new ArrayList<VariantInfoFilter>();
if (query.variantInfoFilters != null) {
Expand All @@ -30,6 +30,7 @@ public Query(Query query) {
});
}
this.id = query.id;
this.picSureId = query.picSureId;
}

private ResultType expectedResultType = ResultType.COUNT;
Expand All @@ -38,11 +39,12 @@ public Query(Query query) {
private List<String> requiredFields = new ArrayList<>();
private List<String> anyRecordOf = new ArrayList<>();
private List<List<String>> anyRecordOfMulti = new ArrayList<>();
private Map<String, DoubleFilter> numericFilters = new HashMap<>();
private Map<String, Filter.DoubleFilter> numericFilters = new HashMap<>();
private Map<String, String[]> categoryFilters = new HashMap<>();
private List<VariantInfoFilter> variantInfoFilters = new ArrayList<>();
private String id;

private String picSureId;

public ResultType getExpectedResultType() {
return expectedResultType;
Expand Down Expand Up @@ -72,7 +74,7 @@ public List<List<String>> getAllAnyRecordOf() {
return anyRecordOfMultiCopy;
}

public Map<String, DoubleFilter> getNumericFilters() {
public Map<String, Filter.DoubleFilter> getNumericFilters() {
return numericFilters;
}

Expand Down Expand Up @@ -111,7 +113,7 @@ public void setAnyRecordOfMulti(Collection<List<String>> anyRecordOfMulti) {
this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
}

public void setNumericFilters(Map<String, DoubleFilter> numericFilters) {
public void setNumericFilters(Map<String, Filter.DoubleFilter> numericFilters) {
this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
}

Expand All @@ -127,19 +129,27 @@ public void setId(String id) {
this.id = id;
}

public String getPicSureId() {
return picSureId;
}

public void setPicSureId(String picSureId) {
this.picSureId = picSureId;
}

public static class VariantInfoFilter {
public VariantInfoFilter() {

}

public VariantInfoFilter(VariantInfoFilter filter) {
this.numericVariantInfoFilters = new TreeMap<String, FloatFilter>(filter.numericVariantInfoFilters);
this.categoryVariantInfoFilters = new TreeMap<String, String[]>(filter.categoryVariantInfoFilters);
this.numericVariantInfoFilters = new TreeMap<>(filter.numericVariantInfoFilters);
this.categoryVariantInfoFilters = new TreeMap<>(filter.categoryVariantInfoFilters);
}

public Map<String, FloatFilter> numericVariantInfoFilters;
public Map<String, Filter.FloatFilter> numericVariantInfoFilters;
public Map<String, String[]> categoryVariantInfoFilters;

public String toString() {
StringBuilder builder = new StringBuilder();
writePartFormat("Numeric Variant Info Filters", numericVariantInfoFilters, builder);
Expand Down Expand Up @@ -207,7 +217,7 @@ public String toString() {

return builder.toString();
}

/**
* For some elements of the query, we will iterate over the list of items and send them each to the string builder
* @param queryPart
Expand All @@ -218,7 +228,7 @@ public String toString() {
private static void writePartFormat(String queryPart, Collection items, StringBuilder builder, boolean allowRollup) {
final Collection collectionToWrite = Optional.ofNullable(items).orElseGet(Collections::emptyList);
//same beginning
builder.append(queryPart + ": [");
builder.append(queryPart + ": [");
//if there are many elements, we want to truncate the display
if(allowRollup && collectionToWrite.size() > 5) {
builder.append("\n");
Expand All @@ -233,17 +243,17 @@ private static void writePartFormat(String queryPart, Collection items, StringBu
//same ending
builder.append("]\n");
}

@SuppressWarnings("rawtypes")
private static void showTopLevelValues(Collection varList, StringBuilder builder) {

Map<String, Integer> countMap = new HashMap<String, Integer>();

for(Object var : varList) {
if(var instanceof String) {
int index = ((String) var).startsWith("\\") ? 1 : 0;
String firstLevel = ((String)var).split("\\\\")[index];

Integer count = countMap.get(firstLevel);
if(count == null) {
count = 1;
Expand All @@ -255,7 +265,7 @@ private static void showTopLevelValues(Collection varList, StringBuilder builder
System.out.println("Object is not string! " + var);
}
}

for(String key : countMap.keySet()) {
builder.append("\t" + countMap.get(key) + " values under " + key + "\n");
}
Expand All @@ -275,12 +285,12 @@ private static void writePartFormat(String queryPart, Map varMap, StringBuilder
}

//for the mapped elements, we never want to roll up the values; always show
builder.append(queryPart + ": [");
builder.append(queryPart + ": [");
String sep1 = "";
for(Object key : varMap.keySet()) {
builder.append(sep1 + key + ": ");
Object value = varMap.get(key);

if(value instanceof Object[]) {
builder.append("{");
String sep2 = "";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import edu.harvard.hms.dbmi.avillach.hpds.service.filesharing.FileSharingService;
import edu.harvard.hms.dbmi.avillach.hpds.service.filesharing.FileSystemService;
import edu.harvard.hms.dbmi.avillach.hpds.service.util.Paginator;
import edu.harvard.hms.dbmi.avillach.hpds.service.util.QueryUUIDGen;
import org.apache.http.entity.ContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -46,7 +47,7 @@ public class PicSureService implements IResourceRS {
@Autowired
public PicSureService(QueryService queryService, TimelineProcessor timelineProcessor, CountProcessor countProcessor,
VariantListProcessor variantListProcessor, AbstractProcessor abstractProcessor,
Paginator paginator, FileSharingService fileSystemService
Paginator paginator, FileSharingService fileSystemService, QueryUUIDGen queryUUIDGen
) {
this.queryService = queryService;
this.timelineProcessor = timelineProcessor;
Expand All @@ -55,6 +56,7 @@ public PicSureService(QueryService queryService, TimelineProcessor timelineProce
this.abstractProcessor = abstractProcessor;
this.paginator = paginator;
this.fileSystemService = fileSystemService;
this.queryUUIDGen = queryUUIDGen;
Crypto.loadDefaultKey();
}

Expand All @@ -76,6 +78,8 @@ public PicSureService(QueryService queryService, TimelineProcessor timelineProce

private final FileSharingService fileSystemService;

private final QueryUUIDGen queryUUIDGen;

private static final String QUERY_METADATA_FIELD = "queryMetadata";
private static final int RESPONSE_CACHE_SIZE = 50;

Expand Down Expand Up @@ -266,8 +270,7 @@ public Response writeQueryResult(
// query IDs within HPDS are a different concept that query IDs in PIC-SURE
// Generally, equivalent queries with different PIC-SURE query IDs will have the SAME
// HPDS query ID.
String hpdsQueryId = UUIDv5.UUIDFromString(query.toString()).toString();
query.setId(hpdsQueryId);
queryUUIDGen.setId(query);
AsyncResult result = queryService.getResultFor(query.getId());
// the queryResult has this DIY retry logic that blocks a system thread.
// I'm not going to do that here. If the service can't find it, you get a 404.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.util.function.Predicate;
import java.util.stream.Collectors;

import edu.harvard.hms.dbmi.avillach.hpds.service.util.QueryUUIDGen;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -41,16 +42,21 @@ public class QueryService {
private final QueryProcessor queryProcessor;
private final TimeseriesProcessor timeseriesProcessor;
private final CountProcessor countProcessor;
private final QueryUUIDGen queryUUIDGen;

HashMap<String, AsyncResult> results = new HashMap<>();


@Autowired
public QueryService (AbstractProcessor abstractProcessor, QueryProcessor queryProcessor, TimeseriesProcessor timeseriesProcessor, CountProcessor countProcessor) {
public QueryService (
AbstractProcessor abstractProcessor, QueryProcessor queryProcessor, TimeseriesProcessor timeseriesProcessor,
CountProcessor countProcessor, QueryUUIDGen queryUUIDGen
) {
this.abstractProcessor = abstractProcessor;
this.queryProcessor = queryProcessor;
this.timeseriesProcessor = timeseriesProcessor;
this.countProcessor = countProcessor;
this.queryUUIDGen = queryUUIDGen;

SMALL_JOB_LIMIT = getIntProp("SMALL_JOB_LIMIT");
SMALL_TASK_THREADS = getIntProp("SMALL_TASK_THREADS");
Expand Down Expand Up @@ -122,7 +128,7 @@ private AsyncResult initializeResult(Query query) throws ClassNotFoundException,
result.queuedTime = System.currentTimeMillis();
result.id = UUIDv5.UUIDFromString(query.toString()).toString();
result.processor = p;
query.setId(result.id);
queryUUIDGen.setId(query);
results.put(result.id, result);
return result;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package edu.harvard.hms.dbmi.avillach.hpds.service.util;

import edu.harvard.dbmi.avillach.util.UUIDv5;
import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
import org.springframework.stereotype.Component;

@Component
public class QueryUUIDGen {
public void setId(Query query) {
query.setId(""); // the id is included in the toString
// I clear it here to keep the ID setting stable for any query
// of identical structure and content
String id = UUIDv5.UUIDFromString(query.toString()).toString();
query.setId(id);
}
}

0 comments on commit 7301781

Please sign in to comment.