Skip to content

Commit

Permalink
Update the template based searching logic for "logical not lexical"
Browse files Browse the repository at this point in the history
  • Loading branch information
@QuyenLy87
QuyenLy87 committed Jan 9, 2024
1 parent 6e8c456 commit 2544188
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 60 deletions.
162 changes: 104 additions & 58 deletions ...g/ihtsdo/otf/transformationandtemplate/service/template/TemplateConceptSearchService.java
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,35 @@
package org.ihtsdo.otf.transformationandtemplate.service.template;

import org.ihtsdo.otf.transformationandtemplate.service.ConstantStrings;
import org.ihtsdo.otf.transformationandtemplate.service.exception.ServiceException;
import org.ihtsdo.otf.rest.client.RestClientException;
import org.ihtsdo.otf.rest.client.terminologyserver.SnowstormRestClient;
import org.ihtsdo.otf.rest.client.terminologyserver.SnowstormRestClientFactory;
import org.ihtsdo.otf.rest.client.terminologyserver.pojo.AxiomPojo;
import org.ihtsdo.otf.rest.client.terminologyserver.pojo.ConceptPojo;
import org.ihtsdo.otf.rest.client.terminologyserver.pojo.DescriptionPojo;
import org.ihtsdo.otf.rest.client.terminologyserver.pojo.RelationshipPojo;
import org.ihtsdo.otf.rest.exception.ResourceNotFoundException;
import org.ihtsdo.otf.transformationandtemplate.service.ConstantStrings;
import org.ihtsdo.otf.transformationandtemplate.service.exception.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.snomed.authoringtemplate.domain.ConceptTemplate;
import org.snomed.authoringtemplate.domain.DescriptionType;
import org.snomed.authoringtemplate.domain.logical.Attribute;
import org.snomed.authoringtemplate.domain.logical.AttributeGroup;
import org.snomed.authoringtemplate.domain.logical.LogicalTemplate;
import org.snomed.authoringtemplate.service.LogicalTemplateParserService;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;

import static org.ihtsdo.otf.rest.client.terminologyserver.pojo.DescriptionPojo.Type.FSN;
import static org.ihtsdo.otf.rest.client.terminologyserver.pojo.DescriptionPojo.Type.SYNONYM;

@Service
public class TemplateConceptSearchService {

Expand All @@ -39,7 +41,10 @@ public class TemplateConceptSearchService {

@Autowired
private LogicalTemplateParserService logicalTemplateParser;


@Autowired
private TemplateConceptTransformService templateConceptTransformService;

@Autowired
private SnowstormRestClientFactory terminologyClientFactory;

Expand All @@ -49,7 +54,12 @@ public class TemplateConceptSearchService {
private static final Logger LOGGER = LoggerFactory.getLogger(TemplateConceptSearchService.class);

private static final int MAX = 200000;


@Value("${transformation.batch.max}")
private int batchMax;

private final ExecutorService executorService = Executors.newFixedThreadPool(10);

public Set<String> searchConceptsByTemplate(String templateName, String branchPath,
Boolean logicalMatch, Boolean lexicalMatch, boolean stated) throws ServiceException, ResourceNotFoundException {

Expand All @@ -70,80 +80,116 @@ public Set<String> searchConceptsByTemplate(String templateName, String branchPa
LogicalTemplate logical = logicalTemplateParser.parseTemplate(conceptTemplate.getLogicalTemplate());
if (lexicalMatch != null) {
Set<String> logicalResult = performLogicalSearch(conceptTemplate, logical, branchPath, true, stated);
return performLexicalSearch(conceptTemplate, logicalResult, branchPath, lexicalMatch);
return performLexicalSearch(conceptTemplate, logicalResult, branchPath, logical, lexicalMatch);
} else {
return performLogicalSearch(conceptTemplate, logical, branchPath, logicalMatch, stated);
}
} catch (IOException e) {
throw new ServiceException("Failed to load tempate " + templateName);
throw new ServiceException("Failed to load template " + templateName);
}
}

private Set<String> performLexicalSearch(ConceptTemplate conceptTemplate,
Set<String> logicalMatched, String branchPath, boolean lexicalMatch) throws ServiceException {
Set<String> logicalMatched, String branchPath, LogicalTemplate logical, boolean lexicalMatch) throws ServiceException {

Set<String> result = new HashSet<>();
if (logicalMatched == null || logicalMatched.isEmpty()) {
LOGGER.info("No results found for logical search.");
return result;
}

Map<Pattern, Set<String>> fsnPatternSlotsMap = TemplateUtil.compilePatterns(
TemplateUtil.getTermTemplates(conceptTemplate, DescriptionType.FSN));

Map<Pattern, Set<String>> synoymPatternSlotsMap = TemplateUtil.compilePatterns(
TemplateUtil.getTermTemplates(conceptTemplate, DescriptionType.SYNONYM));
try {
Collection<ConceptPojo> concepts = terminologyClientFactory.getClient()
.searchConcepts(branchPath, new ArrayList<>(logicalMatched));

Map<String, List<DescriptionPojo>> originalConceptToDescriptionMap = new HashMap<>();
for (ConceptPojo conceptPojo : concepts) {
List<String> synoyms = conceptPojo.getDescriptions()
.stream()
.filter(DescriptionPojo::isActive)
.filter(d -> d.getType() == SYNONYM)
.map(DescriptionPojo::getTerm)
.collect(Collectors.toList());

List<String> fsns = conceptPojo.getDescriptions()
.stream()
.filter(DescriptionPojo::isActive)
.filter(d -> d.getType() == FSN)
.map(DescriptionPojo::getTerm)
.collect(Collectors.toList());

boolean isMatched = false;
for (Pattern pattern : fsnPatternSlotsMap.keySet()) {
isMatched = isPatternMatched(pattern, fsns);
if (!isMatched) {
break;
}
}
for (Pattern pattern : synoymPatternSlotsMap.keySet()) {
isMatched = isPatternMatched(pattern, synoyms);
if (!isMatched) {
break;
}
}
if (lexicalMatch && isMatched) {
result.add(conceptPojo.getConceptId());
} else if (!lexicalMatch && !isMatched){
result.add(conceptPojo.getConceptId());
List<DescriptionPojo> originalDescriptions = new ArrayList<>();
conceptPojo.getDescriptions().forEach(desc -> {
DescriptionPojo descriptionClone = new DescriptionPojo();
BeanUtils.copyProperties(desc, descriptionClone);
originalDescriptions.add(descriptionClone);
});
originalConceptToDescriptionMap.put(conceptPojo.getConceptId(), originalDescriptions);
}

TemplateTransformRequest transformRequest = new TemplateTransformRequest();
transformRequest.setDestinationTemplate(conceptTemplate.getName());
transformRequest.setConceptsToTransform(logicalMatched);
TemplateTransformation transformation = templateConceptTransformService.createTemplateTransformation(branchPath, transformRequest);

// Start transformations in multiple threads
List<Future<TransformationResult>> futureTasks = performTransform(transformation, logical, terminologyClientFactory.getClient());

// Gather transformed concepts and any errors from the transformation jobs
try {
List<TransformationResult> transformationResults = new ArrayList<>();
for (Future<TransformationResult> future : futureTasks) {
transformationResults.add(future.get());
}
transformationResults.forEach(transformationResult -> {
transformationResult.getConcepts().forEach(conceptPojo -> {
int count = 0;
for (DescriptionPojo transformedDesc : conceptPojo.getDescriptions()) {
for (DescriptionPojo originalDesc : originalConceptToDescriptionMap.get(conceptPojo.getConceptId())) {
if (transformedDesc.getTerm().equals(originalDesc.getTerm()) && originalDesc.isActive() && transformedDesc.getType().equals(originalDesc.getType())) {
count++;
break;
}
}
}
boolean isMatched = count > 0 && count == conceptPojo.getDescriptions().size();
if (lexicalMatch && isMatched) {
result.add(conceptPojo.getConceptId());
} else if (!lexicalMatch && !isMatched) {
result.add(conceptPojo.getConceptId());
}
});
});
LOGGER.info("Logical search results={} and lexical search results={}", logicalMatched.size(), result.size());
return result;
} catch (ExecutionException | InterruptedException e) {
throw new ServiceException("Failed to complete lexical template search.", e);
}
LOGGER.info("Logical search results={} and lexical search results={}", logicalMatched.size(), result.size());
return result;
} catch (RestClientException e) {
} catch (RestClientException e) {
throw new ServiceException("Failed to complete lexical template search.", e);
}
}
}

private boolean isPatternMatched(Pattern pattern, Collection<String> terms) {
for (String term : terms) {
if (pattern.matcher(term).matches()) {
return true;
public List<Future<TransformationResult>> performTransform(TemplateTransformation transformation, LogicalTemplate logical, SnowstormRestClient restClient) throws ServiceException {
String branchPath = transformation.getBranchPath();
TemplateTransformRequest transformRequest = transformation.getTransformRequest();
String destinationTemplate = transformRequest.getDestinationTemplate();

List<Future<TransformationResult>> results = new ArrayList<>();
ConceptTemplate destination;
try {
destination = templateService.loadOrThrow(destinationTemplate);
TransformationInputData input = new TransformationInputData(transformRequest);
input.setDestinationSlotToAttributeMap(TemplateUtil.getSlotToAttributeMap(logical, true));
input.setDestinationTemplate(destination);

input.setBranchPath(branchPath);
input.setConceptIdMap(templateConceptTransformService.getDestinationConceptsMap(branchPath, restClient, destination));
List<String> batchJob = null;
int counter=0;
for (String conceptId : transformRequest.getConceptsToTransform()) {
if (batchJob == null) {
batchJob = new ArrayList<>();
}
batchJob.add(conceptId);
counter++;
if (counter % batchMax == 0 || counter == transformRequest.getConceptsToTransform().size()) {
// Do work
final List<String> task = batchJob;
results.add(executorService.submit(() -> templateConceptTransformService.batchTransform(input, task, restClient)));
batchJob = null;
}
}
} catch (IOException e) {
throw new ServiceException("Failed to load template " + destinationTemplate, e);
}
return false;
return results;
}

private Set<String> performLogicalSearch(ConceptTemplate conceptTemplate, LogicalTemplate logical,
Expand All @@ -161,7 +207,7 @@ private Set<String> performLogicalSearch(ConceptTemplate conceptTemplate, Logica
Set<String> results = new HashSet<>(terminologyClientFactory.getClient().eclQuery(branchPath, ecl, MAX, stated));
List<ConceptPojo> conceptPojos = terminologyClientFactory.getClient().searchConcepts(branchPath, new ArrayList<>(results));
Set<String> toRemove = findConceptsNotMatchExactly(conceptPojos, attributeGroups, unGroupedAttributes, stated);
if (toRemove.size() > 0) {
if (!toRemove.isEmpty()) {
LOGGER.info("Total concepts " + toRemove.size() + " are removed from results.");
results.removeAll(toRemove);
}
Expand Down
4 changes: 2 additions & 2 deletions ...htsdo/otf/transformationandtemplate/service/template/TemplateConceptTransformService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ public List<Future<TransformationResult>> transform(TemplateTransformation trans

}

private TransformationResult batchTransform(TransformationInputData input, List<String> conceptIds, SnowstormRestClient restClient) {
public TransformationResult batchTransform(TransformationInputData input, List<String> conceptIds, SnowstormRestClient restClient) {
TransformationResult result = new TransformationResult();
Map<String, String> errors = new HashMap<>();
result.setFailures(errors);
Expand Down Expand Up @@ -245,7 +245,7 @@ private Map<String, ConceptMiniPojo> constructSlotToTargetValueMap(Transformatio
return slotToValuesMap;
}

private Map<String, ConceptMiniPojo> getDestinationConceptsMap(String branchPath, SnowstormRestClient client, ConceptTemplate destination) throws ServiceException {
public Map<String, ConceptMiniPojo> getDestinationConceptsMap(String branchPath, SnowstormRestClient client, ConceptTemplate destination) throws ServiceException {
List<String> conceptIds = new ArrayList<>();
List<Relationship> relationships = destination.getConceptOutline().getClassAxioms().stream().findFirst().get().getRelationships();
for (Relationship rel : relationships) {
Expand Down

0 comments on commit 2544188

Please sign in to comment.