try to speed up the lookup without using the regex

JaneliaSciComp · May 30, 2024 · b808a77 · b808a77
1 parent 4b71537
commit b808a77
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 5 deletions.
diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/mips/FileDataUtils.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/mips/FileDataUtils.java
@@ -25,7 +25,7 @@ public class FileDataUtils {
 
     private static final Map<Path, Map<String, List<String>>> FILE_NAMES_CACHE = new HashMap<>();
 
-    public static FileData lookupVariantFileData(List<String> variantLocations, Pattern variantPattern, String sourceCDMName, String variantSuffix, Function<String, String> variantSuffixMapping) {
+    public static FileData lookupVariantFileData(List<String> variantLocations, String fastLookup, Pattern variantPattern, String sourceCDMName, String variantSuffix, Function<String, String> variantSuffixMapping) {
         if (CollectionUtils.isEmpty(variantLocations)) {
             return null;
         } else {
@@ -34,9 +34,9 @@ public static FileData lookupVariantFileData(List<String> variantLocations, Patt
                     .map(Paths::get)
                     .map(variantPath -> {
                         if (Files.isDirectory(variantPath)) {
-                            return lookupVariantFileDataInDir(variantPath, variantPattern);
+                            return lookupVariantFileDataInDir(variantPath, fastLookup, variantPattern);
                         } else if (Files.isRegularFile(variantPath)) {
-                            return lookupVariantFileDataInArchive(variantPath, variantPattern);
+                            return lookupVariantFileDataInArchive(variantPath, fastLookup, variantPattern);
                         } else {
                             return null;
                         }
@@ -47,9 +47,10 @@ public static FileData lookupVariantFileData(List<String> variantLocations, Patt
         }
     }
 
-    private static FileData lookupVariantFileDataInDir(Path variantPath, Pattern variantPattern) {
+    private static FileData lookupVariantFileDataInDir(Path variantPath, String fastLookup, Pattern variantPattern) {
         Map<String, List<String>> variantDirEntries = getDirEntryNames(variantPath);
         return variantDirEntries.entrySet().stream()
+                .filter(e -> StringUtils.contains(e.getKey(), fastLookup))
                 .filter(e -> {
                     Matcher variantMatcher = variantPattern.matcher(e.getKey());
                     return variantMatcher.find();
@@ -60,9 +61,10 @@ private static FileData lookupVariantFileDataInDir(Path variantPath, Pattern var
                 .orElse(null);
     }
 
-    private static FileData lookupVariantFileDataInArchive(Path variantPath, Pattern variantPattern) {
+    private static FileData lookupVariantFileDataInArchive(Path variantPath, String fastLookup, Pattern variantPattern) {
         Map<String, List<String>> variantArchiveEntries = getZipEntryNames(variantPath);
         return variantArchiveEntries.entrySet().stream()
+                .filter(e -> StringUtils.contains(e.getKey(), fastLookup))
                 .filter(e -> {
                     Matcher variantMatcher = variantPattern.matcher(e.getKey());
                     return variantMatcher.find();

diff --git a/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CreateCDSDataInputCmd.java b/colormipsearch-tools/src/main/java/org/janelia/colormipsearch/cmd/CreateCDSDataInputCmd.java
@@ -418,6 +418,7 @@ private <N extends AbstractNeuronEntity> void populateOtherComputeFilesFromInput
                     ComputeFileType variantFileType = ComputeFileType.fromName(args.variantFileTypeMapping.get(variant.variantType));
                     FileData variantFileData = FileDataUtils.lookupVariantFileData(
                             Collections.singletonList(variant.variantPath),
+                            neuronEntity.getNeuronId(),
                             variantPattern,
                             neuronEntity.getComputeFileName(ComputeFileType.SourceColorDepthImage),
                             variant.variantNameSuffix,