cf: cache search mod operations (#501)

* cf: cache search mod operations

* Remove old failed request catch point

src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java

@@ -6,6 +6,7 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -22,10 +23,12 @@
 import me.itzg.helpers.curseforge.model.ModsSearchResponse;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
+import me.itzg.helpers.errors.RateLimitException;
 import me.itzg.helpers.http.FailedRequestException;
 import me.itzg.helpers.http.Fetch;
 import me.itzg.helpers.http.FileDownloadStatusHandler;
 import me.itzg.helpers.http.SharedFetch;
+import me.itzg.helpers.http.SharedFetch.Options;
 import me.itzg.helpers.http.UriBuilder;
 import me.itzg.helpers.json.ObjectMappers;
 import org.slf4j.Logger;
@@ -41,18 +44,26 @@ public class CurseForgeApiClient implements AutoCloseable {
     public static final String CATEGORY_MC_MODS = "mc-mods";
     public static final String CATEGORY_BUKKIT_PLUGINS = "bukkit-plugins";
     public static final String CATEGORY_WORLDS = "worlds";
+    public static final String API_KEY_VAR = "CF_API_KEY";
+    public static final String ETERNAL_DEVELOPER_CONSOLE_URL = "https://console.curseforge.com/";
 
     private static final String API_KEY_HEADER = "x-api-key";
     static final String MINECRAFT_GAME_ID = "432";
 
+    public static final String OP_SEARCH_MOD_WITH_GAME_ID_SLUG_CLASS_ID = "searchModWithGameIdSlugClassId";
+    private static final Map<String, Duration> CACHE_DURATIONS = new HashMap<>();
+    static {
+        CACHE_DURATIONS.put(OP_SEARCH_MOD_WITH_GAME_ID_SLUG_CLASS_ID, Duration.ofHours(1));
+    }
+
     private final SharedFetch preparedFetch;
     private final UriBuilder uriBuilder;
     private final UriBuilder downloadFallbackUriBuilder;
     private final String gameId;
 
     private final ApiCaching apiCaching;
 
-    public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId,
+    public CurseForgeApiClient(String apiBaseUrl, String apiKey, Options sharedFetchOptions, String gameId,
         ApiCaching apiCaching
     ) {
         this.apiCaching = apiCaching;
@@ -61,7 +72,7 @@ public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options
         }
 
         this.preparedFetch = Fetch.sharedFetch("install-curseforge",
-            (sharedFetchOptions != null ? sharedFetchOptions : SharedFetch.Options.builder().build())
+            (sharedFetchOptions != null ? sharedFetchOptions : Options.builder().build())
                 .withHeader(API_KEY_HEADER, apiKey.trim())
         );
         this.uriBuilder = UriBuilder.withBaseUrl(apiBaseUrl);
@@ -87,6 +98,10 @@ static FileDownloadStatusHandler modFileDownloadStatusHandler(Path outputDir, Lo
         };
     }
 
+    public static Map<String, Duration> getCacheDurations() {
+        return CACHE_DURATIONS;
+    }
+
     @Override
     public void close() {
         preparedFetch.close();
@@ -111,28 +126,34 @@ Mono<CategoryInfo> loadCategoryInfo(Collection<String> applicableClassIdSlugs) {
 
                     return Mono.just(new CategoryInfo(contentClassIds, slugIds));
                 }
-            );
+            )
+            .onErrorMap(FailedRequestException::isForbidden, this::errorMapForbidden);
     }
 
     Mono<CurseForgeMod> searchMod(String slug, int classId) {
-        return preparedFetch.fetch(
-                uriBuilder.resolve("/v1/mods/search?gameId={gameId}&slug={slug}&classId={classId}",
-                    gameId, slug, classId
-                )
-            )
-            .toObject(ModsSearchResponse.class)
-            .assemble()
-            .flatMap(searchResponse -> {
-                if (searchResponse.getData() == null || searchResponse.getData().isEmpty()) {
-                    return Mono.error(new GenericException("No mods found with slug=" + slug));
-                }
-                else if (searchResponse.getData().size() > 1) {
-                    return Mono.error(new GenericException("More than one mod found with slug=" + slug));
-                }
-                else {
-                    return Mono.just(searchResponse.getData().get(0));
-                }
-            });
+        return
+            apiCaching.cache(OP_SEARCH_MOD_WITH_GAME_ID_SLUG_CLASS_ID, CurseForgeMod.class,
+                preparedFetch.fetch(
+                        uriBuilder.resolve("/v1/mods/search?gameId={gameId}&slug={slug}&classId={classId}",
+                            gameId, slug, classId
+                        )
+                    )
+                    .toObject(ModsSearchResponse.class)
+                    .assemble()
+                    .flatMap(searchResponse -> {
+                        if (searchResponse.getData() == null || searchResponse.getData().isEmpty()) {
+                            return Mono.error(new GenericException("No mods found with slug=" + slug));
+                        }
+                        else if (searchResponse.getData().size() > 1) {
+                            return Mono.error(new GenericException("More than one mod found with slug=" + slug));
+                        }
+                        else {
+                            return Mono.just(searchResponse.getData().get(0));
+                        }
+                    })
+                    .onErrorMap(FailedRequestException::isForbidden, this::errorMapForbidden),
+                gameId, slug, classId
+            );
     }
 
     /**
@@ -179,7 +200,8 @@ Mono<Integer> slugToId(CategoryInfo categoryInfo,
                     .findFirst()
                     .map(CurseForgeMod::getId)
                     .orElseThrow(() -> new GenericException("Unable to resolve slug into ID (no matches): " + slug))
-            );
+            )
+            .onErrorMap(FailedRequestException::isForbidden, this::errorMapForbidden);
     }
 
     public Mono<CurseForgeMod> getModInfo(
@@ -193,6 +215,7 @@ public Mono<CurseForgeMod> getModInfo(
                 )
                 .toObject(GetModResponse.class)
                 .assemble()
+                .onErrorMap(FailedRequestException::isForbidden, this::errorMapForbidden)
                 .checkpoint("Getting mod info for " + projectID)
                 .map(GetModResponse::getData),
             projectID
@@ -219,6 +242,7 @@ public Mono<CurseForgeFile> getModFileInfo(
                     }
                     return e;
                 })
+                .onErrorMap(FailedRequestException::isForbidden, this::errorMapForbidden)
                 .map(GetModFileResponse::getData)
                 .checkpoint(),
             projectID, fileID
@@ -285,4 +309,21 @@ private static URI normalizeDownloadUrl(String downloadUrl) {
         );
     }
 
+    public Throwable errorMapForbidden(Throwable throwable) {
+        final FailedRequestException e = (FailedRequestException) throwable;
+
+        log.debug("Failed request details: {}", e.toString());
+
+        if (e.getBody().contains("There might be too much traffic")) {
+            return new RateLimitException(null, String.format("Access to %s has been rate-limited.", uriBuilder.getBaseUrl()), e);
+        }
+        else {
+            return new InvalidParameterException(String.format("Access to %s is forbidden or rate-limit has been exceeded."
+                    + " Ensure %s is set to a valid API key from %s or allow rate-limit to reset.",
+                uriBuilder.getBaseUrl(), API_KEY_VAR, ETERNAL_DEVELOPER_CONSOLE_URL
+            ), e
+            );
+        }
+
+    }
 }

src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java

@@ -68,10 +68,10 @@ public void setSlugCategory(String defaultCategory) {
             + "%nCan also be passed via CF_API_BASE_URL")
     String apiBaseUrl;
 
-    @Option(names = "--api-key", defaultValue = "${env:" + CurseForgeInstaller.API_KEY_VAR + "}",
+    @Option(names = "--api-key", defaultValue = "${env:" + API_KEY_VAR + "}",
         description = "An API key allocated from the Eternal developer console at "
-            + CurseForgeInstaller.ETERNAL_DEVELOPER_CONSOLE_URL +
-            "%nCan also be passed via " + CurseForgeInstaller.API_KEY_VAR
+            + ETERNAL_DEVELOPER_CONSOLE_URL +
+            "%nCan also be passed via " + API_KEY_VAR
     )
     String apiKey;
 
@@ -117,7 +117,8 @@ public Integer call() throws Exception {
         if (modFileRefs != null && !modFileRefs.isEmpty()) {
             try (
                 final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled()
-                    : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs);
+                    : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs)
+                        .setCacheDurations(CurseForgeApiClient.getCacheDurations());
                 final CurseForgeApiClient apiClient = new CurseForgeApiClient(
                     apiBaseUrl, apiKey, sharedFetchArgs.options(),
                     CurseForgeApiClient.MINECRAFT_GAME_ID,

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -45,7 +45,6 @@
 import me.itzg.helpers.curseforge.model.ModLoader;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
-import me.itzg.helpers.errors.RateLimitException;
 import me.itzg.helpers.fabric.FabricLauncherInstaller;
 import me.itzg.helpers.files.Manifests;
 import me.itzg.helpers.files.ResultsFileWriter;
@@ -71,10 +70,8 @@
 @Slf4j
 public class CurseForgeInstaller {
 
-    public static final String API_KEY_VAR = "CF_API_KEY";
     public static final String MODPACK_ZIP_VAR = "CF_MODPACK_ZIP";
 
-    public static final String ETERNAL_DEVELOPER_CONSOLE_URL = "https://console.curseforge.com/";
     public static final String CURSEFORGE_ID = "curseforge";
     public static final String REPO_SUBDIR_MODPACKS = "modpacks";
     public static final String REPO_SUBDIR_MODS = "mods";
@@ -199,19 +196,20 @@ void install(String slug, InstallationEntryPoint entryPoint) {
                 log.warn("API key is not set, so will re-use previous modpack installation of {}",
                     manifest.getSlug() != null ? manifest.getSlug() : "Project ID " + manifest.getModId()
                 );
-                log.warn("Obtain an API key from " + ETERNAL_DEVELOPER_CONSOLE_URL
-                    + " and set the environment variable " + API_KEY_VAR + " in order to restore full functionality.");
+                log.warn("Obtain an API key from " + CurseForgeApiClient.ETERNAL_DEVELOPER_CONSOLE_URL
+                    + " and set the environment variable " + CurseForgeApiClient.API_KEY_VAR + " in order to restore full functionality.");
                 return;
             }
             else {
-                throw new InvalidParameterException("API key is not set. Obtain an API key from " + ETERNAL_DEVELOPER_CONSOLE_URL
-                    + " and set the environment variable " + API_KEY_VAR);
+                throw new InvalidParameterException("API key is not set. Obtain an API key from " + CurseForgeApiClient.ETERNAL_DEVELOPER_CONSOLE_URL
+                    + " and set the environment variable " + CurseForgeApiClient.API_KEY_VAR);
             }
         }
 
         try (
             final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled()
-                : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs);
+                : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs)
+                    .setCacheDurations(CurseForgeApiClient.getCacheDurations());
             final CurseForgeApiClient cfApi = new CurseForgeApiClient(
                 apiBaseUrl, apiKey, sharedFetchOptions,
                 CurseForgeApiClient.MINECRAFT_GAME_ID,
@@ -230,23 +228,6 @@ void install(String slug, InstallationEntryPoint entryPoint) {
                 new InstallContext(slug, cfApi, categoryInfo, manifest)
             );
 
-        } catch (FailedRequestException e) {
-            if (e.getStatusCode() == 403) {
-                log.debug("Failed request details: {}", e.toString());
-
-                if (e.getBody().contains("There might be too much traffic")) {
-                    throw new RateLimitException(null, String.format("Access to %s has been rate-limited.", apiBaseUrl), e);
-                }
-                else {
-                    throw new InvalidParameterException(String.format("Access to %s is forbidden or rate-limit has been exceeded."
-                            + " Ensure %s is set to a valid API key from %s or allow rate-limit to reset.",
-                        apiBaseUrl, API_KEY_VAR, ETERNAL_DEVELOPER_CONSOLE_URL
-                    ), e);
-                }
-            }
-            else {
-                throw e;
-            }
         } catch (IOException e) {
             throw new GenericException("Failed to setup API caching", e);
         }

src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java

@@ -79,10 +79,10 @@ public class InstallCurseForgeCommand implements Callable<Integer> {
         description = "Allows for overriding the CurseForge Eternal API used")
     String apiBaseUrl;
 
-    @Option(names = "--api-key", defaultValue = "${env:" + CurseForgeInstaller.API_KEY_VAR + "}",
+    @Option(names = "--api-key", defaultValue = "${env:" + CurseForgeApiClient.API_KEY_VAR + "}",
         description = "An API key allocated from the Eternal developer console at "
-            + CurseForgeInstaller.ETERNAL_DEVELOPER_CONSOLE_URL +
-            "%nCan also be passed via " + CurseForgeInstaller.API_KEY_VAR
+            + CurseForgeApiClient.ETERNAL_DEVELOPER_CONSOLE_URL +
+            "%nCan also be passed via " + CurseForgeApiClient.API_KEY_VAR
     )
     String apiKey;
 

src/main/java/me/itzg/helpers/http/FailedRequestException.java

@@ -32,6 +32,10 @@ public static boolean isNotFound(Throwable throwable) {
         return isStatus(throwable, HttpResponseStatus.NOT_FOUND);
     }
 
+    public static boolean isForbidden(Throwable throwable) {
+        return isStatus(throwable, HttpResponseStatus.FORBIDDEN);
+    }
+
     public static boolean isStatus(Throwable throwable, HttpResponseStatus... statuses) {
         if (throwable instanceof FailedRequestException) {
             final int actualStatus = ((FailedRequestException) throwable).getStatusCode();