Skip to content

Commit

Permalink
feat: cache recommendations (#334)
Browse files Browse the repository at this point in the history
* feat: cache recommendations

Signed-off-by: Ruben Romero Montes <[email protected]>

* feat: cache recommendations

Signed-off-by: Ruben Romero Montes <[email protected]>

* feat: cache recommendations

Signed-off-by: Ruben Romero Montes <[email protected]>

---------

Signed-off-by: Ruben Romero Montes <[email protected]>
  • Loading branch information
ruromero authored Apr 23, 2024
1 parent 7e97500 commit 43fe292
Show file tree
Hide file tree
Showing 19 changed files with 601 additions and 79 deletions.
26 changes: 23 additions & 3 deletions .tekton/exhort-pull-request.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,27 @@ spec:
workspace: mvn-settings
- name: workspace
workspace: workspace
- name: run-tests
runAfter:
- prefetch-dependencies
- copy-settings
taskSpec:
steps:
- image: registry.redhat.io/ubi9/openjdk-21:1.18
name: builder
workingDir: $(workspaces.source.path)/source
script: mvn -B --settings settings.xml verify -Dquarkus.redis.hosts=redis://localhost/
computeResources:
limits:
memory: 8Gi
requests:
memory: 6Gi
sidecars:
- image: docker.io/redis/redis-stack:7.2.0-v7
name: redis-stack
workspaces:
- name: source
workspace: workspace
- name: build-container
params:
- name: IMAGE
Expand All @@ -232,8 +253,7 @@ spec:
- name: COMMIT_SHA
value: $(tasks.clone-repository.results.commit)
runAfter:
- prefetch-dependencies
- copy-settings
- run-tests
taskRef:
params:
- name: name
Expand Down Expand Up @@ -395,7 +415,7 @@ spec:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storage: 4Gi
status: {}
- name: git-auth
secret:
Expand Down
26 changes: 23 additions & 3 deletions .tekton/exhort-push.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,27 @@ spec:
workspace: mvn-settings
- name: workspace
workspace: workspace
- name: run-tests
runAfter:
- prefetch-dependencies
- copy-settings
taskSpec:
steps:
- image: registry.redhat.io/ubi9/openjdk-21:1.18
name: builder
workingDir: $(workspaces.source.path)/source
script: mvn -B --settings settings.xml verify -Dquarkus.redis.hosts=redis://localhost/
computeResources:
limits:
memory: 8Gi
requests:
memory: 6Gi
sidecars:
- image: docker.io/redis/redis-stack:7.2.0-v7
name: redis-stack
workspaces:
- name: source
workspace: workspace
- name: build-container
params:
- name: IMAGE
Expand All @@ -226,8 +247,7 @@ spec:
- name: COMMIT_SHA
value: $(tasks.clone-repository.results.commit)
runAfter:
- prefetch-dependencies
- copy-settings
- run-tests
taskRef:
params:
- name: name
Expand Down Expand Up @@ -389,7 +409,7 @@ spec:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storage: 2Gi
status: {}
- name: git-auth
secret:
Expand Down
10 changes: 10 additions & 0 deletions deploy/exhort.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ spec:
secretKeyRef:
name: exhort-secret
key: telemetry-write-key
- name: DB_REDIS_HOST
valueFrom:
secretKeyRef:
name: exhort-secret
key: db.host
- name: DB_REDIS_PORT
valueFrom:
secretKeyRef:
name: exhort-secret
key: db.port
livenessProbe:
httpGet:
path: /q/health/live
Expand Down
15 changes: 15 additions & 0 deletions deploy/openshift/template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,16 @@ objects:
secretKeyRef:
name: exhort-secret
key: telemetry-write-key
- name: DB_REDIS_HOST
valueFrom:
secretKeyRef:
name: '${ELASTICACHE_SECRET}'
key: db.endpoint
- name: DB_REDIS_PORT
valueFrom:
secretKeyRef:
name: '${ELASTICACHE_SECRET}'
key: db.port
- name: MONITORING_ENABLED
value: "true"
- name: MONITORING_SENTRY_DSN
Expand Down Expand Up @@ -157,6 +167,11 @@ parameters:
description: Service name
value: exhort
required: true
- name: ELASTICACHE_SECRET
displayName: Elasticache Secret
description: Name of the secret containing the Elasticache settings
value: exhort-elasticache
required: true
- name: SERVICE_PORT
displayName: Service port
description: Service port
Expand Down
8 changes: 8 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-resteasy-reactive-jackson</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-redis-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.camel.quarkus</groupId>
<artifactId>camel-quarkus-jackson</artifactId>
Expand Down Expand Up @@ -209,6 +213,10 @@
<artifactId>camel-quarkus-junit5</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion src/main/docker/Dockerfile.jvm.staged
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ RUN mvn -B --settings settings.xml org.apache.maven.plugins:maven-dependency-plu

COPY src src

RUN mvn verify -B
RUN mvn package -B -DskipTests=true

RUN grep version /build/target/maven-archiver/pom.properties | cut -d '=' -f2 >.env-version
RUN grep artifactId /build/target/maven-archiver/pom.properties | cut -d '=' -f2 >.env-id
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/redhat/exhort/integration/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ private Constants() {}
public static final String GZIP_RESPONSE_PROPERTY = "gzipResponse";
public static final String SBOM_ID_PROPERTY = "sbomId";
public static final String UNSCANNED_REFS_PROPERTY = "unscannedRefs";
public static final String CACHED_RECOMMENDATIONS = "missedRecommendations";

public static final String API_VERSION_V4 = "v4";
public static final String API_VERSION_V3 = "v3";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright 2024 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.redhat.exhort.integration.cache;

import java.util.Map;
import java.util.Set;

import org.apache.camel.Body;
import org.apache.camel.ExchangeProperty;

import com.redhat.exhort.api.PackageRef;
import com.redhat.exhort.integration.Constants;
import com.redhat.exhort.model.trustedcontent.CachedRecommendation;
import com.redhat.exhort.model.trustedcontent.TrustedContentResponse;

public interface CacheService {

public void cacheRecommendations(
@Body TrustedContentResponse response,
@ExchangeProperty(Constants.CACHED_RECOMMENDATIONS) Set<PackageRef> misses);

public Map<PackageRef, CachedRecommendation> getRecommendations(Set<PackageRef> purls);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright 2024 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.redhat.exhort.integration.cache;

import java.time.Duration;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

import org.eclipse.microprofile.config.inject.ConfigProperty;

import com.redhat.exhort.api.PackageRef;
import com.redhat.exhort.model.trustedcontent.CachedRecommendation;
import com.redhat.exhort.model.trustedcontent.TrustedContentResponse;

import io.quarkus.redis.datasource.RedisDataSource;
import io.quarkus.redis.datasource.value.ValueCommands;

import jakarta.enterprise.context.ApplicationScoped;

@ApplicationScoped
public class RedisCacheService implements CacheService {

@ConfigProperty(name = "recommendations.cache.ttl", defaultValue = "1d")
Duration recommendationTtl;

private final ValueCommands<String, CachedRecommendation> recommendationsCommands;

public RedisCacheService(RedisDataSource ds) {
this.recommendationsCommands = ds.value(CachedRecommendation.class);
}

@Override
public void cacheRecommendations(TrustedContentResponse response, Set<PackageRef> misses) {
if (response == null || response.status() == null || !response.status().getOk()) {
return;
}
misses.stream()
.forEach(
v ->
recommendationsCommands.psetex(
"recommendations:" + v.ref(),
recommendationTtl.toMillis(),
new CachedRecommendation(v, response.recommendations().get(v))));
}

@Override
public Map<PackageRef, CachedRecommendation> getRecommendations(Set<PackageRef> purls) {
if (purls == null || purls.isEmpty()) {
return Collections.emptyMap();
}
var result =
recommendationsCommands.mget(
purls.stream().map(p -> "recommendations:" + p.ref()).toArray(String[]::new));
return result.values().stream()
.filter(Objects::nonNull)
.collect(Collectors.toMap(v -> v.ref(), v -> v));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -285,17 +285,13 @@ public ProviderReport buildReport(
}
Map<String, Source> reports = new HashMap<>();
sourcesIssues
.keySet()
.entrySet()
.forEach(
k ->
reports.put(
k,
k.getKey(),
buildReportForSource(
sourcesIssues.get(k),
tree,
privateProviders,
tcResponse,
response.unscanned())));
k.getValue(), tree, privateProviders, tcResponse, response.unscanned())));
return new ProviderReport().status(defaultOkStatus(getProviderName())).sources(reports);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,50 @@

package com.redhat.exhort.integration.trustedcontent;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;

import org.apache.camel.AggregationStrategy;
import org.apache.camel.Exchange;
import org.apache.camel.ExchangeProperty;

import com.redhat.exhort.api.PackageRef;
import com.redhat.exhort.integration.Constants;
import com.redhat.exhort.integration.cache.CacheService;
import com.redhat.exhort.model.trustedcontent.IndexedRecommendation;
import com.redhat.exhort.model.trustedcontent.TrustedContentCachedRequest;
import com.redhat.exhort.model.trustedcontent.TrustedContentResponse;

import io.quarkus.runtime.annotations.RegisterForReflection;

import jakarta.inject.Inject;
import jakarta.inject.Singleton;

@Singleton
@RegisterForReflection
public class TcResponseAggregation implements AggregationStrategy {

@Inject CacheService cacheService;

@Override
public Exchange aggregate(Exchange oldExchange, Exchange newExchange) {
oldExchange.setProperty(
Constants.TRUSTED_CONTENT_PROVIDER,
newExchange.getIn().getBody(TrustedContentResponse.class));
return oldExchange;
}

public TrustedContentResponse aggregateCachedResponse(
@ExchangeProperty(Constants.CACHED_RECOMMENDATIONS) TrustedContentCachedRequest cached,
Exchange exchange)
throws ExecutionException {
var externalResponse = exchange.getIn().getBody(TrustedContentResponse.class);
cacheService.cacheRecommendations(externalResponse, cached.miss());
Map<PackageRef, IndexedRecommendation> recommendations =
new HashMap<>(externalResponse.recommendations());
recommendations.putAll(cached.cached());
exchange.removeProperty(Constants.CACHED_RECOMMENDATIONS);
return new TrustedContentResponse(recommendations, externalResponse.status());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,19 @@ public class TrustedContentIntegration extends EndpointRouteBuilder {

@Inject TrustedContentRequestBuilder requestBuilder;

@Inject TcResponseAggregation aggregation;

@Override
public void configure() {
// fmt:off
from(direct("getTrustedContent"))
.routeId("getTrustedContent")
.routeId("getTrustedContent")
.setBody(method(requestBuilder, "filterCachedRecommendations"))
.to(direct("getRemoteTrustedContent"))
.setBody(method(aggregation, "aggregateCachedResponse"));

from(direct("getRemoteTrustedContent"))
.routeId("getRemoteTrustedContent")
.circuitBreaker()
.faultToleranceConfiguration()
.timeoutEnabled(true)
Expand All @@ -59,6 +67,7 @@ public void configure() {
.endCircuitBreaker()
.onFallback()
.process(responseHandler::processResponseError);

// fmt:on
}

Expand Down
Loading

0 comments on commit 43fe292

Please sign in to comment.