Skip to content

Commit

Permalink
Suppress addition of unnecessary repo-refereces when assembling p2-repos
Browse files Browse the repository at this point in the history
  • Loading branch information
@HannesWell
HannesWell committed Sep 6, 2023
1 parent 107805e commit f00fcf8
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 26 deletions.
11 changes: 9 additions & 2 deletions tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ public class DestinationRepositoryDescriptor {
private final boolean append;
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final List<RepositoryReference> filterablRepositoryReferences;

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append,
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences) {
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences,
List<RepositoryReference> filterablRepositoryReferences) {
this.location = location;
this.name = name;
this.compress = compress;
Expand All @@ -41,12 +43,13 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr
this.append = append;
this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties;
this.repositoryReferences = repositoryReferences;
this.filterablRepositoryReferences = filterablRepositoryReferences;
}

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) {
this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(),
Collections.emptyList());
Collections.emptyList(), Collections.emptyList());
}

public DestinationRepositoryDescriptor(File location, String name) {
Expand Down Expand Up @@ -88,4 +91,8 @@ public Map<String, String> getExtraArtifactRepositoryProperties() {
public List<RepositoryReference> getRepositoryReferences() {
return repositoryReferences == null ? Collections.emptyList() : repositoryReferences;
}

public List<RepositoryReference> getFilterableRepositoryReferences() {
return filterablRepositoryReferences;
}
}
4 changes: 3 additions & 1 deletion ...e/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public interface MirrorApplicationService {
* @param includeRequiredFeatures
* Whether to include features mentioned in the require section of a feature
* @param filterProvided Whether to filter IU/artifacts that are already provided by a referenced repository
* @param excludeNotStrictlyRequiredRepoReferences Whether to exclude repository-references that don't contribute any IU from being added as such.
* @param filterProperties
* additional filter properties to be set in the p2 slicing options. May be
* <code>null</code>
Expand All @@ -64,7 +65,8 @@ public interface MirrorApplicationService {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> seeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException;
boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences,
Map<String, String> filterProperties) throws FacadeException;

/**
* recreates the metadata of an existing repository e.g. to account for changes in the contained
Expand Down
4 changes: 3 additions & 1 deletion tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,8 @@ private static IQuery<IInstallableUnit> createQuery(IUDescription iu) {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> projectSeeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException {
boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences,
Map<String, String> filterProperties) throws FacadeException {
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);

// mirror scope: seed units...
Expand All @@ -164,6 +165,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes
mirrorApp.setIncludeRequiredFeatures(includeRequiredFeatures);
mirrorApp.setIncludePacked(false); // no way, Tycho do no longer support packed artifacts anyways
mirrorApp.setFilterProvided(filterProvided);
mirrorApp.setExcludeNotStrictlyRequiredRepoReferences(excludeNotStrictlyRequiredRepoReferences);
// TODO the p2 mirror tool should support mirroring multiple environments at once
for (TargetEnvironment environment : context.getEnvironments()) {
SlicingOptions options = new SlicingOptions();
Expand Down
100 changes: 92 additions & 8 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,52 @@
*
* Contributors:
* SAP SE - initial API and implementation
* Hannes Wellmann - Implement user-defined filtering and filtering based on relevance for automatically added repo-references
*******************************************************************************/
package org.eclipse.tycho.p2tools;

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import org.apache.commons.lang3.StringUtils;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.equinox.internal.p2.director.PermissiveSlicer;
import org.eclipse.equinox.internal.p2.director.Slicer;
import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.repository.LocalMetadataRepository;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor;
import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions;
import org.eclipse.equinox.p2.metadata.IArtifactKey;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.metadata.IRequirement;
import org.eclipse.equinox.p2.metadata.Version;
import org.eclipse.equinox.p2.metadata.expression.IMatchExpression;
import org.eclipse.equinox.p2.query.CollectionResult;
import org.eclipse.equinox.p2.query.IQueryResult;
import org.eclipse.equinox.p2.query.IQueryable;
import org.eclipse.equinox.p2.query.QueryUtil;
import org.eclipse.equinox.p2.repository.IRepository;
import org.eclipse.equinox.p2.repository.IRepositoryManager;
import org.eclipse.equinox.p2.repository.IRepositoryReference;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
Expand All @@ -56,17 +68,22 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom
private static final String SOURCE_SUFFIX = ".source";
private static final String FEATURE_GROUP = ".feature.group";
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final List<RepositoryReference> repositoryReferences = new ArrayList<>();
private final Set<URI> filterableRepositoryReferenceURIs;
private boolean includeAllSource;
private boolean includeRequiredBundles;
private boolean includeRequiredFeatures;
private boolean filterProvided;
private boolean excludeNotStrictlyRequiredRepoReferences;
private TargetPlatform targetPlatform;

public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
super(agent);
this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties();
this.repositoryReferences = destination.getRepositoryReferences();
this.filterableRepositoryReferenceURIs = destination.getFilterableRepositoryReferences().stream()
.map(TychoMirrorApplication::getNormalizedLocation).collect(Collectors.toSet());
this.repositoryReferences.addAll(destination.getRepositoryReferences());
this.repositoryReferences.addAll(destination.getFilterableRepositoryReferences());
this.removeAddedRepositories = false;
}

Expand Down Expand Up @@ -155,28 +172,42 @@ protected IMetadataRepository initializeDestination(RepositoryDescriptor toInit,
private static Stream<org.eclipse.equinox.p2.repository.spi.RepositoryReference> toSpiRepositoryReferences(
RepositoryReference rr) {
return Stream.of(IRepository.TYPE_METADATA, IRepository.TYPE_ARTIFACT).map(type -> {
URI location = URI.create(rr.getLocation());
URI location = getNormalizedLocation(rr);
int options = rr.isEnable() ? IRepository.ENABLED : IRepository.NONE;
return new org.eclipse.equinox.p2.repository.spi.RepositoryReference(location, rr.getName(), type, options);
});
}

private static URI getNormalizedLocation(RepositoryReference r) {
//P2 does the same before loading the repo and thus IRepository.getLocation() returns the normalized URL.
// In order to avoid stripping of slashes from URI instances do it now before URIs are created.
return URI.create(StringUtils.removeEnd(r.getLocation(), "/"));
}

@Override
protected List<IArtifactKey> collectArtifactKeys(Collection<IInstallableUnit> ius, IProgressMonitor monitor)
throws ProvisionException {
List<IArtifactKey> keys = super.collectArtifactKeys(ius, monitor);
if (isFilterProvidedItems()) {
removeProvidedItems(keys, getArtifactRepositoryManager(), monitor);
if (excludeNotStrictlyRequiredRepoReferences) {
removeUnncessaryRepositoryReferences();
}
}
return keys;
}

private Map<String, List<Version>> fullRepositoryContent;
private List<IRepository<IInstallableUnit>> metadataRepositories;

@Override
protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
removeProvidedItems(units, getMetadataRepositoryManager(), monitor);
fullRepositoryContent = units.stream()
.collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList())));
metadataRepositories = removeProvidedItems(units, getMetadataRepositoryManager(), monitor);
}
return units;
}
Expand All @@ -185,19 +216,68 @@ private boolean isFilterProvidedItems() {
return filterProvided && !repositoryReferences.isEmpty();
}

private <T> void removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
private <T> List<IRepository<T>> removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
try {
URI location = new URI(reference.getLocation());
URI location = getNormalizedLocation(reference);
IRepository<T> repository = loadRepository(repoManager, location, monitor);
referencedRepositories.add(repository);
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
} catch (IllegalArgumentException e) {
if (e.getCause() instanceof URISyntaxException uriException) {
throw new ProvisionException("Can't parse referenced URI!", uriException);
} else {
throw e;
}
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e)));
return referencedRepositories;
}

private void removeUnncessaryRepositoryReferences() {
List<Entry<URI, Set<IInstallableUnit>>> usedRepositoryItems = new ArrayList<>();
for (IRepository<IInstallableUnit> repo : metadataRepositories) {

// Assume that for all units that correspond to artifacts the metadata either has a co-located artifact repository or a references to to one that contains it.
IQueryResult<IInstallableUnit> allUnits = repo.query(QueryUtil.ALL_UNITS, null);
Set<IInstallableUnit> usedRepoContent = stream(allUnits).filter(a -> {
List<Version> fullClosureVersions = fullRepositoryContent.get(a.getId());
return fullClosureVersions != null && fullClosureVersions.contains(a.getVersion());
}).collect(Collectors.toSet());
usedRepositoryItems.add(Map.entry(repo.getLocation(), usedRepoContent));
}
// Remove references that contribute nothing or whose relevant content is also provided by another one
usedRepositoryItems.removeIf(repo -> {
Set<IInstallableUnit> content = repo.getValue();
return content.isEmpty() || usedRepositoryItems.stream().filter(e -> e != repo).map(Entry::getValue)
.anyMatch(c -> c.size() >= content.size() && c.containsAll(content));
});
Set<URI> retainedRepoLocations = usedRepositoryItems.stream().map(Entry::getKey).collect(Collectors.toSet());

removeRepositoryReferencesIf(getDestinationMetadataRepository(),
rr -> filterableRepositoryReferenceURIs.contains(rr.getLocation())
&& !retainedRepoLocations.contains(rr.getLocation()));
}

//TODO: Create a regular way for in P2 for that
private static void removeRepositoryReferencesIf(IMetadataRepository metadataRepository,
Predicate<IRepositoryReference> filter) {
if (metadataRepository instanceof LocalMetadataRepository localRepo) {
try {
Field repositoriesField = LocalMetadataRepository.class.getDeclaredField("repositories");
repositoriesField.trySetAccessible();
Method save = LocalMetadataRepository.class.getDeclaredMethod("save");
save.trySetAccessible();
@SuppressWarnings("unchecked")
Set<IRepositoryReference> repositories = (Set<IRepositoryReference>) repositoriesField.get(localRepo);
repositories.removeIf(filter);
save.invoke(localRepo);
} catch (ReflectiveOperationException e) {
throw new IllegalStateException("Failed to clean-up references from assembled repository", e);
}
}
}

//TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311
Expand Down Expand Up @@ -244,4 +324,8 @@ public void setFilterProvided(boolean filterProvided) {
this.filterProvided = filterProvided;
}

public void setExcludeNotStrictlyRequiredRepoReferences(boolean excludeNotStrictlyRequiredRepoReferences) {
this.excludeNotStrictlyRequiredRepoReferences = excludeNotStrictlyRequiredRepoReferences;
}

}
18 changes: 10 additions & 8 deletions tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,13 @@ public void testMirrorNothing() throws Exception {
Collection<DependencySeed> noSeeds = Collections.emptyList();

subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, noSeeds, context, false, false, false,
false, false, null);
false, false, false, null);
}

@Test
public void testMirrorFeatureWithContent() throws Exception {
subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false,
false, false, false, false, null);
false, false, false, false, false, null);

logVerifier.expectNoWarnings();
assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.4.0.v20080512.jar").exists());
Expand All @@ -121,9 +121,10 @@ public void testExtraArtifactRepositoryProperties() throws Exception {
extraArtifactRepositoryProperties.put("p2.mirrorsURL", "http://some.where.else");
extraArtifactRepositoryProperties.put("foo", "bar");
destinationRepo = new DestinationRepositoryDescriptor(tempFolder.newFolder("dest2"), DEFAULT_NAME, false, false,
false, false, true, extraArtifactRepositoryProperties, Collections.emptyList());
false, false, true, extraArtifactRepositoryProperties, Collections.emptyList(),
Collections.emptyList());
subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false,
false, false, false, false, null);
false, false, false, false, false, null);

logVerifier.expectNoWarnings();
File artifactsXml = repoFile(destinationRepo, "artifacts.xml");
Expand All @@ -146,7 +147,7 @@ public void testExtraArtifactRepositoryProperties() throws Exception {
@Test
public void testMirrorPatch() throws Exception {
subject.mirrorReactor(sourceRepos("patch", "e352"), destinationRepo, seedFor(FEATURE_PATCH_IU), context, false,
false, false, false, false, null);
false, false, false, false, false, null);

//TODO why mirror tool emits a warning here? logVerifier.expectNoWarnings();
assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.5.0.v20090525.jar").exists());
Expand All @@ -156,7 +157,7 @@ public void testMirrorPatch() throws Exception {
@Test
public void testMirrorFeatureAndPatch() throws Exception {
subject.mirrorReactor(sourceRepos("patch", "e352"), destinationRepo,
seedFor(SIMPLE_FEATURE_IU, FEATURE_PATCH_IU), context, false, false, false, false, false, null);
seedFor(SIMPLE_FEATURE_IU, FEATURE_PATCH_IU), context, false, false, false, false, false, false, null);

assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.5.0.v20090525.jar").exists());
assertTrue(repoFile(destinationRepo, "features/" + SIMPLE_FEATURE + "_1.0.0.jar").exists());
Expand All @@ -175,7 +176,7 @@ public void testMirrorWithMissingMandatoryContent() throws Exception {
* warning is issued.
*/
subject.mirrorReactor(sourceRepos("patch"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false, false,
false, false, false, null);
false, false, false, false, null);

logVerifier.expectWarning(not(is("")));
}
Expand All @@ -189,7 +190,8 @@ public void testMirrorForSeedWithNullIU() throws Exception {
List<DependencySeed> seeds = Collections
.singletonList(new DependencySeed(null, "org.eclipse.core.runtime", null));

subject.mirrorReactor(sourceRepos("e342"), destinationRepo, seeds, context, false, false, false, false, false, null);
subject.mirrorReactor(sourceRepos("e342"), destinationRepo, seeds, context, false, false, false, false, false,
false, null);

assertTrue(repoFile(destinationRepo, "plugins/org.eclipse.core.runtime_3.4.0.v20080512.jar").exists());
}
Expand Down
Loading

0 comments on commit f00fcf8

Please sign in to comment.