Skip to content

Commit

Permalink
Suppress addition of unnecessary repo-refereces when assembling p2-repos
Browse files Browse the repository at this point in the history
  • Loading branch information
@HannesWell
HannesWell committed Sep 5, 2023
1 parent f963c10 commit 5cfd3a0
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 20 deletions.
11 changes: 9 additions & 2 deletions tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ public class DestinationRepositoryDescriptor {
private final boolean append;
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final List<RepositoryReference> filterablRepositoryReferences;

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append,
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences) {
Map<String, String> extraArtifactRepositoryProperties, List<RepositoryReference> repositoryReferences,
List<RepositoryReference> filterablRepositoryReferences) {
this.location = location;
this.name = name;
this.compress = compress;
Expand All @@ -41,12 +43,13 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr
this.append = append;
this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties;
this.repositoryReferences = repositoryReferences;
this.filterablRepositoryReferences = filterablRepositoryReferences;
}

public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress,
boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) {
this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(),
Collections.emptyList());
Collections.emptyList(), Collections.emptyList());
}

public DestinationRepositoryDescriptor(File location, String name) {
Expand Down Expand Up @@ -88,4 +91,8 @@ public Map<String, String> getExtraArtifactRepositoryProperties() {
public List<RepositoryReference> getRepositoryReferences() {
return repositoryReferences == null ? Collections.emptyList() : repositoryReferences;
}

public List<RepositoryReference> getFilterableRepositoryReferences() {
return filterablRepositoryReferences;
}
}
4 changes: 3 additions & 1 deletion ...e/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public interface MirrorApplicationService {
* @param includeRequiredFeatures
* Whether to include features mentioned in the require section of a feature
* @param filterProvided Whether to filter IU/artifacts that are already provided by a referenced repository
* @param excludeNotStrictlyRequiredRepoReferences Whether to exclude repository-references that don't contribute any IU from being added as such.
* @param filterProperties
* additional filter properties to be set in the p2 slicing options. May be
* <code>null</code>
Expand All @@ -64,7 +65,8 @@ public interface MirrorApplicationService {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> seeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException;
boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences,
Map<String, String> filterProperties) throws FacadeException;

/**
* recreates the metadata of an existing repository e.g. to account for changes in the contained
Expand Down
4 changes: 3 additions & 1 deletion tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,8 @@ private static IQuery<IInstallableUnit> createQuery(IUDescription iu) {
public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination,
Collection<DependencySeed> projectSeeds, BuildContext context, boolean includeAllDependencies,
boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures,
boolean filterProvided, Map<String, String> filterProperties) throws FacadeException {
boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences,
Map<String, String> filterProperties) throws FacadeException {
final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent);

// mirror scope: seed units...
Expand All @@ -164,6 +165,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes
mirrorApp.setIncludeRequiredFeatures(includeRequiredFeatures);
mirrorApp.setIncludePacked(false); // no way, Tycho do no longer support packed artifacts anyways
mirrorApp.setFilterProvided(filterProvided);
mirrorApp.setExcludeNotStrictlyRequiredRepoReferences(excludeNotStrictlyRequiredRepoReferences);
// TODO the p2 mirror tool should support mirroring multiple environments at once
for (TargetEnvironment environment : context.getEnvironments()) {
SlicingOptions options = new SlicingOptions();
Expand Down
97 changes: 88 additions & 9 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,53 @@
*
* Contributors:
* SAP SE - initial API and implementation
* Hannes Wellmann - Implement user-defined filtering and filtering based on relevance for automatically added repo-references
*******************************************************************************/
package org.eclipse.tycho.p2tools;

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;

import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import org.apache.commons.lang3.StringUtils;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.equinox.internal.p2.director.PermissiveSlicer;
import org.eclipse.equinox.internal.p2.director.Slicer;
import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
import org.eclipse.equinox.internal.p2.metadata.repository.LocalMetadataRepository;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor;
import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions;
import org.eclipse.equinox.p2.metadata.IArtifactKey;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.metadata.IRequirement;
import org.eclipse.equinox.p2.metadata.Version;
import org.eclipse.equinox.p2.metadata.expression.ExpressionUtil;
import org.eclipse.equinox.p2.metadata.expression.IMatchExpression;
import org.eclipse.equinox.p2.query.CollectionResult;
import org.eclipse.equinox.p2.query.IQuery;
import org.eclipse.equinox.p2.query.IQueryResult;
import org.eclipse.equinox.p2.query.IQueryable;
import org.eclipse.equinox.p2.query.QueryUtil;
import org.eclipse.equinox.p2.repository.IRepository;
import org.eclipse.equinox.p2.repository.IRepositoryManager;
import org.eclipse.equinox.p2.repository.IRepositoryReference;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
Expand All @@ -57,16 +70,27 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom
private static final String FEATURE_GROUP = ".feature.group";
private final Map<String, String> extraArtifactRepositoryProperties;
private final List<RepositoryReference> repositoryReferences;
private final Set<URI> filterableRepositoryReferenceURIs;
private boolean includeAllSource;
private boolean includeRequiredBundles;
private boolean includeRequiredFeatures;
private boolean filterProvided;
private boolean excludeNotStrictlyRequiredRepoReferences;
private TargetPlatform targetPlatform;

public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) {
super(agent);
this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties();
this.repositoryReferences = destination.getRepositoryReferences();
this.filterableRepositoryReferenceURIs = destination.getFilterableRepositoryReferences().stream()
.map(r -> StringUtils.removeEnd(r.getLocation(), "/")).map(URI::create).collect(Collectors.toSet());
this.repositoryReferences = Stream
.of(destination.getRepositoryReferences(), destination.getFilterableRepositoryReferences())
.flatMap(List::stream).map(
r -> r.getLocation().endsWith("/")
? new RepositoryReference(r.getName(), StringUtils.removeEnd(r.getLocation(), "/"),
r.isEnable())
: r)
.toList();
this.removeAddedRepositories = false;
}

Expand Down Expand Up @@ -166,16 +190,66 @@ protected List<IArtifactKey> collectArtifactKeys(Collection<IInstallableUnit> iu
throws ProvisionException {
List<IArtifactKey> keys = super.collectArtifactKeys(ius, monitor);
if (isFilterProvidedItems()) {
removeProvidedItems(keys, getArtifactRepositoryManager(), monitor);
var repos = removeProvidedItems(keys, getArtifactRepositoryManager(), monitor);
if (excludeNotStrictlyRequiredRepoReferences) {
removeUnncessaryRepositoryReferences(repos);
}
}
return keys;
}

private static final IQuery<IArtifactKey> ALL_ARTIFACTS = QueryUtil.createMatchQuery(//
IArtifactKey.class, ExpressionUtil.TRUE_EXPRESSION);

private void removeUnncessaryRepositoryReferences(List<Entry<URI, IRepository<IArtifactKey>>> repositoryItems) {
List<Entry<URI, Set<IArtifactKey>>> usedRepositoryItems = new ArrayList<>();
for (Entry<URI, IRepository<IArtifactKey>> repo : repositoryItems) {

IQueryResult<IArtifactKey> allArtifacts = repo.getValue().query(ALL_ARTIFACTS, null);
Set<IArtifactKey> usedRepoContent = stream(allArtifacts).filter(a -> {
List<Version> fullClosureVersions = fullRepositoryContent.get(a.getId());
return fullClosureVersions != null && fullClosureVersions.contains(a.getVersion());
}).collect(Collectors.toSet());
usedRepositoryItems.add(Map.entry(repo.getKey(), usedRepoContent));
}
// Remove references that contribute nothing or whose relevant content is also provided by another one
usedRepositoryItems.removeIf(repo -> {
Set<IArtifactKey> content = repo.getValue();
return content.isEmpty() || usedRepositoryItems.stream().filter(e -> e != repo).map(Entry::getValue)
.anyMatch(c -> c.size() >= content.size() && c.containsAll(content));
});
Set<URI> retainedRepoLocations = usedRepositoryItems.stream().map(Entry::getKey).collect(Collectors.toSet());

removeRepositoryReferencesIf(getDestinationMetadataRepository(),
rr -> filterableRepositoryReferenceURIs.contains(rr.getLocation())
&& !retainedRepoLocations.contains(rr.getLocation()));
}

private static void removeRepositoryReferencesIf(IMetadataRepository metadataRepository,
Predicate<IRepositoryReference> filter) {
if (metadataRepository instanceof LocalMetadataRepository localRepo) {
try {
Field repositoriesField = LocalMetadataRepository.class.getDeclaredField("repositories");
repositoriesField.trySetAccessible();
@SuppressWarnings("unchecked")
Set<IRepositoryReference> repositories = (Set<IRepositoryReference>) repositoriesField.get(localRepo);
repositories.removeIf(filter);
// localRepo.addReferences(List.of()); //TODO: Trigger a save;

} catch (ReflectiveOperationException e) { // ignore
}
}
}

private Map<String, List<Version>> fullRepositoryContent;

@Override
protected Set<IInstallableUnit> collectUnits(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)
throws ProvisionException {
Set<IInstallableUnit> units = super.collectUnits(slice, monitor);
if (isFilterProvidedItems()) {
fullRepositoryContent = units.stream()
.collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList())));
removeProvidedItems(units, getMetadataRepositoryManager(), monitor);
}
return units;
Expand All @@ -185,19 +259,20 @@ private boolean isFilterProvidedItems() {
return filterProvided && !repositoryReferences.isEmpty();
}

private <T> void removeProvidedItems(Collection<T> allElements, IRepositoryManager<T> repoManager,
IProgressMonitor monitor) throws ProvisionException {
List<IRepository<T>> referencedRepositories = new ArrayList<>();
private <T> List<Entry<URI, IRepository<T>>> removeProvidedItems(Collection<T> allElements,
IRepositoryManager<T> repoManager, IProgressMonitor monitor) throws ProvisionException {
List<Entry<URI, IRepository<T>>> referencedRepositories = new ArrayList<>();
for (RepositoryReference reference : repositoryReferences) {
try {
URI location = new URI(reference.getLocation());
IRepository<T> repository = loadRepository(repoManager, location, monitor);
referencedRepositories.add(repository);
referencedRepositories.add(Map.entry(location, repository));
} catch (URISyntaxException e) {
throw new ProvisionException("Can't parse referenced URI!", e);
}
}
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e)));
allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo.getValue(), e)));
return referencedRepositories;
}

//TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311
Expand Down Expand Up @@ -236,12 +311,16 @@ public void setIncludeRequiredBundles(boolean includeRequiredBundles) {
this.includeRequiredBundles = includeRequiredBundles;
}

public void setIncludeRequiredFeatures(boolean includeRequiredFeatures) {
this.includeRequiredFeatures = includeRequiredFeatures;
}

public void setFilterProvided(boolean filterProvided) {
this.filterProvided = filterProvided;
}

public void setIncludeRequiredFeatures(boolean includeRequiredFeatures) {
this.includeRequiredFeatures = includeRequiredFeatures;
public void setExcludeNotStrictlyRequiredRepoReferences(boolean excludeNotStrictlyRequiredRepoReferences) {
this.excludeNotStrictlyRequiredRepoReferences = excludeNotStrictlyRequiredRepoReferences;
}

}
3 changes: 2 additions & 1 deletion tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ public void testExtraArtifactRepositoryProperties() throws Exception {
extraArtifactRepositoryProperties.put("p2.mirrorsURL", "http://some.where.else");
extraArtifactRepositoryProperties.put("foo", "bar");
destinationRepo = new DestinationRepositoryDescriptor(tempFolder.newFolder("dest2"), DEFAULT_NAME, false, false,
false, false, true, extraArtifactRepositoryProperties, Collections.emptyList());
false, false, true, extraArtifactRepositoryProperties, Collections.emptyList(),
Collections.emptyList());
subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false,
false, false, false, false, null);

Expand Down
19 changes: 13 additions & 6 deletions ...-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.Map;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
Expand Down Expand Up @@ -75,6 +74,12 @@
public class AssembleRepositoryMojo extends AbstractRepositoryMojo {

public static class RepositoryReferenceFilter {
/**
* If repository references providing no relevant content or only content also provided by
* others, should be excluded from being added automatically as referenced repositories of
* the assembled repository.
*/
boolean excludeNotStrictlyRequired = true;
/** The list of location patterns that exclude matching repository references. */
List<String> exclude = List.of();
/**
Expand Down Expand Up @@ -309,14 +314,15 @@ public void execute() throws MojoExecutionException, MojoFailureException {
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.collect(Collectors.toCollection(ArrayList::new));
.toList();
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
List<RepositoryReference> autoRepositoryRefeferences = new ArrayList<>();
if (addPomRepositoryReferences) {
getProject().getRepositories().stream() //
.filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
.filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
.map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
.forEach(repositoryReferences::add);
.forEach(autoRepositoryRefeferences::add);
}
if (addIUTargetRepositoryReferences) {
projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
Expand All @@ -325,14 +331,15 @@ public void execute() throws MojoExecutionException, MojoFailureException {
.flatMap(iu -> iu.getRepositories().stream())
.filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
.map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
.forEach(repositoryReferences::add);
.forEach(autoRepositoryRefeferences::add);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles,
!createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences);
!createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences,
autoRepositoryRefeferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, profileProperties);
filterProvided, repositoryReferenceFilter.excludeNotStrictlyRequired, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
Expand Down

0 comments on commit 5cfd3a0

Please sign in to comment.