From 5cfd3a0cd70a3d45f4002f711b5b895b1f94a05c Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sun, 3 Sep 2023 01:17:20 +0200 Subject: [PATCH] Suppress addition of unnecessary repo-refereces when assembling p2-repos --- .../DestinationRepositoryDescriptor.java | 11 ++- .../facade/MirrorApplicationService.java | 4 +- .../p2tools/MirrorApplicationServiceImpl.java | 4 +- .../tycho/p2tools/TychoMirrorApplication.java | 97 +++++++++++++++++-- .../p2tools/MirrorApplicationServiceTest.java | 3 +- .../p2/repository/AssembleRepositoryMojo.java | 19 ++-- 6 files changed, 118 insertions(+), 20 deletions(-) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java index c434eefc7e..8edc208cd3 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/DestinationRepositoryDescriptor.java @@ -28,10 +28,12 @@ public class DestinationRepositoryDescriptor { private final boolean append; private final Map extraArtifactRepositoryProperties; private final List repositoryReferences; + private final List filterablRepositoryReferences; public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress, boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append, - Map extraArtifactRepositoryProperties, List repositoryReferences) { + Map extraArtifactRepositoryProperties, List repositoryReferences, + List filterablRepositoryReferences) { this.location = location; this.name = name; this.compress = compress; @@ -41,12 +43,13 @@ public DestinationRepositoryDescriptor(File location, String name, boolean compr this.append = append; this.extraArtifactRepositoryProperties = extraArtifactRepositoryProperties; this.repositoryReferences = repositoryReferences; + this.filterablRepositoryReferences = filterablRepositoryReferences; } public DestinationRepositoryDescriptor(File location, String name, boolean compress, boolean xzCompress, boolean keepNonXzIndexFiles, boolean metaDataOnly, boolean append) { this(location, name, compress, xzCompress, keepNonXzIndexFiles, metaDataOnly, append, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); } public DestinationRepositoryDescriptor(File location, String name) { @@ -88,4 +91,8 @@ public Map getExtraArtifactRepositoryProperties() { public List getRepositoryReferences() { return repositoryReferences == null ? Collections.emptyList() : repositoryReferences; } + + public List getFilterableRepositoryReferences() { + return filterablRepositoryReferences; + } } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java index 4bc7a855f4..5a309f6ad3 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2/tools/mirroring/facade/MirrorApplicationService.java @@ -55,6 +55,7 @@ public interface MirrorApplicationService { * @param includeRequiredFeatures * Whether to include features mentioned in the require section of a feature * @param filterProvided Whether to filter IU/artifacts that are already provided by a referenced repository + * @param excludeNotStrictlyRequiredRepoReferences Whether to exclude repository-references that don't contribute any IU from being added as such. * @param filterProperties * additional filter properties to be set in the p2 slicing options. May be * null @@ -64,7 +65,8 @@ public interface MirrorApplicationService { public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination, Collection seeds, BuildContext context, boolean includeAllDependencies, boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures, - boolean filterProvided, Map filterProperties) throws FacadeException; + boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences, + Map filterProperties) throws FacadeException; /** * recreates the metadata of an existing repository e.g. to account for changes in the contained diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java index cc249ffaf9..b5705cbc14 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java @@ -153,7 +153,8 @@ private static IQuery createQuery(IUDescription iu) { public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDescriptor destination, Collection projectSeeds, BuildContext context, boolean includeAllDependencies, boolean includeAllSource, boolean includeRequiredBundles, boolean includeRequiredFeatures, - boolean filterProvided, Map filterProperties) throws FacadeException { + boolean filterProvided, boolean excludeNotStrictlyRequiredRepoReferences, + Map filterProperties) throws FacadeException { final TychoMirrorApplication mirrorApp = createMirrorApplication(sources, destination, agent); // mirror scope: seed units... @@ -164,6 +165,7 @@ public void mirrorReactor(RepositoryReferences sources, DestinationRepositoryDes mirrorApp.setIncludeRequiredFeatures(includeRequiredFeatures); mirrorApp.setIncludePacked(false); // no way, Tycho do no longer support packed artifacts anyways mirrorApp.setFilterProvided(filterProvided); + mirrorApp.setExcludeNotStrictlyRequiredRepoReferences(excludeNotStrictlyRequiredRepoReferences); // TODO the p2 mirror tool should support mirroring multiple environments at once for (TargetEnvironment environment : context.getEnvironments()) { SlicingOptions options = new SlicingOptions(); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java index a70ea329e3..8cf7662604 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/TychoMirrorApplication.java @@ -9,11 +9,15 @@ * * Contributors: * SAP SE - initial API and implementation + * Hannes Wellmann - Implement user-defined filtering and filtering based on relevance for automatically added repo-references *******************************************************************************/ package org.eclipse.tycho.p2tools; import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -21,14 +25,19 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.apache.commons.lang3.StringUtils; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.equinox.internal.p2.director.PermissiveSlicer; import org.eclipse.equinox.internal.p2.director.Slicer; import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability; +import org.eclipse.equinox.internal.p2.metadata.repository.LocalMetadataRepository; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor; @@ -36,13 +45,17 @@ import org.eclipse.equinox.p2.metadata.IArtifactKey; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IRequirement; +import org.eclipse.equinox.p2.metadata.Version; +import org.eclipse.equinox.p2.metadata.expression.ExpressionUtil; import org.eclipse.equinox.p2.metadata.expression.IMatchExpression; import org.eclipse.equinox.p2.query.CollectionResult; +import org.eclipse.equinox.p2.query.IQuery; import org.eclipse.equinox.p2.query.IQueryResult; import org.eclipse.equinox.p2.query.IQueryable; import org.eclipse.equinox.p2.query.QueryUtil; import org.eclipse.equinox.p2.repository.IRepository; import org.eclipse.equinox.p2.repository.IRepositoryManager; +import org.eclipse.equinox.p2.repository.IRepositoryReference; import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository; @@ -57,16 +70,27 @@ public class TychoMirrorApplication extends org.eclipse.tycho.p2tools.copiedfrom private static final String FEATURE_GROUP = ".feature.group"; private final Map extraArtifactRepositoryProperties; private final List repositoryReferences; + private final Set filterableRepositoryReferenceURIs; private boolean includeAllSource; private boolean includeRequiredBundles; private boolean includeRequiredFeatures; private boolean filterProvided; + private boolean excludeNotStrictlyRequiredRepoReferences; private TargetPlatform targetPlatform; public TychoMirrorApplication(IProvisioningAgent agent, DestinationRepositoryDescriptor destination) { super(agent); this.extraArtifactRepositoryProperties = destination.getExtraArtifactRepositoryProperties(); - this.repositoryReferences = destination.getRepositoryReferences(); + this.filterableRepositoryReferenceURIs = destination.getFilterableRepositoryReferences().stream() + .map(r -> StringUtils.removeEnd(r.getLocation(), "/")).map(URI::create).collect(Collectors.toSet()); + this.repositoryReferences = Stream + .of(destination.getRepositoryReferences(), destination.getFilterableRepositoryReferences()) + .flatMap(List::stream).map( + r -> r.getLocation().endsWith("/") + ? new RepositoryReference(r.getName(), StringUtils.removeEnd(r.getLocation(), "/"), + r.isEnable()) + : r) + .toList(); this.removeAddedRepositories = false; } @@ -166,16 +190,66 @@ protected List collectArtifactKeys(Collection iu throws ProvisionException { List keys = super.collectArtifactKeys(ius, monitor); if (isFilterProvidedItems()) { - removeProvidedItems(keys, getArtifactRepositoryManager(), monitor); + var repos = removeProvidedItems(keys, getArtifactRepositoryManager(), monitor); + if (excludeNotStrictlyRequiredRepoReferences) { + removeUnncessaryRepositoryReferences(repos); + } } return keys; } + private static final IQuery ALL_ARTIFACTS = QueryUtil.createMatchQuery(// + IArtifactKey.class, ExpressionUtil.TRUE_EXPRESSION); + + private void removeUnncessaryRepositoryReferences(List>> repositoryItems) { + List>> usedRepositoryItems = new ArrayList<>(); + for (Entry> repo : repositoryItems) { + + IQueryResult allArtifacts = repo.getValue().query(ALL_ARTIFACTS, null); + Set usedRepoContent = stream(allArtifacts).filter(a -> { + List fullClosureVersions = fullRepositoryContent.get(a.getId()); + return fullClosureVersions != null && fullClosureVersions.contains(a.getVersion()); + }).collect(Collectors.toSet()); + usedRepositoryItems.add(Map.entry(repo.getKey(), usedRepoContent)); + } + // Remove references that contribute nothing or whose relevant content is also provided by another one + usedRepositoryItems.removeIf(repo -> { + Set content = repo.getValue(); + return content.isEmpty() || usedRepositoryItems.stream().filter(e -> e != repo).map(Entry::getValue) + .anyMatch(c -> c.size() >= content.size() && c.containsAll(content)); + }); + Set retainedRepoLocations = usedRepositoryItems.stream().map(Entry::getKey).collect(Collectors.toSet()); + + removeRepositoryReferencesIf(getDestinationMetadataRepository(), + rr -> filterableRepositoryReferenceURIs.contains(rr.getLocation()) + && !retainedRepoLocations.contains(rr.getLocation())); + } + + private static void removeRepositoryReferencesIf(IMetadataRepository metadataRepository, + Predicate filter) { + if (metadataRepository instanceof LocalMetadataRepository localRepo) { + try { + Field repositoriesField = LocalMetadataRepository.class.getDeclaredField("repositories"); + repositoriesField.trySetAccessible(); + @SuppressWarnings("unchecked") + Set repositories = (Set) repositoriesField.get(localRepo); + repositories.removeIf(filter); +// localRepo.addReferences(List.of()); //TODO: Trigger a save; + + } catch (ReflectiveOperationException e) { // ignore + } + } + } + + private Map> fullRepositoryContent; + @Override protected Set collectUnits(IQueryable slice, IProgressMonitor monitor) throws ProvisionException { Set units = super.collectUnits(slice, monitor); if (isFilterProvidedItems()) { + fullRepositoryContent = units.stream() + .collect(groupingBy(IInstallableUnit::getId, mapping(IInstallableUnit::getVersion, toList()))); removeProvidedItems(units, getMetadataRepositoryManager(), monitor); } return units; @@ -185,19 +259,20 @@ private boolean isFilterProvidedItems() { return filterProvided && !repositoryReferences.isEmpty(); } - private void removeProvidedItems(Collection allElements, IRepositoryManager repoManager, - IProgressMonitor monitor) throws ProvisionException { - List> referencedRepositories = new ArrayList<>(); + private List>> removeProvidedItems(Collection allElements, + IRepositoryManager repoManager, IProgressMonitor monitor) throws ProvisionException { + List>> referencedRepositories = new ArrayList<>(); for (RepositoryReference reference : repositoryReferences) { try { URI location = new URI(reference.getLocation()); IRepository repository = loadRepository(repoManager, location, monitor); - referencedRepositories.add(repository); + referencedRepositories.add(Map.entry(location, repository)); } catch (URISyntaxException e) { throw new ProvisionException("Can't parse referenced URI!", e); } } - allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo, e))); + allElements.removeIf(e -> referencedRepositories.stream().anyMatch(repo -> contains(repo.getValue(), e))); + return referencedRepositories; } //TODO: just call IRepositoryManager.loadRepository() once available: https://github.com/eclipse-equinox/p2/pull/311 @@ -236,12 +311,16 @@ public void setIncludeRequiredBundles(boolean includeRequiredBundles) { this.includeRequiredBundles = includeRequiredBundles; } + public void setIncludeRequiredFeatures(boolean includeRequiredFeatures) { + this.includeRequiredFeatures = includeRequiredFeatures; + } + public void setFilterProvided(boolean filterProvided) { this.filterProvided = filterProvided; } - public void setIncludeRequiredFeatures(boolean includeRequiredFeatures) { - this.includeRequiredFeatures = includeRequiredFeatures; + public void setExcludeNotStrictlyRequiredRepoReferences(boolean excludeNotStrictlyRequiredRepoReferences) { + this.excludeNotStrictlyRequiredRepoReferences = excludeNotStrictlyRequiredRepoReferences; } } diff --git a/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java b/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java index fcfc787e5a..5a97bd11ad 100644 --- a/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java +++ b/tycho-core/src/test/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceTest.java @@ -121,7 +121,8 @@ public void testExtraArtifactRepositoryProperties() throws Exception { extraArtifactRepositoryProperties.put("p2.mirrorsURL", "http://some.where.else"); extraArtifactRepositoryProperties.put("foo", "bar"); destinationRepo = new DestinationRepositoryDescriptor(tempFolder.newFolder("dest2"), DEFAULT_NAME, false, false, - false, false, true, extraArtifactRepositoryProperties, Collections.emptyList()); + false, false, true, extraArtifactRepositoryProperties, Collections.emptyList(), + Collections.emptyList()); subject.mirrorReactor(sourceRepos("patch", "e342"), destinationRepo, seedFor(SIMPLE_FEATURE_IU), context, false, false, false, false, false, null); diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java index 2d4dcbdfe7..ec893377cc 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java @@ -21,7 +21,6 @@ import java.util.Map; import java.util.function.Predicate; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -75,6 +74,12 @@ public class AssembleRepositoryMojo extends AbstractRepositoryMojo { public static class RepositoryReferenceFilter { + /** + * If repository references providing no relevant content or only content also provided by + * others, should be excluded from being added automatically as referenced repositories of + * the assembled repository. + */ + boolean excludeNotStrictlyRequired = true; /** The list of location patterns that exclude matching repository references. */ List exclude = List.of(); /** @@ -309,14 +314,15 @@ public void execute() throws MojoExecutionException, MojoFailureException { .map(Category::getRepositoryReferences)// .flatMap(List::stream)// .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// - .collect(Collectors.toCollection(ArrayList::new)); + .toList(); Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); + List autoRepositoryRefeferences = new ArrayList<>(); if (addPomRepositoryReferences) { getProject().getRepositories().stream() // .filter(pomRepo -> "p2".equals(pomRepo.getLayout())) .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl())) .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)) - .forEach(repositoryReferences::add); + .forEach(autoRepositoryRefeferences::add); } if (addIUTargetRepositoryReferences) { projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream() @@ -325,14 +331,15 @@ public void execute() throws MojoExecutionException, MojoFailureException { .flatMap(iu -> iu.getRepositories().stream()) .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation())) .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true)) - .forEach(repositoryReferences::add); + .forEach(autoRepositoryRefeferences::add); } DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles, - !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences); + !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences, + autoRepositoryRefeferences); mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(), includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures, - filterProvided, profileProperties); + filterProvided, repositoryReferenceFilter.excludeNotStrictlyRequired, profileProperties); if (generateOSGiRepository) { XMLResourceGenerator resourceGenerator = new XMLResourceGenerator(); resourceGenerator.name(repositoryName);