From fae50e2e1ca641dded8ccd814409f0be6c3721ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 22 Jan 2024 10:45:58 +0100 Subject: [PATCH 1/4] Fork QueryableArray from P2 --- .../p2tools/copiedfromp2/QueryableArray.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java new file mode 100644 index 0000000000..ea5696f60b --- /dev/null +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright (c) 2008, 2023 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Cloudsmith Inc. - query indexes + *******************************************************************************/ +package org.eclipse.tycho.p2tools.copiedfromp2; + +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.eclipse.equinox.internal.p2.core.helpers.CollectionUtils; +import org.eclipse.equinox.internal.p2.metadata.InstallableUnit; +import org.eclipse.equinox.internal.p2.metadata.TranslationSupport; +import org.eclipse.equinox.internal.p2.metadata.index.CapabilityIndex; +import org.eclipse.equinox.internal.p2.metadata.index.IdIndex; +import org.eclipse.equinox.internal.p2.metadata.index.IndexProvider; +import org.eclipse.equinox.p2.metadata.IInstallableUnit; +import org.eclipse.equinox.p2.metadata.KeyWithLocale; +import org.eclipse.equinox.p2.metadata.index.IIndex; + +public class QueryableArray extends IndexProvider { + private final List dataSet; + private IIndex capabilityIndex; + private IIndex idIndex; + private TranslationSupport translationSupport; + + public QueryableArray(IInstallableUnit[] ius) { + dataSet = CollectionUtils.unmodifiableList(ius); + } + + public QueryableArray(Collection ius) { + dataSet = List.copyOf(ius); + } + + @Override + public Iterator everything() { + return dataSet.iterator(); + } + + @Override + public boolean contains(IInstallableUnit element) { + return dataSet.contains(element); + } + + @Override + public synchronized IIndex getIndex(String memberName) { + if (InstallableUnit.MEMBER_PROVIDED_CAPABILITIES.equals(memberName)) { + if (capabilityIndex == null) + capabilityIndex = new CapabilityIndex(dataSet.iterator()); + return capabilityIndex; + } + if (InstallableUnit.MEMBER_ID.equals(memberName)) { + if (idIndex == null) + idIndex = new IdIndex(dataSet.iterator()); + return idIndex; + } + return null; + } + + @Override + public synchronized Object getManagedProperty(Object client, String memberName, Object key) { + if (!(client instanceof IInstallableUnit)) + return null; + IInstallableUnit iu = (IInstallableUnit) client; + if (InstallableUnit.MEMBER_TRANSLATED_PROPERTIES.equals(memberName)) { + if (translationSupport == null) + translationSupport = new TranslationSupport(this); + return key instanceof KeyWithLocale ? translationSupport.getIUProperty(iu, (KeyWithLocale) key) + : translationSupport.getIUProperty(iu, key.toString()); + } + return null; + } +} From f60ce115ca8a71b548321d16ef29159a7ebac0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 22 Jan 2024 11:07:03 +0100 Subject: [PATCH 2/4] Speed up assembly of repositories / products --- .../ImmutableInMemoryMetadataRepository.java | 18 ++++-------------- .../AbstractSlicerResolutionStrategy.java | 6 +++--- .../tycho/p2resolver/DependencyCollector.java | 4 ++-- .../p2resolver/FinalTargetPlatformImpl.java | 4 +++- .../tycho/p2resolver/P2ResolverImpl.java | 4 ++-- .../p2resolver/PomInstallableUnitStore.java | 4 ++-- .../ProjectorResolutionStrategy.java | 4 ++-- .../p2resolver/TargetPlatformFactoryImpl.java | 4 ++-- .../p2tools/copiedfromp2/QueryableArray.java | 11 +++++++---- .../tycho/p2tools/copiedfromp2/Slicer.java | 5 ++--- .../tycho/p2resolver/PublisherServiceTest.java | 2 +- 11 files changed, 30 insertions(+), 36 deletions(-) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/ImmutableInMemoryMetadataRepository.java b/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/ImmutableInMemoryMetadataRepository.java index e9b60b8603..2d506b7876 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/ImmutableInMemoryMetadataRepository.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2/repository/ImmutableInMemoryMetadataRepository.java @@ -24,28 +24,18 @@ import org.eclipse.core.runtime.IStatus; import org.eclipse.core.runtime.OperationCanceledException; import org.eclipse.core.runtime.Status; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.p2.core.IPool; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.metadata.IInstallableUnit; -import org.eclipse.equinox.p2.query.IQuery; -import org.eclipse.equinox.p2.query.IQueryResult; -import org.eclipse.equinox.p2.query.IQueryable; import org.eclipse.equinox.p2.repository.IRepositoryReference; import org.eclipse.equinox.p2.repository.IRunnableWithProgress; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; -public class ImmutableInMemoryMetadataRepository implements IMetadataRepository { +public class ImmutableInMemoryMetadataRepository extends QueryableArray implements IMetadataRepository { - private final IQueryable units; - - public ImmutableInMemoryMetadataRepository(Set units) { - this.units = new QueryableArray(units); - } - - @Override - public IQueryResult query(IQuery query, IProgressMonitor monitor) { - return units.query(query, monitor); + public ImmutableInMemoryMetadataRepository(Set units, boolean copy) { + super(units, copy); } @Override diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/AbstractSlicerResolutionStrategy.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/AbstractSlicerResolutionStrategy.java index f2d64c9a95..7f38eebcc3 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/AbstractSlicerResolutionStrategy.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/AbstractSlicerResolutionStrategy.java @@ -27,7 +27,6 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.MultiStatus; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability; import org.eclipse.equinox.internal.p2.metadata.RequiredCapability; import org.eclipse.equinox.internal.p2.metadata.RequiredPropertiesMatch; @@ -47,6 +46,7 @@ import org.eclipse.tycho.core.shared.MavenLogger; import org.eclipse.tycho.core.shared.StatusTool; import org.eclipse.tycho.p2.resolver.ResolverException; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; import org.eclipse.tycho.p2tools.copiedfromp2.Slicer; abstract class AbstractSlicerResolutionStrategy extends AbstractResolutionStrategy { @@ -94,9 +94,9 @@ protected final IQueryable slice(Map propertie seedIUs.add(createUnitRequiring("tycho-ee", null, data.getEEResolutionHints().getMandatoryRequires())); } - IQueryable baseIUCollection = new QueryableArray(availableIUs); + IQueryable baseIUCollection = new QueryableArray(availableIUs, false); Slicer slicer = newSlicer((query, monitor1) -> { - +// IQueryResult queryResult = baseIUCollection.query(query, monitor1); if (queryResult.isEmpty()) { IQueryable additionalUnitStore = data.getAdditionalUnitStore(); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/DependencyCollector.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/DependencyCollector.java index 80f327cd79..379ecc873b 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/DependencyCollector.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/DependencyCollector.java @@ -21,7 +21,6 @@ import org.eclipse.core.runtime.IStatus; import org.eclipse.core.runtime.MultiStatus; import org.eclipse.core.runtime.Status; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.internal.p2.metadata.RequiredCapability; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.metadata.IInstallableUnit; @@ -31,6 +30,7 @@ import org.eclipse.equinox.p2.query.QueryUtil; import org.eclipse.tycho.core.shared.MavenLogger; import org.eclipse.tycho.p2.publisher.FeatureDependenciesAction; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; public class DependencyCollector extends AbstractResolutionStrategy { @@ -51,7 +51,7 @@ public Collection resolve(Map properties, IPro result.addAll(data.getRootIUs()); - IQueryable availableUIsQueryable = new QueryableArray(data.getAvailableIUs()); + IQueryable availableUIsQueryable = new QueryableArray(data.getAvailableIUs(), false); for (IInstallableUnit iu : data.getRootIUs()) { collectIncludedIUs(availableUIsQueryable, result, errors, iu, true, monitor); } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/FinalTargetPlatformImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/FinalTargetPlatformImpl.java index fd15069b48..caa3cc6c63 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/FinalTargetPlatformImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/FinalTargetPlatformImpl.java @@ -30,6 +30,7 @@ public class FinalTargetPlatformImpl extends TargetPlatformBaseImpl { private IArtifactRepository artifactRepository; + private IMetadataRepository metadataRepository; public FinalTargetPlatformImpl(LinkedHashSet installableUnits, ExecutionEnvironmentResolutionHints executionEnvironment, IRawArtifactFileProvider jointArtifacts, @@ -39,6 +40,7 @@ public FinalTargetPlatformImpl(LinkedHashSet installableUnits, super(installableUnits, executionEnvironment, jointArtifacts, localArtifactRepository, reactorProjectLookup, mavenArtifactLookup, shadowed); this.artifactRepository = artifactRepository; + this.metadataRepository = new ImmutableInMemoryMetadataRepository(installableUnits, false); } @Override @@ -48,7 +50,7 @@ public void reportUsedLocalIUs(Collection usedUnits) { @Override public IMetadataRepository getMetadataRepository() { - return new ImmutableInMemoryMetadataRepository(installableUnits); + return metadataRepository; } @Override diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2ResolverImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2ResolverImpl.java index e02bc5fbd3..21db213c82 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2ResolverImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2ResolverImpl.java @@ -34,7 +34,6 @@ import org.apache.felix.resolver.util.CopyOnWriteSet; import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.p2.metadata.IArtifactKey; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IProvidedCapability; @@ -75,6 +74,7 @@ import org.eclipse.tycho.p2.resolver.ResolverException; import org.eclipse.tycho.p2.target.facade.TargetPlatformConfigurationStub; import org.eclipse.tycho.p2.target.facade.TargetPlatformFactory; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; import org.eclipse.tycho.p2tools.copiedfromp2.Slicer; import org.eclipse.tycho.targetplatform.P2TargetPlatform; @@ -329,7 +329,7 @@ public List getAdditionalRequirements() { public P2ResolutionResult resolveInstallableUnit(TargetPlatform context, String id, String versionRange) { P2TargetPlatform targetPlatform = getTargetFromContext(context); - IQueryable queriable = new QueryableArray(targetPlatform.getInstallableUnits()); + IQueryable queriable = new QueryableArray(targetPlatform.getInstallableUnits(), false); VersionRange range = new VersionRange(versionRange); IRequirement requirement = MetadataFactory.createRequirement(IInstallableUnit.NAMESPACE_IU_ID, id, range, null, diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/PomInstallableUnitStore.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/PomInstallableUnitStore.java index 577da78ab8..833f07ce0c 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/PomInstallableUnitStore.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/PomInstallableUnitStore.java @@ -31,7 +31,6 @@ import org.apache.maven.project.MavenProject; import org.codehaus.plexus.logging.Logger; import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.publisher.IPublisherInfo; import org.eclipse.equinox.p2.publisher.PublisherInfo; @@ -49,6 +48,7 @@ import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper; import org.eclipse.tycho.p2.resolver.WrappedArtifact; import org.eclipse.tycho.p2maven.InstallableUnitGenerator; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; class PomInstallableUnitStore implements IQueryable { @@ -200,7 +200,7 @@ private IQueryable getPomIUs() { } }); } - collection = new QueryableArray(installableUnitLookUp.keySet()); + collection = new QueryableArray(installableUnitLookUp.keySet(), false); } return collection; } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/ProjectorResolutionStrategy.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/ProjectorResolutionStrategy.java index 8bc48e5be9..3deb523f2e 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/ProjectorResolutionStrategy.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/ProjectorResolutionStrategy.java @@ -27,7 +27,6 @@ import org.eclipse.core.runtime.MultiStatus; import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.equinox.internal.p2.director.Explanation; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.internal.p2.director.SimplePlanner; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IRequirement; @@ -39,6 +38,7 @@ import org.eclipse.tycho.core.shared.StatusTool; import org.eclipse.tycho.p2.resolver.ResolverException; import org.eclipse.tycho.p2tools.copiedfromp2.Projector; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; import org.eclipse.tycho.p2tools.copiedfromp2.Slicer; public class ProjectorResolutionStrategy extends AbstractSlicerResolutionStrategy { @@ -109,7 +109,7 @@ protected Collection getRequiredCapabilities(IInstallableUnit iu) }; projector.encode(createUnitRequiring("tycho", seedUnits, seedRequires), EMPTY_IU_ARRAY /* alreadyExistingRoots */, - new QueryableArray(List.of()) /* installedIUs */, seedUnits /* newRoots */, monitor); + new QueryableArray(List.of(), false) /* installedIUs */, seedUnits /* newRoots */, monitor); IStatus s = projector.invokeSolver(monitor); if (s.getSeverity() == IStatus.ERROR) { Set explanation = getExplanation(projector); // suppress "Cannot complete the request. Generating details." diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/TargetPlatformFactoryImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/TargetPlatformFactoryImpl.java index 6f069e6b6a..e31cfbb8e8 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/TargetPlatformFactoryImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2resolver/TargetPlatformFactoryImpl.java @@ -44,7 +44,6 @@ import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.core.runtime.Path; import org.eclipse.core.runtime.URIUtil; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.metadata.IArtifactKey; @@ -114,6 +113,7 @@ import org.eclipse.tycho.p2.target.facade.TargetPlatformFactory; import org.eclipse.tycho.p2maven.ListCompositeArtifactRepository; import org.eclipse.tycho.p2maven.advices.MavenPropertiesAdvice; +import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray; import org.eclipse.tycho.targetplatform.P2TargetPlatform; import org.eclipse.tycho.targetplatform.TargetDefinition; import org.eclipse.tycho.targetplatform.TargetDefinitionContent; @@ -334,7 +334,7 @@ private List getMissingJunitBundles(ReactorProject project, Se Collection entries = eclipseProject.getClasspathEntries(); for (ProjectClasspathEntry entry : entries) { if (entry instanceof JUnitClasspathContainerEntry junit) { - IQueryable queriable = new QueryableArray(externalUIs); + IQueryable queriable = new QueryableArray(externalUIs, false); Collection artifacts = junit.getArtifacts(); for (JUnitBundle bundle : artifacts) { MavenArtifactKey maven = ClasspathReader.toMaven(bundle); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java index ea5696f60b..039e6dd7e2 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java @@ -18,7 +18,6 @@ import java.util.Iterator; import java.util.List; -import org.eclipse.equinox.internal.p2.core.helpers.CollectionUtils; import org.eclipse.equinox.internal.p2.metadata.InstallableUnit; import org.eclipse.equinox.internal.p2.metadata.TranslationSupport; import org.eclipse.equinox.internal.p2.metadata.index.CapabilityIndex; @@ -29,17 +28,21 @@ import org.eclipse.equinox.p2.metadata.index.IIndex; public class QueryableArray extends IndexProvider { - private final List dataSet; + private final Collection dataSet; private IIndex capabilityIndex; private IIndex idIndex; private TranslationSupport translationSupport; public QueryableArray(IInstallableUnit[] ius) { - dataSet = CollectionUtils.unmodifiableList(ius); + this(List.of(ius), false); } public QueryableArray(Collection ius) { - dataSet = List.copyOf(ius); + this(ius, true); + } + + public QueryableArray(Collection ius, boolean copy) { + dataSet = copy ? List.copyOf(ius) : ius; } @Override diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java index 817dbc9f0e..9aa8c9cf5f 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java @@ -35,7 +35,6 @@ import org.eclipse.equinox.internal.p2.core.helpers.LogHelper; import org.eclipse.equinox.internal.p2.core.helpers.Tracing; import org.eclipse.equinox.internal.p2.director.Messages; -import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.internal.p2.metadata.InstallableUnit; import org.eclipse.equinox.internal.p2.metadata.InstallableUnitPatch; import org.eclipse.equinox.p2.metadata.IInstallableUnit; @@ -105,11 +104,11 @@ public IQueryable slice(Collection ius, IPro if (result.getSeverity() == IStatus.ERROR) { return null; } - return new QueryableArray(considered); + return new QueryableArray(considered, false); } private void computeNonGreedyIUs() { - IQueryable queryable = new QueryableArray(considered); + IQueryable queryable = new QueryableArray(considered, false); for (IInstallableUnit iu : queryable.query(QueryUtil.ALL_UNITS, new NullProgressMonitor())) { iu = iu.unresolved(); Collection reqs = getRequirements(iu); diff --git a/tycho-core/src/test/java/org/eclipse/tycho/p2resolver/PublisherServiceTest.java b/tycho-core/src/test/java/org/eclipse/tycho/p2resolver/PublisherServiceTest.java index 1a5eaf0315..c6adcc27d2 100644 --- a/tycho-core/src/test/java/org/eclipse/tycho/p2resolver/PublisherServiceTest.java +++ b/tycho-core/src/test/java/org/eclipse/tycho/p2resolver/PublisherServiceTest.java @@ -69,7 +69,7 @@ public void initSubject() throws Exception { LinkedHashSet installableUnits = new LinkedHashSet<>(); installableUnits.add(InstallableUnitUtil.createFeatureIU("org.eclipse.example.original_feature", "1.0.0")); - IMetadataRepository context = new ImmutableInMemoryMetadataRepository(installableUnits); + IMetadataRepository context = new ImmutableInMemoryMetadataRepository(installableUnits, true); // TODO these publishers don't produce artifacts, so we could run without file system outputRepository = new PublishingRepositoryImpl(lookup(IProvisioningAgent.class), From 08ce3d251a7b447b5932015676b9d9cb0264a89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 22 Jan 2024 11:43:17 +0100 Subject: [PATCH 3/4] Fork RecreateRepositoryApplication from P2 --- .../RecreateRepositoryApplication.java | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java new file mode 100644 index 0000000000..ea57e19dd5 --- /dev/null +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright (c) 2009, 2022 IBM Corporation and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * IBM Corporation - initial API and implementation + * Sonatype Inc - ongoing development + * Mykola Nikishov - multiple artifact checksums + *******************************************************************************/ + +package org.eclipse.tycho.p2tools.copiedfromp2; + +import java.io.File; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.URIUtil; +import org.eclipse.equinox.internal.p2.artifact.processors.checksum.ChecksumUtilities; +import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository; +import org.eclipse.equinox.internal.p2.core.helpers.LogHelper; +import org.eclipse.equinox.p2.core.IProvisioningAgent; +import org.eclipse.equinox.p2.core.ProvisionException; +import org.eclipse.equinox.p2.internal.repository.tools.Messages; +import org.eclipse.equinox.p2.metadata.IArtifactKey; +import org.eclipse.equinox.p2.query.IQueryResult; +import org.eclipse.equinox.p2.repository.IRepository; +import org.eclipse.equinox.p2.repository.IRepositoryManager; +import org.eclipse.equinox.p2.repository.artifact.ArtifactKeyQuery; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager; +import org.eclipse.equinox.p2.repository.artifact.IFileArtifactRepository; +import org.eclipse.equinox.p2.repository.artifact.spi.ArtifactDescriptor; +import org.eclipse.osgi.util.NLS; + +public class RecreateRepositoryApplication extends AbstractApplication { + private URI repoLocation; + private String repoName = null; + boolean removeArtifactRepo = true; + private Map repoProperties = null; + private Map repoMap = null; + + public RecreateRepositoryApplication(IProvisioningAgent agent) { + super(agent); + } + + @Override + public IStatus run(IProgressMonitor monitor) throws ProvisionException { + try { + IArtifactRepository repository = initialize(monitor); + removeRepository(repository, monitor); + recreateRepository(monitor); + } finally { + if (removeArtifactRepo) { + IArtifactRepositoryManager repositoryManager = getArtifactRepositoryManager(); + repositoryManager.removeRepository(repoLocation); + } + } + + return Status.OK_STATUS; + } + + public void setArtifactRepository(URI repository) { + this.repoLocation = repository; + } + + private IArtifactRepository initialize(IProgressMonitor monitor) throws ProvisionException { + IArtifactRepositoryManager repositoryManager = getArtifactRepositoryManager(); + removeArtifactRepo = !repositoryManager.contains(repoLocation); + + IArtifactRepository repository = repositoryManager.loadRepository(repoLocation, + IRepositoryManager.REPOSITORY_HINT_MODIFIABLE, monitor); + + if (repository == null || !repository.isModifiable()) + throw new ProvisionException( + NLS.bind(Messages.exception_destinationNotModifiable, repository.getLocation())); + if (!(repository instanceof IFileArtifactRepository)) + throw new ProvisionException(NLS.bind(Messages.exception_notLocalFileRepo, repository.getLocation())); + + repoName = repository.getName(); + repoProperties = repository.getProperties(); + + repoMap = new HashMap<>(); + IQueryResult keys = repository.query(ArtifactKeyQuery.ALL_KEYS, null); + for (IArtifactKey key : keys) { + IArtifactDescriptor[] descriptors = repository.getArtifactDescriptors(key); + repoMap.put(key, descriptors); + } + + return repository; + } + + private void removeRepository(IArtifactRepository repository, IProgressMonitor monitor) throws ProvisionException { + IArtifactRepositoryManager manager = getArtifactRepositoryManager(); + manager.removeRepository(repository.getLocation()); + + boolean compressed = Boolean.parseBoolean(repoProperties.get(IRepository.PROP_COMPRESSED)); + URI realLocation = SimpleArtifactRepository.getActualLocation(repository.getLocation(), compressed); + File realFile = URIUtil.toFile(realLocation); + if (!realFile.exists() || !realFile.delete()) + throw new ProvisionException(NLS.bind(Messages.exception_unableToRemoveRepo, realFile.toString())); + } + + private void recreateRepository(IProgressMonitor monitor) throws ProvisionException { + IArtifactRepositoryManager manager = getArtifactRepositoryManager(); + + IArtifactRepository repository = manager.createRepository(repoLocation, repoName, + IArtifactRepositoryManager.TYPE_SIMPLE_REPOSITORY, repoProperties); + if (!(repository instanceof IFileArtifactRepository)) + throw new ProvisionException(NLS.bind(Messages.exception_notLocalFileRepo, repository.getLocation())); + + IFileArtifactRepository simple = (IFileArtifactRepository) repository; + for (IArtifactKey key : repoMap.keySet()) { + IArtifactDescriptor[] descriptors = repoMap.get(key); + + Set files = new HashSet<>(); + for (IArtifactDescriptor descriptor : descriptors) { + File artifactFile = simple.getArtifactFile(descriptor); + files.add(artifactFile); + + String size = Long.toString(artifactFile.length()); + + ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor); + newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size); + newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size); + + Map checksums = new HashMap<>(); + List checksumsToSkip = Collections.emptyList(); + IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip); + if (!status.isOK()) + // TODO handle errors in some way + LogHelper.log(status); + + Map checksumsToProperties = ChecksumUtilities + .checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums); + newDescriptor.addProperties(checksumsToProperties); + + repository.addDescriptor(newDescriptor, null); + } + } + } +} From e2446f309f82ed904b71c46ff8fbb44553872884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 22 Jan 2024 12:02:40 +0100 Subject: [PATCH 4/4] Execute recreation of repository in a batch operation --- .../p2tools/MirrorApplicationServiceImpl.java | 7 +- .../copiedfromp2/MirrorApplication.java | 34 ++++++++-- .../RecreateRepositoryApplication.java | 66 ++++++++++--------- .../tycho/p2tools/copiedfromp2/Slicer.java | 28 ++++---- 4 files changed, 81 insertions(+), 54 deletions(-) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java index 76a629c9eb..1644b9b832 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java @@ -53,7 +53,6 @@ import org.eclipse.equinox.p2.internal.repository.mirroring.Mirroring; import org.eclipse.equinox.p2.internal.repository.tools.Activator; import org.eclipse.equinox.p2.internal.repository.tools.Messages; -import org.eclipse.equinox.p2.internal.repository.tools.RecreateRepositoryApplication; import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor; import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions; import org.eclipse.equinox.p2.internal.repository.tools.XZCompressor; @@ -82,6 +81,7 @@ import org.eclipse.tycho.p2.tools.mirroring.facade.IUDescription; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorOptions; +import org.eclipse.tycho.p2tools.copiedfromp2.RecreateRepositoryApplication; @Component(role = MirrorApplicationService.class) public class MirrorApplicationServiceImpl implements MirrorApplicationService { @@ -239,10 +239,7 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati artifactsXz.delete(); } descriptor.setLocation(location.toURI()); - //TODO this is to trigger loading of the osgi services and we can not pass the agent directly see - // https://github.com/eclipse-equinox/p2/issues/151 - agent.getService(IArtifactRepositoryManager.class); - RecreateRepositoryApplication application = new RecreateRepositoryApplication(); + RecreateRepositoryApplication application = new RecreateRepositoryApplication(agent); application.setArtifactRepository(descriptor.getRepoLocation()); try { application.run(new NullProgressMonitor()); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java index c3218ca581..66fd0b12c2 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.concurrent.atomic.AtomicReference; import org.eclipse.core.runtime.IConfigurationElement; import org.eclipse.core.runtime.IExecutableExtension; @@ -211,7 +212,8 @@ else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$ @Override public IStatus run(IProgressMonitor monitor) throws ProvisionException { - IStatus mirrorStatus = Status.OK_STATUS; + AtomicReference mirrorStatus = new AtomicReference<>(Status.OK_STATUS); + AtomicReference exception = new AtomicReference<>(); try { initializeRepos(new NullProgressMonitor()); initializeLogs(); @@ -220,20 +222,38 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException { IQueryable slice = slice(new NullProgressMonitor()); Set units = collectUnits(slice, monitor); if (destinationArtifactRepository != null) { - mirrorStatus = mirrorArtifacts(units, new NullProgressMonitor()); - if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR) - return mirrorStatus; + destinationArtifactRepository.executeBatch(m -> { + try { + mirrorStatus.set(mirrorArtifacts(units, m)); + } catch (ProvisionException e) { + exception.set(e); + } + }, new NullProgressMonitor()); + if (exception.get() != null) { + throw exception.get(); + } + if (failOnError && mirrorStatus.get().getSeverity() == IStatus.ERROR) + return mirrorStatus.get(); } if (destinationMetadataRepository != null) { - mirrorMetadata(units, new NullProgressMonitor()); + destinationMetadataRepository.executeBatch(m -> { + try { + mirrorMetadata(units, m); + } catch (ProvisionException e) { + exception.set(e); + } + }, new NullProgressMonitor()); + if (exception.get() != null) { + throw exception.get(); + } } } finally { finalizeRepositories(); finalizeLogs(); } - if (mirrorStatus.isOK()) + if (mirrorStatus.get().isOK()) return Status.OK_STATUS; - return mirrorStatus; + return mirrorStatus.get(); } private IStatus mirrorArtifacts(Collection slice, IProgressMonitor monitor) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java index ea57e19dd5..22573e21f2 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java @@ -27,11 +27,11 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.MultiStatus; import org.eclipse.core.runtime.Status; import org.eclipse.core.runtime.URIUtil; import org.eclipse.equinox.internal.p2.artifact.processors.checksum.ChecksumUtilities; import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository; -import org.eclipse.equinox.internal.p2.core.helpers.LogHelper; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.internal.repository.tools.Messages; @@ -63,7 +63,10 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException { try { IArtifactRepository repository = initialize(monitor); removeRepository(repository, monitor); - recreateRepository(monitor); + MultiStatus status = recreateRepository(monitor); + if (status.isOK()) { + return status; + } } finally { if (removeArtifactRepo) { IArtifactRepositoryManager repositoryManager = getArtifactRepositoryManager(); @@ -115,7 +118,7 @@ private void removeRepository(IArtifactRepository repository, IProgressMonitor m throw new ProvisionException(NLS.bind(Messages.exception_unableToRemoveRepo, realFile.toString())); } - private void recreateRepository(IProgressMonitor monitor) throws ProvisionException { + private MultiStatus recreateRepository(IProgressMonitor monitor) throws ProvisionException { IArtifactRepositoryManager manager = getArtifactRepositoryManager(); IArtifactRepository repository = manager.createRepository(repoLocation, repoName, @@ -124,33 +127,36 @@ private void recreateRepository(IProgressMonitor monitor) throws ProvisionExcept throw new ProvisionException(NLS.bind(Messages.exception_notLocalFileRepo, repository.getLocation())); IFileArtifactRepository simple = (IFileArtifactRepository) repository; - for (IArtifactKey key : repoMap.keySet()) { - IArtifactDescriptor[] descriptors = repoMap.get(key); - - Set files = new HashSet<>(); - for (IArtifactDescriptor descriptor : descriptors) { - File artifactFile = simple.getArtifactFile(descriptor); - files.add(artifactFile); - - String size = Long.toString(artifactFile.length()); - - ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor); - newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size); - newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size); - - Map checksums = new HashMap<>(); - List checksumsToSkip = Collections.emptyList(); - IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip); - if (!status.isOK()) - // TODO handle errors in some way - LogHelper.log(status); - - Map checksumsToProperties = ChecksumUtilities - .checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums); - newDescriptor.addProperties(checksumsToProperties); - - repository.addDescriptor(newDescriptor, null); + MultiStatus multiStatus = new MultiStatus(getClass(), 0, "Problem while recreate repository"); + repository.executeBatch(m -> { + for (IArtifactKey key : repoMap.keySet()) { + IArtifactDescriptor[] descriptors = repoMap.get(key); + + Set files = new HashSet<>(); + for (IArtifactDescriptor descriptor : descriptors) { + File artifactFile = simple.getArtifactFile(descriptor); + files.add(artifactFile); + + String size = Long.toString(artifactFile.length()); + + ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor); + newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size); + newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size); + + Map checksums = new HashMap<>(); + List checksumsToSkip = Collections.emptyList(); + IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip); + if (!status.isOK()) { + multiStatus.add(status); + } + Map checksumsToProperties = ChecksumUtilities + .checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums); + newDescriptor.addProperties(checksumsToProperties); + + repository.addDescriptor(newDescriptor, null); + } } - } + }, monitor); + return multiStatus; } } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java index 9aa8c9cf5f..f5eb88ed40 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java @@ -205,24 +205,28 @@ private Collection getRequirements(IInstallableUnit iu) { return aggregatedRequirements; } + private Set consideredRequirements = new HashSet<>(); + private void expandRequirement(IInstallableUnit iu, IRequirement req) { if (req.getMax() == 0) { return; } - List selected = selectIUsForRequirement(possibilites, req).toList(); - for (IInstallableUnit match : selected) { - Map iuSlice = slice.get(match.getId()); - if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) { - toProcess.add(match); + if (consideredRequirements.add(req)) { + List selected = selectIUsForRequirement(possibilites, req).toList(); + for (IInstallableUnit match : selected) { + Map iuSlice = slice.get(match.getId()); + if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) { + toProcess.add(match); + } } - } - if (selected.isEmpty()) { - if (req.getMin() == 0) { - if (DEBUG) { - System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$ + if (selected.isEmpty()) { + if (req.getMin() == 0) { + if (DEBUG) { + System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$ + } + } else { + result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req))); } - } else { - result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req))); } } }