Speedup repository assembly

tycho-core/src/main/java/org/eclipse/tycho/p2/repository/ImmutableInMemoryMetadataRepository.java

@@ -24,28 +24,18 @@
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.eclipse.core.runtime.Status;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.p2.core.IPool;
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
-import org.eclipse.equinox.p2.query.IQuery;
-import org.eclipse.equinox.p2.query.IQueryResult;
-import org.eclipse.equinox.p2.query.IQueryable;
 import org.eclipse.equinox.p2.repository.IRepositoryReference;
 import org.eclipse.equinox.p2.repository.IRunnableWithProgress;
 import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 
-public class ImmutableInMemoryMetadataRepository implements IMetadataRepository {
+public class ImmutableInMemoryMetadataRepository extends QueryableArray implements IMetadataRepository {
 
-    private final IQueryable<IInstallableUnit> units;
-
-    public ImmutableInMemoryMetadataRepository(Set<IInstallableUnit> units) {
-        this.units = new QueryableArray(units);
-    }
-
-    @Override
-    public IQueryResult<IInstallableUnit> query(IQuery<IInstallableUnit> query, IProgressMonitor monitor) {
-        return units.query(query, monitor);
+    public ImmutableInMemoryMetadataRepository(Set<IInstallableUnit> units, boolean copy) {
+        super(units, copy);
     }
 
     @Override

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/AbstractSlicerResolutionStrategy.java

@@ -27,7 +27,6 @@
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.MultiStatus;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability;
 import org.eclipse.equinox.internal.p2.metadata.RequiredCapability;
 import org.eclipse.equinox.internal.p2.metadata.RequiredPropertiesMatch;
@@ -47,6 +46,7 @@
 import org.eclipse.tycho.core.shared.MavenLogger;
 import org.eclipse.tycho.core.shared.StatusTool;
 import org.eclipse.tycho.p2.resolver.ResolverException;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 import org.eclipse.tycho.p2tools.copiedfromp2.Slicer;
 
 abstract class AbstractSlicerResolutionStrategy extends AbstractResolutionStrategy {
@@ -94,9 +94,9 @@ protected final IQueryable<IInstallableUnit> slice(Map<String, String> propertie
             seedIUs.add(createUnitRequiring("tycho-ee", null, data.getEEResolutionHints().getMandatoryRequires()));
         }
 
-        IQueryable<IInstallableUnit> baseIUCollection = new QueryableArray(availableIUs);
+        IQueryable<IInstallableUnit> baseIUCollection = new QueryableArray(availableIUs, false);
         Slicer slicer = newSlicer((query, monitor1) -> {
-
+//
             IQueryResult<IInstallableUnit> queryResult = baseIUCollection.query(query, monitor1);
             if (queryResult.isEmpty()) {
                 IQueryable<IInstallableUnit> additionalUnitStore = data.getAdditionalUnitStore();

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/DependencyCollector.java

@@ -21,7 +21,6 @@
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.MultiStatus;
 import org.eclipse.core.runtime.Status;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.internal.p2.metadata.RequiredCapability;
 import org.eclipse.equinox.p2.core.ProvisionException;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
@@ -31,6 +30,7 @@
 import org.eclipse.equinox.p2.query.QueryUtil;
 import org.eclipse.tycho.core.shared.MavenLogger;
 import org.eclipse.tycho.p2.publisher.FeatureDependenciesAction;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 
 public class DependencyCollector extends AbstractResolutionStrategy {
 
@@ -51,7 +51,7 @@ public Collection<IInstallableUnit> resolve(Map<String, String> properties, IPro
 
         result.addAll(data.getRootIUs());
 
-        IQueryable<IInstallableUnit> availableUIsQueryable = new QueryableArray(data.getAvailableIUs());
+        IQueryable<IInstallableUnit> availableUIsQueryable = new QueryableArray(data.getAvailableIUs(), false);
         for (IInstallableUnit iu : data.getRootIUs()) {
             collectIncludedIUs(availableUIsQueryable, result, errors, iu, true, monitor);
         }

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/FinalTargetPlatformImpl.java

@@ -30,6 +30,7 @@
 public class FinalTargetPlatformImpl extends TargetPlatformBaseImpl {
 
     private IArtifactRepository artifactRepository;
+    private IMetadataRepository metadataRepository;
 
     public FinalTargetPlatformImpl(LinkedHashSet<IInstallableUnit> installableUnits,
             ExecutionEnvironmentResolutionHints executionEnvironment, IRawArtifactFileProvider jointArtifacts,
@@ -39,6 +40,7 @@ public FinalTargetPlatformImpl(LinkedHashSet<IInstallableUnit> installableUnits,
         super(installableUnits, executionEnvironment, jointArtifacts, localArtifactRepository, reactorProjectLookup,
                 mavenArtifactLookup, shadowed);
         this.artifactRepository = artifactRepository;
+        this.metadataRepository = new ImmutableInMemoryMetadataRepository(installableUnits, false);
     }
 
     @Override
@@ -48,7 +50,7 @@ public void reportUsedLocalIUs(Collection<IInstallableUnit> usedUnits) {
 
     @Override
     public IMetadataRepository getMetadataRepository() {
-        return new ImmutableInMemoryMetadataRepository(installableUnits);
+        return metadataRepository;
     }
 
     @Override

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/P2ResolverImpl.java

@@ -34,7 +34,6 @@
 
 import org.apache.felix.resolver.util.CopyOnWriteSet;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.p2.metadata.IArtifactKey;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
 import org.eclipse.equinox.p2.metadata.IProvidedCapability;
@@ -75,6 +74,7 @@
 import org.eclipse.tycho.p2.resolver.ResolverException;
 import org.eclipse.tycho.p2.target.facade.TargetPlatformConfigurationStub;
 import org.eclipse.tycho.p2.target.facade.TargetPlatformFactory;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 import org.eclipse.tycho.p2tools.copiedfromp2.Slicer;
 import org.eclipse.tycho.targetplatform.P2TargetPlatform;
 
@@ -329,7 +329,7 @@ public List<IRequirement> getAdditionalRequirements() {
     public P2ResolutionResult resolveInstallableUnit(TargetPlatform context, String id, String versionRange) {
 
         P2TargetPlatform targetPlatform = getTargetFromContext(context);
-        IQueryable<IInstallableUnit> queriable = new QueryableArray(targetPlatform.getInstallableUnits());
+        IQueryable<IInstallableUnit> queriable = new QueryableArray(targetPlatform.getInstallableUnits(), false);
 
         VersionRange range = new VersionRange(versionRange);
         IRequirement requirement = MetadataFactory.createRequirement(IInstallableUnit.NAMESPACE_IU_ID, id, range, null,

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/PomInstallableUnitStore.java

@@ -31,7 +31,6 @@
 import org.apache.maven.project.MavenProject;
 import org.codehaus.plexus.logging.Logger;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
 import org.eclipse.equinox.p2.publisher.IPublisherInfo;
 import org.eclipse.equinox.p2.publisher.PublisherInfo;
@@ -49,6 +48,7 @@
 import org.eclipse.tycho.p2.repository.RepositoryLayoutHelper;
 import org.eclipse.tycho.p2.resolver.WrappedArtifact;
 import org.eclipse.tycho.p2maven.InstallableUnitGenerator;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 
 class PomInstallableUnitStore implements IQueryable<IInstallableUnit> {
 
@@ -200,7 +200,7 @@ private IQueryable<IInstallableUnit> getPomIUs() {
                     }
                 });
             }
-            collection = new QueryableArray(installableUnitLookUp.keySet());
+            collection = new QueryableArray(installableUnitLookUp.keySet(), false);
         }
         return collection;
     }

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/ProjectorResolutionStrategy.java

@@ -27,7 +27,6 @@
 import org.eclipse.core.runtime.MultiStatus;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.equinox.internal.p2.director.Explanation;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.internal.p2.director.SimplePlanner;
 import org.eclipse.equinox.p2.metadata.IInstallableUnit;
 import org.eclipse.equinox.p2.metadata.IRequirement;
@@ -39,6 +38,7 @@
 import org.eclipse.tycho.core.shared.StatusTool;
 import org.eclipse.tycho.p2.resolver.ResolverException;
 import org.eclipse.tycho.p2tools.copiedfromp2.Projector;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 import org.eclipse.tycho.p2tools.copiedfromp2.Slicer;
 
 public class ProjectorResolutionStrategy extends AbstractSlicerResolutionStrategy {
@@ -109,7 +109,7 @@ protected Collection<IRequirement> getRequiredCapabilities(IInstallableUnit iu)
         };
         projector.encode(createUnitRequiring("tycho", seedUnits, seedRequires),
                 EMPTY_IU_ARRAY /* alreadyExistingRoots */,
-                new QueryableArray(List.of()) /* installedIUs */, seedUnits /* newRoots */, monitor);
+                new QueryableArray(List.of(), false) /* installedIUs */, seedUnits /* newRoots */, monitor);
         IStatus s = projector.invokeSolver(monitor);
         if (s.getSeverity() == IStatus.ERROR) {
             Set<Explanation> explanation = getExplanation(projector); // suppress "Cannot complete the request.  Generating details."

tycho-core/src/main/java/org/eclipse/tycho/p2resolver/TargetPlatformFactoryImpl.java

@@ -44,7 +44,6 @@
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.URIUtil;
-import org.eclipse.equinox.internal.p2.director.QueryableArray;
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.core.ProvisionException;
 import org.eclipse.equinox.p2.metadata.IArtifactKey;
@@ -114,6 +113,7 @@
 import org.eclipse.tycho.p2.target.facade.TargetPlatformFactory;
 import org.eclipse.tycho.p2maven.ListCompositeArtifactRepository;
 import org.eclipse.tycho.p2maven.advices.MavenPropertiesAdvice;
+import org.eclipse.tycho.p2tools.copiedfromp2.QueryableArray;
 import org.eclipse.tycho.targetplatform.P2TargetPlatform;
 import org.eclipse.tycho.targetplatform.TargetDefinition;
 import org.eclipse.tycho.targetplatform.TargetDefinitionContent;
@@ -334,7 +334,7 @@ private List<MavenArtifactKey> getMissingJunitBundles(ReactorProject project, Se
                         Collection<ProjectClasspathEntry> entries = eclipseProject.getClasspathEntries();
                         for (ProjectClasspathEntry entry : entries) {
                             if (entry instanceof JUnitClasspathContainerEntry junit) {
-                                IQueryable<IInstallableUnit> queriable = new QueryableArray(externalUIs);
+                                IQueryable<IInstallableUnit> queriable = new QueryableArray(externalUIs, false);
                                 Collection<JUnitBundle> artifacts = junit.getArtifacts();
                                 for (JUnitBundle bundle : artifacts) {
                                     MavenArtifactKey maven = ClasspathReader.toMaven(bundle);

tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java

@@ -53,7 +53,6 @@
 import org.eclipse.equinox.p2.internal.repository.mirroring.Mirroring;
 import org.eclipse.equinox.p2.internal.repository.tools.Activator;
 import org.eclipse.equinox.p2.internal.repository.tools.Messages;
-import org.eclipse.equinox.p2.internal.repository.tools.RecreateRepositoryApplication;
 import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor;
 import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions;
 import org.eclipse.equinox.p2.internal.repository.tools.XZCompressor;
@@ -82,6 +81,7 @@
 import org.eclipse.tycho.p2.tools.mirroring.facade.IUDescription;
 import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService;
 import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorOptions;
+import org.eclipse.tycho.p2tools.copiedfromp2.RecreateRepositoryApplication;
 
 @Component(role = MirrorApplicationService.class)
 public class MirrorApplicationServiceImpl implements MirrorApplicationService {
@@ -239,10 +239,7 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati
             artifactsXz.delete();
         }
         descriptor.setLocation(location.toURI());
-        //TODO this is to trigger loading of the osgi services and we can not pass the agent directly see 
-        // https://github.com/eclipse-equinox/p2/issues/151
-        agent.getService(IArtifactRepositoryManager.class);
-        RecreateRepositoryApplication application = new RecreateRepositoryApplication();
+        RecreateRepositoryApplication application = new RecreateRepositoryApplication(agent);
         application.setArtifactRepository(descriptor.getRepoLocation());
         try {
             application.run(new NullProgressMonitor());

tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java

@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.StringTokenizer;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.eclipse.core.runtime.IExecutableExtension;
@@ -211,7 +212,8 @@ else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$
 
     @Override
     public IStatus run(IProgressMonitor monitor) throws ProvisionException {
-        IStatus mirrorStatus = Status.OK_STATUS;
+        AtomicReference<IStatus> mirrorStatus = new AtomicReference<>(Status.OK_STATUS);
+        AtomicReference<ProvisionException> exception = new AtomicReference<>();
         try {
             initializeRepos(new NullProgressMonitor());
             initializeLogs();
@@ -220,20 +222,38 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException {
             IQueryable<IInstallableUnit> slice = slice(new NullProgressMonitor());
             Set<IInstallableUnit> units = collectUnits(slice, monitor);
             if (destinationArtifactRepository != null) {
-                mirrorStatus = mirrorArtifacts(units, new NullProgressMonitor());
-                if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR)
-                    return mirrorStatus;
+                destinationArtifactRepository.executeBatch(m -> {
+                    try {
+                        mirrorStatus.set(mirrorArtifacts(units, m));
+                    } catch (ProvisionException e) {
+                        exception.set(e);
+                    }
+                }, new NullProgressMonitor());
+                if (exception.get() != null) {
+                    throw exception.get();
+                }
+                if (failOnError && mirrorStatus.get().getSeverity() == IStatus.ERROR)
+                    return mirrorStatus.get();
             }
             if (destinationMetadataRepository != null) {
-                mirrorMetadata(units, new NullProgressMonitor());
+                destinationMetadataRepository.executeBatch(m -> {
+                    try {
+                        mirrorMetadata(units, m);
+                    } catch (ProvisionException e) {
+                        exception.set(e);
+                    }
+                }, new NullProgressMonitor());
+                if (exception.get() != null) {
+                    throw exception.get();
+                }
             }
         } finally {
             finalizeRepositories();
             finalizeLogs();
         }
-        if (mirrorStatus.isOK())
+        if (mirrorStatus.get().isOK())
             return Status.OK_STATUS;
-        return mirrorStatus;
+        return mirrorStatus.get();
     }
 
     private IStatus mirrorArtifacts(Collection<IInstallableUnit> slice, IProgressMonitor monitor)

tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/QueryableArray.java

@@ -0,0 +1,86 @@
+/*******************************************************************************
+ * Copyright (c) 2008, 2023 IBM Corporation and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     IBM Corporation - initial API and implementation
+ *     Cloudsmith Inc. - query indexes
+ *******************************************************************************/
+package org.eclipse.tycho.p2tools.copiedfromp2;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.eclipse.equinox.internal.p2.metadata.InstallableUnit;
+import org.eclipse.equinox.internal.p2.metadata.TranslationSupport;
+import org.eclipse.equinox.internal.p2.metadata.index.CapabilityIndex;
+import org.eclipse.equinox.internal.p2.metadata.index.IdIndex;
+import org.eclipse.equinox.internal.p2.metadata.index.IndexProvider;
+import org.eclipse.equinox.p2.metadata.IInstallableUnit;
+import org.eclipse.equinox.p2.metadata.KeyWithLocale;
+import org.eclipse.equinox.p2.metadata.index.IIndex;
+
+public class QueryableArray extends IndexProvider<IInstallableUnit> {
+    private final Collection<IInstallableUnit> dataSet;
+    private IIndex<IInstallableUnit> capabilityIndex;
+    private IIndex<IInstallableUnit> idIndex;
+    private TranslationSupport translationSupport;
+
+    public QueryableArray(IInstallableUnit[] ius) {
+        this(List.of(ius), false);
+    }
+
+    public QueryableArray(Collection<IInstallableUnit> ius) {
+        this(ius, true);
+    }
+
+    public QueryableArray(Collection<IInstallableUnit> ius, boolean copy) {
+        dataSet = copy ? List.copyOf(ius) : ius;
+    }
+
+    @Override
+    public Iterator<IInstallableUnit> everything() {
+        return dataSet.iterator();
+    }
+
+    @Override
+    public boolean contains(IInstallableUnit element) {
+        return dataSet.contains(element);
+    }
+
+    @Override
+    public synchronized IIndex<IInstallableUnit> getIndex(String memberName) {
+        if (InstallableUnit.MEMBER_PROVIDED_CAPABILITIES.equals(memberName)) {
+            if (capabilityIndex == null)
+                capabilityIndex = new CapabilityIndex(dataSet.iterator());
+            return capabilityIndex;
+        }
+        if (InstallableUnit.MEMBER_ID.equals(memberName)) {
+            if (idIndex == null)
+                idIndex = new IdIndex(dataSet.iterator());
+            return idIndex;
+        }
+        return null;
+    }
+
+    @Override
+    public synchronized Object getManagedProperty(Object client, String memberName, Object key) {
+        if (!(client instanceof IInstallableUnit))
+            return null;
+        IInstallableUnit iu = (IInstallableUnit) client;
+        if (InstallableUnit.MEMBER_TRANSLATED_PROPERTIES.equals(memberName)) {
+            if (translationSupport == null)
+                translationSupport = new TranslationSupport(this);
+            return key instanceof KeyWithLocale ? translationSupport.getIUProperty(iu, (KeyWithLocale) key)
+                    : translationSupport.getIUProperty(iu, key.toString());
+        }
+        return null;
+    }
+}