From c86d9ea4255c12cd43591c55278959e763297400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Mon, 22 Jan 2024 12:02:40 +0100 Subject: [PATCH] Execute recreation of repository in a batch operation --- .../p2tools/MirrorApplicationServiceImpl.java | 7 +- .../copiedfromp2/MirrorApplication.java | 34 ++++++++-- .../RecreateRepositoryApplication.java | 66 ++++++++++--------- .../tycho/p2tools/copiedfromp2/Slicer.java | 28 ++++---- 4 files changed, 81 insertions(+), 54 deletions(-) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java index 76a629c9eb..1644b9b832 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java @@ -53,7 +53,6 @@ import org.eclipse.equinox.p2.internal.repository.mirroring.Mirroring; import org.eclipse.equinox.p2.internal.repository.tools.Activator; import org.eclipse.equinox.p2.internal.repository.tools.Messages; -import org.eclipse.equinox.p2.internal.repository.tools.RecreateRepositoryApplication; import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor; import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions; import org.eclipse.equinox.p2.internal.repository.tools.XZCompressor; @@ -82,6 +81,7 @@ import org.eclipse.tycho.p2.tools.mirroring.facade.IUDescription; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorOptions; +import org.eclipse.tycho.p2tools.copiedfromp2.RecreateRepositoryApplication; @Component(role = MirrorApplicationService.class) public class MirrorApplicationServiceImpl implements MirrorApplicationService { @@ -239,10 +239,7 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati artifactsXz.delete(); } descriptor.setLocation(location.toURI()); - //TODO this is to trigger loading of the osgi services and we can not pass the agent directly see - // https://github.com/eclipse-equinox/p2/issues/151 - agent.getService(IArtifactRepositoryManager.class); - RecreateRepositoryApplication application = new RecreateRepositoryApplication(); + RecreateRepositoryApplication application = new RecreateRepositoryApplication(agent); application.setArtifactRepository(descriptor.getRepoLocation()); try { application.run(new NullProgressMonitor()); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java index c3218ca581..66fd0b12c2 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; import java.util.StringTokenizer; +import java.util.concurrent.atomic.AtomicReference; import org.eclipse.core.runtime.IConfigurationElement; import org.eclipse.core.runtime.IExecutableExtension; @@ -211,7 +212,8 @@ else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$ @Override public IStatus run(IProgressMonitor monitor) throws ProvisionException { - IStatus mirrorStatus = Status.OK_STATUS; + AtomicReference mirrorStatus = new AtomicReference<>(Status.OK_STATUS); + AtomicReference exception = new AtomicReference<>(); try { initializeRepos(new NullProgressMonitor()); initializeLogs(); @@ -220,20 +222,38 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException { IQueryable slice = slice(new NullProgressMonitor()); Set units = collectUnits(slice, monitor); if (destinationArtifactRepository != null) { - mirrorStatus = mirrorArtifacts(units, new NullProgressMonitor()); - if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR) - return mirrorStatus; + destinationArtifactRepository.executeBatch(m -> { + try { + mirrorStatus.set(mirrorArtifacts(units, m)); + } catch (ProvisionException e) { + exception.set(e); + } + }, new NullProgressMonitor()); + if (exception.get() != null) { + throw exception.get(); + } + if (failOnError && mirrorStatus.get().getSeverity() == IStatus.ERROR) + return mirrorStatus.get(); } if (destinationMetadataRepository != null) { - mirrorMetadata(units, new NullProgressMonitor()); + destinationMetadataRepository.executeBatch(m -> { + try { + mirrorMetadata(units, m); + } catch (ProvisionException e) { + exception.set(e); + } + }, new NullProgressMonitor()); + if (exception.get() != null) { + throw exception.get(); + } } } finally { finalizeRepositories(); finalizeLogs(); } - if (mirrorStatus.isOK()) + if (mirrorStatus.get().isOK()) return Status.OK_STATUS; - return mirrorStatus; + return mirrorStatus.get(); } private IStatus mirrorArtifacts(Collection slice, IProgressMonitor monitor) diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java index ea57e19dd5..22573e21f2 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java @@ -27,11 +27,11 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.MultiStatus; import org.eclipse.core.runtime.Status; import org.eclipse.core.runtime.URIUtil; import org.eclipse.equinox.internal.p2.artifact.processors.checksum.ChecksumUtilities; import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository; -import org.eclipse.equinox.internal.p2.core.helpers.LogHelper; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.internal.repository.tools.Messages; @@ -63,7 +63,10 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException { try { IArtifactRepository repository = initialize(monitor); removeRepository(repository, monitor); - recreateRepository(monitor); + MultiStatus status = recreateRepository(monitor); + if (status.isOK()) { + return status; + } } finally { if (removeArtifactRepo) { IArtifactRepositoryManager repositoryManager = getArtifactRepositoryManager(); @@ -115,7 +118,7 @@ private void removeRepository(IArtifactRepository repository, IProgressMonitor m throw new ProvisionException(NLS.bind(Messages.exception_unableToRemoveRepo, realFile.toString())); } - private void recreateRepository(IProgressMonitor monitor) throws ProvisionException { + private MultiStatus recreateRepository(IProgressMonitor monitor) throws ProvisionException { IArtifactRepositoryManager manager = getArtifactRepositoryManager(); IArtifactRepository repository = manager.createRepository(repoLocation, repoName, @@ -124,33 +127,36 @@ private void recreateRepository(IProgressMonitor monitor) throws ProvisionExcept throw new ProvisionException(NLS.bind(Messages.exception_notLocalFileRepo, repository.getLocation())); IFileArtifactRepository simple = (IFileArtifactRepository) repository; - for (IArtifactKey key : repoMap.keySet()) { - IArtifactDescriptor[] descriptors = repoMap.get(key); - - Set files = new HashSet<>(); - for (IArtifactDescriptor descriptor : descriptors) { - File artifactFile = simple.getArtifactFile(descriptor); - files.add(artifactFile); - - String size = Long.toString(artifactFile.length()); - - ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor); - newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size); - newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size); - - Map checksums = new HashMap<>(); - List checksumsToSkip = Collections.emptyList(); - IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip); - if (!status.isOK()) - // TODO handle errors in some way - LogHelper.log(status); - - Map checksumsToProperties = ChecksumUtilities - .checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums); - newDescriptor.addProperties(checksumsToProperties); - - repository.addDescriptor(newDescriptor, null); + MultiStatus multiStatus = new MultiStatus(getClass(), 0, "Problem while recreate repository"); + repository.executeBatch(m -> { + for (IArtifactKey key : repoMap.keySet()) { + IArtifactDescriptor[] descriptors = repoMap.get(key); + + Set files = new HashSet<>(); + for (IArtifactDescriptor descriptor : descriptors) { + File artifactFile = simple.getArtifactFile(descriptor); + files.add(artifactFile); + + String size = Long.toString(artifactFile.length()); + + ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor); + newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size); + newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size); + + Map checksums = new HashMap<>(); + List checksumsToSkip = Collections.emptyList(); + IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip); + if (!status.isOK()) { + multiStatus.add(status); + } + Map checksumsToProperties = ChecksumUtilities + .checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums); + newDescriptor.addProperties(checksumsToProperties); + + repository.addDescriptor(newDescriptor, null); + } } - } + }, monitor); + return multiStatus; } } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java index 9aa8c9cf5f..f5eb88ed40 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java @@ -205,24 +205,28 @@ private Collection getRequirements(IInstallableUnit iu) { return aggregatedRequirements; } + private Set consideredRequirements = new HashSet<>(); + private void expandRequirement(IInstallableUnit iu, IRequirement req) { if (req.getMax() == 0) { return; } - List selected = selectIUsForRequirement(possibilites, req).toList(); - for (IInstallableUnit match : selected) { - Map iuSlice = slice.get(match.getId()); - if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) { - toProcess.add(match); + if (consideredRequirements.add(req)) { + List selected = selectIUsForRequirement(possibilites, req).toList(); + for (IInstallableUnit match : selected) { + Map iuSlice = slice.get(match.getId()); + if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) { + toProcess.add(match); + } } - } - if (selected.isEmpty()) { - if (req.getMin() == 0) { - if (DEBUG) { - System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$ + if (selected.isEmpty()) { + if (req.getMin() == 0) { + if (DEBUG) { + System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$ + } + } else { + result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req))); } - } else { - result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req))); } } }