Skip to content

Commit

Permalink
Execute recreation of repository in a batch operation
Browse files Browse the repository at this point in the history
  • Loading branch information
@laeubi
laeubi committed Jan 22, 2024
1 parent 7e190a8 commit c86d9ea
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 54 deletions.
7 changes: 2 additions & 5 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/MirrorApplicationServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.eclipse.equinox.p2.internal.repository.mirroring.Mirroring;
import org.eclipse.equinox.p2.internal.repository.tools.Activator;
import org.eclipse.equinox.p2.internal.repository.tools.Messages;
import org.eclipse.equinox.p2.internal.repository.tools.RecreateRepositoryApplication;
import org.eclipse.equinox.p2.internal.repository.tools.RepositoryDescriptor;
import org.eclipse.equinox.p2.internal.repository.tools.SlicingOptions;
import org.eclipse.equinox.p2.internal.repository.tools.XZCompressor;
Expand Down Expand Up @@ -82,6 +81,7 @@
import org.eclipse.tycho.p2.tools.mirroring.facade.IUDescription;
import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService;
import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorOptions;
import org.eclipse.tycho.p2tools.copiedfromp2.RecreateRepositoryApplication;

@Component(role = MirrorApplicationService.class)
public class MirrorApplicationServiceImpl implements MirrorApplicationService {
Expand Down Expand Up @@ -239,10 +239,7 @@ public void recreateArtifactRepository(DestinationRepositoryDescriptor destinati
artifactsXz.delete();
}
descriptor.setLocation(location.toURI());
//TODO this is to trigger loading of the osgi services and we can not pass the agent directly see
// https://github.com/eclipse-equinox/p2/issues/151
agent.getService(IArtifactRepositoryManager.class);
RecreateRepositoryApplication application = new RecreateRepositoryApplication();
RecreateRepositoryApplication application = new RecreateRepositoryApplication(agent);
application.setArtifactRepository(descriptor.getRepoLocation());
try {
application.run(new NullProgressMonitor());
Expand Down
34 changes: 27 additions & 7 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/MirrorApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicReference;

import org.eclipse.core.runtime.IConfigurationElement;
import org.eclipse.core.runtime.IExecutableExtension;
Expand Down Expand Up @@ -211,7 +212,8 @@ else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$

@Override
public IStatus run(IProgressMonitor monitor) throws ProvisionException {
IStatus mirrorStatus = Status.OK_STATUS;
AtomicReference<IStatus> mirrorStatus = new AtomicReference<>(Status.OK_STATUS);
AtomicReference<ProvisionException> exception = new AtomicReference<>();
try {
initializeRepos(new NullProgressMonitor());
initializeLogs();
Expand All @@ -220,20 +222,38 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException {
IQueryable<IInstallableUnit> slice = slice(new NullProgressMonitor());
Set<IInstallableUnit> units = collectUnits(slice, monitor);
if (destinationArtifactRepository != null) {
mirrorStatus = mirrorArtifacts(units, new NullProgressMonitor());
if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR)
return mirrorStatus;
destinationArtifactRepository.executeBatch(m -> {
try {
mirrorStatus.set(mirrorArtifacts(units, m));
} catch (ProvisionException e) {
exception.set(e);
}
}, new NullProgressMonitor());
if (exception.get() != null) {
throw exception.get();
}
if (failOnError && mirrorStatus.get().getSeverity() == IStatus.ERROR)
return mirrorStatus.get();
}
if (destinationMetadataRepository != null) {
mirrorMetadata(units, new NullProgressMonitor());
destinationMetadataRepository.executeBatch(m -> {
try {
mirrorMetadata(units, m);
} catch (ProvisionException e) {
exception.set(e);
}
}, new NullProgressMonitor());
if (exception.get() != null) {
throw exception.get();
}
}
} finally {
finalizeRepositories();
finalizeLogs();
}
if (mirrorStatus.isOK())
if (mirrorStatus.get().isOK())
return Status.OK_STATUS;
return mirrorStatus;
return mirrorStatus.get();
}

private IStatus mirrorArtifacts(Collection<IInstallableUnit> slice, IProgressMonitor monitor)
Expand Down
66 changes: 36 additions & 30 deletions ...e/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/RecreateRepositoryApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@

import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.MultiStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.URIUtil;
import org.eclipse.equinox.internal.p2.artifact.processors.checksum.ChecksumUtilities;
import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository;
import org.eclipse.equinox.internal.p2.core.helpers.LogHelper;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.internal.repository.tools.Messages;
Expand Down Expand Up @@ -63,7 +63,10 @@ public IStatus run(IProgressMonitor monitor) throws ProvisionException {
try {
IArtifactRepository repository = initialize(monitor);
removeRepository(repository, monitor);
recreateRepository(monitor);
MultiStatus status = recreateRepository(monitor);
if (status.isOK()) {
return status;
}
} finally {
if (removeArtifactRepo) {
IArtifactRepositoryManager repositoryManager = getArtifactRepositoryManager();
Expand Down Expand Up @@ -115,7 +118,7 @@ private void removeRepository(IArtifactRepository repository, IProgressMonitor m
throw new ProvisionException(NLS.bind(Messages.exception_unableToRemoveRepo, realFile.toString()));
}

private void recreateRepository(IProgressMonitor monitor) throws ProvisionException {
private MultiStatus recreateRepository(IProgressMonitor monitor) throws ProvisionException {
IArtifactRepositoryManager manager = getArtifactRepositoryManager();

IArtifactRepository repository = manager.createRepository(repoLocation, repoName,
Expand All @@ -124,33 +127,36 @@ private void recreateRepository(IProgressMonitor monitor) throws ProvisionExcept
throw new ProvisionException(NLS.bind(Messages.exception_notLocalFileRepo, repository.getLocation()));

IFileArtifactRepository simple = (IFileArtifactRepository) repository;
for (IArtifactKey key : repoMap.keySet()) {
IArtifactDescriptor[] descriptors = repoMap.get(key);

Set<File> files = new HashSet<>();
for (IArtifactDescriptor descriptor : descriptors) {
File artifactFile = simple.getArtifactFile(descriptor);
files.add(artifactFile);

String size = Long.toString(artifactFile.length());

ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor);
newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size);
newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size);

Map<String, String> checksums = new HashMap<>();
List<String> checksumsToSkip = Collections.emptyList();
IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip);
if (!status.isOK())
// TODO handle errors in some way
LogHelper.log(status);

Map<String, String> checksumsToProperties = ChecksumUtilities
.checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums);
newDescriptor.addProperties(checksumsToProperties);

repository.addDescriptor(newDescriptor, null);
MultiStatus multiStatus = new MultiStatus(getClass(), 0, "Problem while recreate repository");
repository.executeBatch(m -> {
for (IArtifactKey key : repoMap.keySet()) {
IArtifactDescriptor[] descriptors = repoMap.get(key);

Set<File> files = new HashSet<>();
for (IArtifactDescriptor descriptor : descriptors) {
File artifactFile = simple.getArtifactFile(descriptor);
files.add(artifactFile);

String size = Long.toString(artifactFile.length());

ArtifactDescriptor newDescriptor = new ArtifactDescriptor(descriptor);
newDescriptor.setProperty(IArtifactDescriptor.ARTIFACT_SIZE, size);
newDescriptor.setProperty(IArtifactDescriptor.DOWNLOAD_SIZE, size);

Map<String, String> checksums = new HashMap<>();
List<String> checksumsToSkip = Collections.emptyList();
IStatus status = ChecksumUtilities.calculateChecksums(artifactFile, checksums, checksumsToSkip);
if (!status.isOK()) {
multiStatus.add(status);
}
Map<String, String> checksumsToProperties = ChecksumUtilities
.checksumsToProperties(IArtifactDescriptor.DOWNLOAD_CHECKSUM, checksums);
newDescriptor.addProperties(checksumsToProperties);

repository.addDescriptor(newDescriptor, null);
}
}
}
}, monitor);
return multiStatus;
}
}
28 changes: 16 additions & 12 deletions tycho-core/src/main/java/org/eclipse/tycho/p2tools/copiedfromp2/Slicer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,24 +205,28 @@ private Collection<IRequirement> getRequirements(IInstallableUnit iu) {
return aggregatedRequirements;
}

private Set<IRequirement> consideredRequirements = new HashSet<>();

private void expandRequirement(IInstallableUnit iu, IRequirement req) {
if (req.getMax() == 0) {
return;
}
List<IInstallableUnit> selected = selectIUsForRequirement(possibilites, req).toList();
for (IInstallableUnit match : selected) {
Map<Version, IInstallableUnit> iuSlice = slice.get(match.getId());
if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) {
toProcess.add(match);
if (consideredRequirements.add(req)) {
List<IInstallableUnit> selected = selectIUsForRequirement(possibilites, req).toList();
for (IInstallableUnit match : selected) {
Map<Version, IInstallableUnit> iuSlice = slice.get(match.getId());
if ((iuSlice == null || !iuSlice.containsKey(match.getVersion())) && considered.add(match)) {
toProcess.add(match);
}
}
}
if (selected.isEmpty()) {
if (req.getMin() == 0) {
if (DEBUG) {
System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$
if (selected.isEmpty()) {
if (req.getMin() == 0) {
if (DEBUG) {
System.out.println("No IU found to satisfy optional dependency of " + iu + " on req " + req); //$NON-NLS-1$//$NON-NLS-2$
}
} else {
result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req)));
}
} else {
result.add(Status.warning(NLS.bind(Messages.Planner_Unsatisfied_dependency, iu, req)));
}
}
}
Expand Down

0 comments on commit c86d9ea

Please sign in to comment.