Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support assembling and archiving repositories in parallel #2850

Merged
merged 1 commit into from
Oct 23, 2023
Merged

Support assembling and archiving repositories in parallel #2850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,10 @@ default Closeable lock(File file) {
* advisory only, i.e. all processes must use the same locking mechanism.
*/
Closeable lock(File file, long timeout);

/**
* Locks the given file for this JVM to protect read/write access from multiple threads in this
* JVM on it.
*/
Closeable lockVirtually(File file);
}
36 changes: 31 additions & 5 deletions tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,30 +19,56 @@
import java.nio.file.Path;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.codehaus.plexus.component.annotations.Component;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.LockTimeoutException;

@Component(role = FileLockService.class)
public class FileLockServiceImpl implements FileLockService {
record FileLocks(FileLockerImpl fileLocker, Lock vmLock) {
}

private final Map<Path, FileLockerImpl> lockers = new ConcurrentHashMap<>();
private final Map<Path, FileLocks> lockers = new ConcurrentHashMap<>();

@Override
public Closeable lock(File file, long timeout) {
FileLockerImpl locker = getFileLocker(file.toPath());
FileLocks locks = getFileLocker(file.toPath());
FileLockerImpl locker = locks.fileLocker();
try {
if (!locks.vmLock().tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new LockTimeoutException("lock timeout: Could not acquire lock on file " + locker.lockMarkerFile
+ " for " + timeout + " msec");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new LockTimeoutException("Interrupted", e);
}
locker.lock(timeout);
return locker::release;
return () -> {
locks.fileLocker().release();
locks.vmLock().unlock();
};
}

@Override
public Closeable lockVirtually(File file) {
FileLocks locks = getFileLocker(file.toPath());
locks.vmLock().lock();
return locks.vmLock()::unlock;
}

FileLockerImpl getFileLocker(Path file) {
FileLocks getFileLocker(Path file) {
Path key;
try {
key = file.toRealPath();
} catch (IOException e) {
key = file.toAbsolutePath().normalize();
}
return lockers.computeIfAbsent(key, FileLockerImpl::new);
return lockers.computeIfAbsent(key, f -> new FileLocks(new FileLockerImpl(f), new ReentrantLock()));
}

}
2 changes: 1 addition & 1 deletion tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ private File newTestFile() throws IOException {
}

private Path getLockMarkerFile(File file) {
return subject.getFileLocker(file.toPath()).lockMarkerFile;
return subject.getFileLocker(file.toPath()).fileLocker().lockMarkerFile;
}

private boolean isLocked(File file) throws IOException {
Expand Down
30 changes: 14 additions & 16 deletions ...y-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.codehaus.plexus.archiver.Archiver;
import org.codehaus.plexus.archiver.ArchiverException;
import org.codehaus.plexus.archiver.util.DefaultFileSet;
import org.eclipse.tycho.FileLockService;

/**
* <p>
Expand All @@ -33,7 +34,6 @@
*/
@Mojo(name = "archive-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true)
public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Component(role = Archiver.class, hint = "zip")
private Archiver inflater;
Expand All @@ -52,27 +52,25 @@ public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo {
@Parameter(defaultValue = "false")
private boolean skipArchive;

@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (skipArchive) {
return;
}

synchronized (LOCK) {
File destFile = getBuildDirectory().getChild(finalName + ".zip");

try {
inflater.addFileSet(DefaultFileSet.fileSet(getAssemblyRepositoryLocation()).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
} catch (IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}

getProject().getArtifact().setFile(destFile);
File repositoryLocation = getAssemblyRepositoryLocation();
File destFile = getBuildDirectory().getChild(finalName + ".zip");
try (var repoLock = fileLockService.lockVirtually(repositoryLocation);
var destLock = fileLockService.lockVirtually(destFile);) {
inflater.addFileSet(DefaultFileSet.fileSet(repositoryLocation).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException | IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}
getProject().getArtifact().setFile(destFile);
}

}
153 changes: 76 additions & 77 deletions ...-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.MatchPattern;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.PackagingType;
import org.eclipse.tycho.ReactorProject;
import org.eclipse.tycho.TychoConstants;
Expand Down Expand Up @@ -87,7 +88,6 @@ public static class RepositoryReferenceFilter {
public List<String> exclude = List.of();
}

private static final Object LOCK = new Object();
/**
* <p>
* By default, this goal creates a p2 repository. Set this to <code>false</code> if only a p2
Expand Down Expand Up @@ -309,92 +309,91 @@ public static class RepositoryReferenceFilter {

@Component(role = TychoProject.class, hint = PackagingType.TYPE_ECLIPSE_REPOSITORY)
private EclipseRepositoryProject eclipseRepositoryProject;
@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}
File destination = getAssemblyRepositoryLocation();
try (var locking = fileLockService.lockVirtually(destination)) {
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}

reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(),
getSession(), RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.toList();
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
List<RepositoryReference> autoRepositoryRefeferences = new ArrayList<>();
if (addPomRepositoryReferences) {
getProject().getRepositories().stream() //
.filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
.filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
.map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
.forEach(autoRepositoryRefeferences::add);
}
if (addIUTargetRepositoryReferences) {
projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
.flatMap(tpFile -> tpFile.getLocations().stream())
.filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast)
.flatMap(iu -> iu.getRepositories().stream())
.filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
.map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
.forEach(autoRepositoryRefeferences::add);
reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), getSession(),
RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.toList();
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
List<RepositoryReference> autoRepositoryRefeferences = new ArrayList<>();
if (addPomRepositoryReferences) {
getProject().getRepositories().stream() //
.filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
.filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
.map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
.forEach(autoRepositoryRefeferences::add);
}
if (addIUTargetRepositoryReferences) {
projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
.flatMap(tpFile -> tpFile.getLocations().stream())
.filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast)
.flatMap(iu -> iu.getRepositories().stream())
.filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
.map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
.forEach(autoRepositoryRefeferences::add);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true,
extraArtifactRepositoryProperties, repositoryReferences, autoRepositoryRefeferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File plugins = new File(destination, "plugins");
if (plugins.isDirectory()) {
File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles,
!createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences,
autoRepositoryRefeferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File plugins = new File(destination, "plugins");
if (plugins.isDirectory()) {
File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
File features = new File(destination, "features");
if (features.isDirectory()) {
File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
for (File featureFile : files) {
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
}
File features = new File(destination, "features");
if (features.isDirectory()) {
File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
for (File featureFile : files) {
try {
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
}
try {
String filename = compress ? repositoryFileName + ".gz" : repositoryFileName;
resourceGenerator.save(new File(destination, filename));
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
}
} catch (FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
try {
String filename = compress ? repositoryFileName + ".gz" : repositoryFileName;
resourceGenerator.save(new File(destination, filename));
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
}
} catch (IOException | FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
}
}

Expand Down
30 changes: 16 additions & 14 deletions ...c/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@
package org.eclipse.tycho.plugins.p2.repository;

import java.io.File;
import java.io.IOException;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Component;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.p2.tools.DestinationRepositoryDescriptor;
import org.eclipse.tycho.p2.tools.FacadeException;
import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService;
Expand All @@ -36,7 +38,7 @@
*/
@Mojo(name = "fix-artifacts-metadata", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true)
public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Parameter(defaultValue = "${project.name}")
private String repositoryName;

Expand All @@ -59,24 +61,24 @@ public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {
@Parameter(defaultValue = "true")
private boolean keepNonXzIndexFiles;

@Component()
@Component
MirrorApplicationService mirrorApp;
@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true);
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
File destination = getAssemblyRepositoryLocation();
try (var locking = fileLockService.lockVirtually(destination)) {
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true);
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (IOException | FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
}
}

Expand Down
Loading
Loading