Skip to content

Commit

Permalink
Support assembling and archiving repositories in parallel
Browse files Browse the repository at this point in the history 
Assembling and arching repositories is a relatively long running task
and repository-projects are usually build as one of the last projects in
the reactor. So the chances are relatively high, that (if multiple
projects are present) multiple repositories are assembled at the same.
Therefore it should be possible to build multiple repositories in
parallel and there should not be one global lock for each
repository-related mojo.
  • Loading branch information
@HannesWell @laeubi
HannesWell authored and laeubi committed Oct 23, 2023
1 parent edb7287 commit 7d1eb64
Show file tree
Hide file tree
Showing 8 changed files with 162 additions and 120 deletions.
6 changes: 6 additions & 0 deletions tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,10 @@ default Closeable lock(File file) {
* advisory only, i.e. all processes must use the same locking mechanism.
*/
Closeable lock(File file, long timeout);

/**
* Locks the given file for this JVM to protect read/write access from multiple threads in this
* JVM on it.
*/
Closeable lockVirtually(File file);
}
36 changes: 31 additions & 5 deletions tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,30 +19,56 @@
import java.nio.file.Path;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.codehaus.plexus.component.annotations.Component;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.LockTimeoutException;

@Component(role = FileLockService.class)
public class FileLockServiceImpl implements FileLockService {
record FileLocks(FileLockerImpl fileLocker, Lock vmLock) {
}

private final Map<Path, FileLockerImpl> lockers = new ConcurrentHashMap<>();
private final Map<Path, FileLocks> lockers = new ConcurrentHashMap<>();

@Override
public Closeable lock(File file, long timeout) {
FileLockerImpl locker = getFileLocker(file.toPath());
FileLocks locks = getFileLocker(file.toPath());
FileLockerImpl locker = locks.fileLocker();
try {
if (!locks.vmLock().tryLock(timeout, TimeUnit.MILLISECONDS)) {
throw new LockTimeoutException("lock timeout: Could not acquire lock on file " + locker.lockMarkerFile
+ " for " + timeout + " msec");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new LockTimeoutException("Interrupted", e);
}
locker.lock(timeout);
return locker::release;
return () -> {
locks.fileLocker().release();
locks.vmLock().unlock();
};
}

@Override
public Closeable lockVirtually(File file) {
FileLocks locks = getFileLocker(file.toPath());
locks.vmLock().lock();
return locks.vmLock()::unlock;
}

FileLockerImpl getFileLocker(Path file) {
FileLocks getFileLocker(Path file) {
Path key;
try {
key = file.toRealPath();
} catch (IOException e) {
key = file.toAbsolutePath().normalize();
}
return lockers.computeIfAbsent(key, FileLockerImpl::new);
return lockers.computeIfAbsent(key, f -> new FileLocks(new FileLockerImpl(f), new ReentrantLock()));
}

}
2 changes: 1 addition & 1 deletion tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ private File newTestFile() throws IOException {
}

private Path getLockMarkerFile(File file) {
return subject.getFileLocker(file.toPath()).lockMarkerFile;
return subject.getFileLocker(file.toPath()).fileLocker().lockMarkerFile;
}

private boolean isLocked(File file) throws IOException {
Expand Down
30 changes: 14 additions & 16 deletions ...y-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.codehaus.plexus.archiver.Archiver;
import org.codehaus.plexus.archiver.ArchiverException;
import org.codehaus.plexus.archiver.util.DefaultFileSet;
import org.eclipse.tycho.FileLockService;

/**
* <p>
Expand All @@ -33,7 +34,6 @@
*/
@Mojo(name = "archive-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true)
public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Component(role = Archiver.class, hint = "zip")
private Archiver inflater;
Expand All @@ -52,27 +52,25 @@ public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo {
@Parameter(defaultValue = "false")
private boolean skipArchive;

@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
if (skipArchive) {
return;
}

synchronized (LOCK) {
File destFile = getBuildDirectory().getChild(finalName + ".zip");

try {
inflater.addFileSet(DefaultFileSet.fileSet(getAssemblyRepositoryLocation()).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
} catch (IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}

getProject().getArtifact().setFile(destFile);
File repositoryLocation = getAssemblyRepositoryLocation();
File destFile = getBuildDirectory().getChild(finalName + ".zip");
try (var repoLock = fileLockService.lockVirtually(repositoryLocation);
var destLock = fileLockService.lockVirtually(destFile);) {
inflater.addFileSet(DefaultFileSet.fileSet(repositoryLocation).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException | IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}
getProject().getArtifact().setFile(destFile);
}

}
153 changes: 76 additions & 77 deletions ...-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.MatchPattern;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.PackagingType;
import org.eclipse.tycho.ReactorProject;
import org.eclipse.tycho.TychoConstants;
Expand Down Expand Up @@ -87,7 +88,6 @@ public static class RepositoryReferenceFilter {
public List<String> exclude = List.of();
}

private static final Object LOCK = new Object();
/**
* <p>
* By default, this goal creates a p2 repository. Set this to <code>false</code> if only a p2
Expand Down Expand Up @@ -309,92 +309,91 @@ public static class RepositoryReferenceFilter {

@Component(role = TychoProject.class, hint = PackagingType.TYPE_ECLIPSE_REPOSITORY)
private EclipseRepositoryProject eclipseRepositoryProject;
@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}
File destination = getAssemblyRepositoryLocation();
try (var locking = fileLockService.lockVirtually(destination)) {
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}

reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(),
getSession(), RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.toList();
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
List<RepositoryReference> autoRepositoryRefeferences = new ArrayList<>();
if (addPomRepositoryReferences) {
getProject().getRepositories().stream() //
.filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
.filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
.map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
.forEach(autoRepositoryRefeferences::add);
}
if (addIUTargetRepositoryReferences) {
projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
.flatMap(tpFile -> tpFile.getLocations().stream())
.filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast)
.flatMap(iu -> iu.getRepositories().stream())
.filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
.map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
.forEach(autoRepositoryRefeferences::add);
reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), getSession(),
RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.toList();
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
List<RepositoryReference> autoRepositoryRefeferences = new ArrayList<>();
if (addPomRepositoryReferences) {
getProject().getRepositories().stream() //
.filter(pomRepo -> "p2".equals(pomRepo.getLayout()))
.filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl()))
.map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true))
.forEach(autoRepositoryRefeferences::add);
}
if (addIUTargetRepositoryReferences) {
projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream()
.flatMap(tpFile -> tpFile.getLocations().stream())
.filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast)
.flatMap(iu -> iu.getRepositories().stream())
.filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation()))
.map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true))
.forEach(autoRepositoryRefeferences::add);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true,
extraArtifactRepositoryProperties, repositoryReferences, autoRepositoryRefeferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File plugins = new File(destination, "plugins");
if (plugins.isDirectory()) {
File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles,
!createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences,
autoRepositoryRefeferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File plugins = new File(destination, "plugins");
if (plugins.isDirectory()) {
File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
File features = new File(destination, "features");
if (features.isDirectory()) {
File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
for (File featureFile : files) {
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
}
File features = new File(destination, "features");
if (features.isDirectory()) {
File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile());
for (File featureFile : files) {
try {
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
}
try {
String filename = compress ? repositoryFileName + ".gz" : repositoryFileName;
resourceGenerator.save(new File(destination, filename));
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
}
} catch (FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
try {
String filename = compress ? repositoryFileName + ".gz" : repositoryFileName;
resourceGenerator.save(new File(destination, filename));
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
}
} catch (IOException | FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
}
}

Expand Down
30 changes: 16 additions & 14 deletions ...c/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@
package org.eclipse.tycho.plugins.p2.repository;

import java.io.File;
import java.io.IOException;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Component;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.eclipse.tycho.FileLockService;
import org.eclipse.tycho.p2.tools.DestinationRepositoryDescriptor;
import org.eclipse.tycho.p2.tools.FacadeException;
import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService;
Expand All @@ -36,7 +38,7 @@
*/
@Mojo(name = "fix-artifacts-metadata", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true)
public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Parameter(defaultValue = "${project.name}")
private String repositoryName;

Expand All @@ -59,24 +61,24 @@ public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {
@Parameter(defaultValue = "true")
private boolean keepNonXzIndexFiles;

@Component()
@Component
MirrorApplicationService mirrorApp;
@Component
private FileLockService fileLockService;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true);
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
File destination = getAssemblyRepositoryLocation();
try (var locking = fileLockService.lockVirtually(destination)) {
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true);
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (IOException | FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
}
}

Expand Down
Loading

0 comments on commit 7d1eb64

Please sign in to comment.