From f934b1e7e5b705a28f9c1068499bc54f3cbb953c Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sun, 3 Sep 2023 23:58:41 +0200 Subject: [PATCH] Support assembling and archiving repositories in parallel Assembling and arching repositories is a relatively long running task and repository-projects are usually build as one of the last projects in the reactor. So the chances are relatively high, that (if multiple projects are present) multiple repositories are assembled at the same. Therefore it should be possible to build multiple repositories in parallel and there should not be one global lock for each repository-related mojo. --- .../org/eclipse/tycho/FileLockService.java | 6 + .../core/locking/FileLockServiceImpl.java | 36 ++++- .../core/locking/FileLockServiceTest.java | 2 +- .../p2/repository/ArchiveRepositoryMojo.java | 30 ++-- .../p2/repository/AssembleRepositoryMojo.java | 153 +++++++++--------- .../FixArtifactsMetadataMetadataMojo.java | 30 ++-- .../RemapArtifactToMavenRepositoriesMojo.java | 18 ++- .../tycho/test/util/NoopFileLockService.java | 7 +- 8 files changed, 162 insertions(+), 120 deletions(-) diff --git a/tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java b/tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java index e775f3c804..4aa33f92c3 100644 --- a/tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java +++ b/tycho-api/src/main/java/org/eclipse/tycho/FileLockService.java @@ -37,4 +37,10 @@ default Closeable lock(File file) { * advisory only, i.e. all processes must use the same locking mechanism. */ Closeable lock(File file, long timeout); + + /** + * Locks the given file for this JVM to protect read/write access from multiple threads in this + * JVM on it. + */ + Closeable lockVirtually(File file); } diff --git a/tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java b/tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java index 01dd82efbb..51991a83a8 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/core/locking/FileLockServiceImpl.java @@ -19,30 +19,56 @@ import java.nio.file.Path; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.codehaus.plexus.component.annotations.Component; import org.eclipse.tycho.FileLockService; +import org.eclipse.tycho.LockTimeoutException; @Component(role = FileLockService.class) public class FileLockServiceImpl implements FileLockService { + record FileLocks(FileLockerImpl fileLocker, Lock vmLock) { + } - private final Map lockers = new ConcurrentHashMap<>(); + private final Map lockers = new ConcurrentHashMap<>(); @Override public Closeable lock(File file, long timeout) { - FileLockerImpl locker = getFileLocker(file.toPath()); + FileLocks locks = getFileLocker(file.toPath()); + FileLockerImpl locker = locks.fileLocker(); + try { + if (!locks.vmLock().tryLock(timeout, TimeUnit.MILLISECONDS)) { + throw new LockTimeoutException("lock timeout: Could not acquire lock on file " + locker.lockMarkerFile + + " for " + timeout + " msec"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new LockTimeoutException("Interrupted", e); + } locker.lock(timeout); - return locker::release; + return () -> { + locks.fileLocker().release(); + locks.vmLock().unlock(); + }; + } + + @Override + public Closeable lockVirtually(File file) { + FileLocks locks = getFileLocker(file.toPath()); + locks.vmLock().lock(); + return locks.vmLock()::unlock; } - FileLockerImpl getFileLocker(Path file) { + FileLocks getFileLocker(Path file) { Path key; try { key = file.toRealPath(); } catch (IOException e) { key = file.toAbsolutePath().normalize(); } - return lockers.computeIfAbsent(key, FileLockerImpl::new); + return lockers.computeIfAbsent(key, f -> new FileLocks(new FileLockerImpl(f), new ReentrantLock())); } } diff --git a/tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java b/tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java index a28b4cc356..7a3ca09bbd 100644 --- a/tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java +++ b/tycho-core/src/test/java/org/eclipse/tycho/core/locking/FileLockServiceTest.java @@ -149,7 +149,7 @@ private File newTestFile() throws IOException { } private Path getLockMarkerFile(File file) { - return subject.getFileLocker(file.toPath()).lockMarkerFile; + return subject.getFileLocker(file.toPath()).fileLocker().lockMarkerFile; } private boolean isLocked(File file) throws IOException { diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java index edf83d6e73..57b577d387 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java @@ -25,6 +25,7 @@ import org.codehaus.plexus.archiver.Archiver; import org.codehaus.plexus.archiver.ArchiverException; import org.codehaus.plexus.archiver.util.DefaultFileSet; +import org.eclipse.tycho.FileLockService; /** *

@@ -33,7 +34,6 @@ */ @Mojo(name = "archive-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true) public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo { - private static final Object LOCK = new Object(); @Component(role = Archiver.class, hint = "zip") private Archiver inflater; @@ -52,27 +52,25 @@ public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo { @Parameter(defaultValue = "false") private boolean skipArchive; + @Component + private FileLockService fileLockService; + @Override public void execute() throws MojoExecutionException, MojoFailureException { if (skipArchive) { return; } - - synchronized (LOCK) { - File destFile = getBuildDirectory().getChild(finalName + ".zip"); - - try { - inflater.addFileSet(DefaultFileSet.fileSet(getAssemblyRepositoryLocation()).prefixed("")); - inflater.setDestFile(destFile); - inflater.createArchive(); - } catch (ArchiverException e) { - throw new MojoExecutionException("Error packing p2 repository", e); - } catch (IOException e) { - throw new MojoExecutionException("Error packing p2 repository", e); - } - - getProject().getArtifact().setFile(destFile); + File repositoryLocation = getAssemblyRepositoryLocation(); + File destFile = getBuildDirectory().getChild(finalName + ".zip"); + try (var repoLock = fileLockService.lockVirtually(repositoryLocation); + var destLock = fileLockService.lockVirtually(destFile);) { + inflater.addFileSet(DefaultFileSet.fileSet(repositoryLocation).prefixed("")); + inflater.setDestFile(destFile); + inflater.createArchive(); + } catch (ArchiverException | IOException e) { + throw new MojoExecutionException("Error packing p2 repository", e); } + getProject().getArtifact().setFile(destFile); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java index ab381756a0..955a5a05ae 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java @@ -30,6 +30,7 @@ import org.apache.maven.plugins.annotations.Parameter; import org.codehaus.plexus.util.FileUtils; import org.codehaus.plexus.util.MatchPattern; +import org.eclipse.tycho.FileLockService; import org.eclipse.tycho.PackagingType; import org.eclipse.tycho.ReactorProject; import org.eclipse.tycho.TychoConstants; @@ -87,7 +88,6 @@ public static class RepositoryReferenceFilter { public List exclude = List.of(); } - private static final Object LOCK = new Object(); /** *

* By default, this goal creates a p2 repository. Set this to false if only a p2 @@ -309,92 +309,91 @@ public static class RepositoryReferenceFilter { @Component(role = TychoProject.class, hint = PackagingType.TYPE_ECLIPSE_REPOSITORY) private EclipseRepositoryProject eclipseRepositoryProject; + @Component + private FileLockService fileLockService; @Override public void execute() throws MojoExecutionException, MojoFailureException { - synchronized (LOCK) { - try { - File destination = getAssemblyRepositoryLocation(); - destination.mkdirs(); - copyResources(destination); - - final ReactorProject reactorProject = getReactorProject(); - Collection projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject); - if (projectSeeds.isEmpty()) { - getLog().warn("No content specified for p2 repository"); - return; - } + File destination = getAssemblyRepositoryLocation(); + try (var locking = fileLockService.lockVirtually(destination)) { + destination.mkdirs(); + copyResources(destination); + + final ReactorProject reactorProject = getReactorProject(); + Collection projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject); + if (projectSeeds.isEmpty()) { + getLog().warn("No content specified for p2 repository"); + return; + } - reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory); - RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), - getSession(), RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE); - sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject())); - - List repositoryReferences = getCategories(categoriesDirectory).stream()// - .map(Category::getRepositoryReferences)// - .flatMap(List::stream)// - .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// - .toList(); - Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); - List autoRepositoryRefeferences = new ArrayList<>(); - if (addPomRepositoryReferences) { - getProject().getRepositories().stream() // - .filter(pomRepo -> "p2".equals(pomRepo.getLayout())) - .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl())) - .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)) - .forEach(autoRepositoryRefeferences::add); - } - if (addIUTargetRepositoryReferences) { - projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream() - .flatMap(tpFile -> tpFile.getLocations().stream()) - .filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast) - .flatMap(iu -> iu.getRepositories().stream()) - .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation())) - .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true)) - .forEach(autoRepositoryRefeferences::add); + reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory); + RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), getSession(), + RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE); + sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject())); + + List repositoryReferences = getCategories(categoriesDirectory).stream()// + .map(Category::getRepositoryReferences)// + .flatMap(List::stream)// + .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// + .toList(); + Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); + List autoRepositoryRefeferences = new ArrayList<>(); + if (addPomRepositoryReferences) { + getProject().getRepositories().stream() // + .filter(pomRepo -> "p2".equals(pomRepo.getLayout())) + .filter(pomRepo -> autoReferencesFilter.test(pomRepo.getUrl())) + .map(pomRepo -> new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)) + .forEach(autoRepositoryRefeferences::add); + } + if (addIUTargetRepositoryReferences) { + projectManager.getTargetPlatformConfiguration(getProject()).getTargets().stream() + .flatMap(tpFile -> tpFile.getLocations().stream()) + .filter(InstallableUnitLocation.class::isInstance).map(InstallableUnitLocation.class::cast) + .flatMap(iu -> iu.getRepositories().stream()) + .filter(iuRepo -> autoReferencesFilter.test(iuRepo.getLocation())) + .map(iuRepo -> new RepositoryReference(null, iuRepo.getLocation(), true)) + .forEach(autoRepositoryRefeferences::add); + } + DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination, + repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true, + extraArtifactRepositoryProperties, repositoryReferences, autoRepositoryRefeferences); + mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(), + includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures, + filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties); + if (generateOSGiRepository) { + XMLResourceGenerator resourceGenerator = new XMLResourceGenerator(); + resourceGenerator.name(repositoryName); + resourceGenerator.base(destination.toURI()); + File plugins = new File(destination, "plugins"); + if (plugins.isDirectory()) { + File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + try { + resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files))); + } catch (Exception e) { + throw new MojoExecutionException("Could not read p2 repository plugins", e); + } } - DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( - destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles, - !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences, - autoRepositoryRefeferences); - mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(), - includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures, - filterProvided, repositoryReferenceFilter.addOnlyProviding, profileProperties); - if (generateOSGiRepository) { - XMLResourceGenerator resourceGenerator = new XMLResourceGenerator(); - resourceGenerator.name(repositoryName); - resourceGenerator.base(destination.toURI()); - File plugins = new File(destination, "plugins"); - if (plugins.isDirectory()) { - File[] files = plugins.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + File features = new File(destination, "features"); + if (features.isDirectory()) { + File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); + for (File featureFile : files) { try { - resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files))); - } catch (Exception e) { - throw new MojoExecutionException("Could not read p2 repository plugins", e); - } - } - File features = new File(destination, "features"); - if (features.isDirectory()) { - File[] files = features.listFiles(path -> path.getName().endsWith(".jar") && path.isFile()); - for (File featureFile : files) { - try { - Feature feature = Feature.readJar(featureFile); - feature.toResource().forEach(resourceGenerator::resource); - } catch (IOException e) { - throw new MojoExecutionException("Could not read feature " + featureFile, e); - } + Feature feature = Feature.readJar(featureFile); + feature.toResource().forEach(resourceGenerator::resource); + } catch (IOException e) { + throw new MojoExecutionException("Could not read feature " + featureFile, e); } } - try { - String filename = compress ? repositoryFileName + ".gz" : repositoryFileName; - resourceGenerator.save(new File(destination, filename)); - } catch (IOException e) { - throw new MojoExecutionException("Could not write OSGi Repository!", e); - } } - } catch (FacadeException e) { - throw new MojoExecutionException("Could not assemble p2 repository", e); + try { + String filename = compress ? repositoryFileName + ".gz" : repositoryFileName; + resourceGenerator.save(new File(destination, filename)); + } catch (IOException e) { + throw new MojoExecutionException("Could not write OSGi Repository!", e); + } } + } catch (IOException | FacadeException e) { + throw new MojoExecutionException("Could not assemble p2 repository", e); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java index b63d97b0df..6a7e7d592d 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java @@ -13,6 +13,7 @@ package org.eclipse.tycho.plugins.p2.repository; import java.io.File; +import java.io.IOException; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -20,6 +21,7 @@ import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; +import org.eclipse.tycho.FileLockService; import org.eclipse.tycho.p2.tools.DestinationRepositoryDescriptor; import org.eclipse.tycho.p2.tools.FacadeException; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService; @@ -36,7 +38,7 @@ */ @Mojo(name = "fix-artifacts-metadata", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true) public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo { - private static final Object LOCK = new Object(); + @Parameter(defaultValue = "${project.name}") private String repositoryName; @@ -59,24 +61,24 @@ public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo { @Parameter(defaultValue = "true") private boolean keepNonXzIndexFiles; - @Component() + @Component MirrorApplicationService mirrorApp; + @Component + private FileLockService fileLockService; @Override public void execute() throws MojoExecutionException, MojoFailureException { - synchronized (LOCK) { - try { - File destination = getAssemblyRepositoryLocation(); - if (!destination.isDirectory()) { - throw new MojoExecutionException( - "Could not update p2 repository, directory does not exist: " + destination); - } - DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( - destination, repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true); - mirrorApp.recreateArtifactRepository(destinationRepoDescriptor); - } catch (FacadeException e) { - throw new MojoExecutionException("Could not update p2 repository", e); + File destination = getAssemblyRepositoryLocation(); + try (var locking = fileLockService.lockVirtually(destination)) { + if (!destination.isDirectory()) { + throw new MojoExecutionException( + "Could not update p2 repository, directory does not exist: " + destination); } + DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination, + repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true); + mirrorApp.recreateArtifactRepository(destinationRepoDescriptor); + } catch (IOException | FacadeException e) { + throw new MojoExecutionException("Could not update p2 repository", e); } } diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java index 0600d4b007..15d69a1ed9 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java @@ -9,6 +9,8 @@ *******************************************************************************/ package org.eclipse.tycho.plugins.p2.repository; +import java.io.File; +import java.io.IOException; import java.net.URI; import org.apache.maven.artifact.repository.ArtifactRepository; @@ -17,6 +19,7 @@ import org.apache.maven.plugins.annotations.Component; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; +import org.eclipse.tycho.FileLockService; import org.eclipse.tycho.p2.tools.FacadeException; import org.eclipse.tycho.p2.tools.mirroring.facade.MirrorApplicationService; @@ -25,22 +28,25 @@ * artifacts the can be resolved to Maven repositories so the URL under Maven repository is used for * fetching and artifact is not duplicated inside this repo. */ -@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE) +@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true) public class RemapArtifactToMavenRepositoriesMojo extends AbstractRepositoryMojo { - @Component() + @Component MirrorApplicationService mirrorApp; + @Component + private FileLockService fileLockService; @Override public void execute() throws MojoExecutionException, MojoFailureException { - try { - mirrorApp.addMavenMappingRules(getAssemblyRepositoryLocation(), - getProject().getRemoteArtifactRepositories().stream() // + File location = getAssemblyRepositoryLocation(); + try (var locking = fileLockService.lockVirtually(location)) { + mirrorApp.addMavenMappingRules( // + location, getProject().getRemoteArtifactRepositories().stream() // .filter(artifactRepo -> artifactRepo.getLayout().getId().equals("default")) // .map(ArtifactRepository::getUrl) // .map(URI::create) // .toArray(URI[]::new)); - } catch (FacadeException e) { + } catch (IOException | FacadeException e) { throw new MojoExecutionException(e.getMessage(), e); } } diff --git a/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/NoopFileLockService.java b/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/NoopFileLockService.java index 18a8f9fdc2..566e9d38b0 100644 --- a/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/NoopFileLockService.java +++ b/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/NoopFileLockService.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2011, 2012 SAP SE and others. + * Copyright (c) 2011, 2023 SAP SE and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -22,6 +22,11 @@ public class NoopFileLockService implements FileLockService { @Override public Closeable lock(File file, long timeout) { + return lockVirtually(file); + } + + @Override + public Closeable lockVirtually(File file) { return () -> { }; }