Skip to content

Commit

Permalink
Support assembling and archiving repositories in parallel
Browse files Browse the repository at this point in the history 
Assembling and arching repositories is a relatively long running task
and repository-projects are usually build as one of the last projects in
the reactor. So the chances are relatively high, that (if multiple
projects are present) multiple repositories are assembled at the same.
Therefore it should be possible to build multiple repositories in
parallel and there should not be one global lock for each
repository-related mojo.
  • Loading branch information
@HannesWell
HannesWell committed Sep 23, 2023
1 parent 8faacee commit c58f295
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 115 deletions.
22 changes: 22 additions & 0 deletions ...-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AbstractRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,33 @@
package org.eclipse.tycho.plugins.p2.repository;

import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Parameter;
import org.eclipse.tycho.core.maven.AbstractP2Mojo;

public abstract class AbstractRepositoryMojo extends AbstractP2Mojo {
private static final Map<File, Lock> REPOSITORY_LOCK = new ConcurrentHashMap<>();

interface Locking extends AutoCloseable {
@Override
void close(); // Don't throw an Exception
}

static Locking lockFor(File file) throws MojoExecutionException {
try {
Lock lock = REPOSITORY_LOCK.computeIfAbsent(file.getCanonicalFile(), k -> new ReentrantLock());
lock.lock();
return lock::unlock;
} catch (IOException e) {
throw new MojoExecutionException("Error while canonicalizing repository location", e);
}
}

@Parameter
private File repositoryLocation;
Expand Down
27 changes: 11 additions & 16 deletions ...y-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/ArchiveRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
*/
@Mojo(name = "archive-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true)
public final class ArchiveRepositoryMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Component(role = Archiver.class, hint = "zip")
private Archiver inflater;
Expand All @@ -57,22 +56,18 @@ public void execute() throws MojoExecutionException, MojoFailureException {
if (skipArchive) {
return;
}

synchronized (LOCK) {
File destFile = getBuildDirectory().getChild(finalName + ".zip");

try {
inflater.addFileSet(DefaultFileSet.fileSet(getAssemblyRepositoryLocation()).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
} catch (IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}

getProject().getArtifact().setFile(destFile);
File repositoryLocation = getAssemblyRepositoryLocation();
File destFile = getBuildDirectory().getChild(finalName + ".zip");
try (var repoLock = lockFor(repositoryLocation); var destLock = lockFor(destFile);) {
inflater.addFileSet(DefaultFileSet.fileSet(repositoryLocation).prefixed(""));
inflater.setDestFile(destFile);
inflater.createArchive();
} catch (ArchiverException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
} catch (IOException e) {
throw new MojoExecutionException("Error packing p2 repository", e);
}
getProject().getArtifact().setFile(destFile);
}

}
159 changes: 78 additions & 81 deletions ...-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ public static class RepositoryReferenceFilter {
List<String> include;
}

private static final Object LOCK = new Object();
/**
* <p>
* By default, this goal creates a p2 repository. Set this to <code>false</code> if only a p2
Expand Down Expand Up @@ -285,102 +284,100 @@ public static class RepositoryReferenceFilter {

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}
File destination = getAssemblyRepositoryLocation();
try (var locking = lockFor(destination)) {
destination.mkdirs();
copyResources(destination);

final ReactorProject reactorProject = getReactorProject();
Collection<DependencySeed> projectSeeds = TychoProjectUtils.getDependencySeeds(reactorProject);
if (projectSeeds.isEmpty()) {
getLog().warn("No content specified for p2 repository");
return;
}

reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(),
getSession(), RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.collect(Collectors.toCollection(ArrayList::new));
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
if (addPomRepositoryReferences) {
for (Repository pomRepo : getProject().getRepositories()) {
if ("p2".equals(pomRepo.getLayout())) {
String locationURL = pomRepo.getUrl();
if (autoReferencesFilter.test(locationURL)) {
repositoryReferences.add(new RepositoryReference(pomRepo.getName(), locationURL, true));
}
reactorProject.setContextValue(TychoConstants.CTX_METADATA_ARTIFACT_LOCATION, categoriesDirectory);
RepositoryReferences sources = repositoryReferenceTool.getVisibleRepositories(getProject(), getSession(),
RepositoryReferenceTool.REPOSITORIES_INCLUDE_CURRENT_MODULE);
sources.setTargetPlatform(TychoProjectUtils.getTargetPlatform(getReactorProject()));

List<RepositoryReference> repositoryReferences = getCategories(categoriesDirectory).stream()//
.map(Category::getRepositoryReferences)//
.flatMap(List::stream)//
.map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))//
.collect(Collectors.toCollection(ArrayList::new));
Predicate<String> autoReferencesFilter = buildRepositoryReferenceLocationFilter();
if (addPomRepositoryReferences) {
for (Repository pomRepo : getProject().getRepositories()) {
if ("p2".equals(pomRepo.getLayout())) {
String locationURL = pomRepo.getUrl();
if (autoReferencesFilter.test(locationURL)) {
repositoryReferences.add(new RepositoryReference(pomRepo.getName(), locationURL, true));
}
}
}
if (addIUTargetRepositoryReferences) {
for (TargetDefinitionFile targetDefinitionFile : projectManager
.getTargetPlatformConfiguration(getProject()).getTargets()) {
for (Location location : targetDefinitionFile.getLocations()) {
if (location instanceof InstallableUnitLocation iu) {
for (var iuRepo : iu.getRepositories()) {
String locationURL = iuRepo.getLocation();
if (autoReferencesFilter.test(locationURL)) {
repositoryReferences.add(new RepositoryReference(null, locationURL, true));
}
}
if (addIUTargetRepositoryReferences) {
for (TargetDefinitionFile targetDefinitionFile : projectManager
.getTargetPlatformConfiguration(getProject()).getTargets()) {
for (Location location : targetDefinitionFile.getLocations()) {
if (location instanceof InstallableUnitLocation iu) {
for (var iuRepo : iu.getRepositories()) {
String locationURL = iuRepo.getLocation();
if (autoReferencesFilter.test(locationURL)) {
repositoryReferences.add(new RepositoryReference(null, locationURL, true));
}
}
}
}
}
}

DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles,
!createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File pluginsResult = new File(destination, "plugins");
if (pluginsResult.isDirectory()) {
File[] files = pluginsResult
.listFiles(pathname -> pathname.isFile() && pathname.getName().endsWith(".jar"));
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true,
extraArtifactRepositoryProperties, repositoryReferences);
mirrorApp.mirrorReactor(sources, destinationRepoDescriptor, projectSeeds, getBuildContext(),
includeAllDependencies, includeAllSources, includeRequiredPlugins, includeRequiredFeatures,
filterProvided, profileProperties);
if (generateOSGiRepository) {
XMLResourceGenerator resourceGenerator = new XMLResourceGenerator();
resourceGenerator.name(repositoryName);
resourceGenerator.base(destination.toURI());
File pluginsResult = new File(destination, "plugins");
if (pluginsResult.isDirectory()) {
File[] files = pluginsResult
.listFiles(pathname -> pathname.isFile() && pathname.getName().endsWith(".jar"));
try {
resourceGenerator.repository(new FileSetRepository("plugins", Arrays.asList(files)));
} catch (Exception e) {
throw new MojoExecutionException("Could not read p2 repository plugins", e);
}
File featureResult = new File(destination, "features");
if (featureResult.isDirectory()) {
File[] files = featureResult
.listFiles(pathname -> pathname.isFile() && pathname.getName().endsWith(".jar"));
for (File featureFile : files) {
try {
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
}
File featureResult = new File(destination, "features");
if (featureResult.isDirectory()) {
File[] files = featureResult
.listFiles(pathname -> pathname.isFile() && pathname.getName().endsWith(".jar"));
for (File featureFile : files) {
try {
Feature feature = Feature.readJar(featureFile);
feature.toResource().forEach(resourceGenerator::resource);
} catch (IOException e) {
throw new MojoExecutionException("Could not read feature " + featureFile, e);
}
}
try {
if (compress) {
resourceGenerator.save(new File(destination, repositoryFileName + ".gz"));
} else {
resourceGenerator.save(new File(destination, repositoryFileName));
}
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
try {
if (compress) {
resourceGenerator.save(new File(destination, repositoryFileName + ".gz"));
} else {
resourceGenerator.save(new File(destination, repositoryFileName));
}
} catch (IOException e) {
throw new MojoExecutionException("Could not write OSGi Repository!", e);
}
} catch (FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
}
} catch (FacadeException e) {
throw new MojoExecutionException("Could not assemble p2 repository", e);
}
}

Expand Down
26 changes: 12 additions & 14 deletions ...c/main/java/org/eclipse/tycho/plugins/p2/repository/FixArtifactsMetadataMetadataMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
*/
@Mojo(name = "fix-artifacts-metadata", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true)
public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {
private static final Object LOCK = new Object();

@Parameter(defaultValue = "${project.name}")
private String repositoryName;

Expand Down Expand Up @@ -65,20 +65,18 @@ public class FixArtifactsMetadataMetadataMojo extends AbstractRepositoryMojo {

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
synchronized (LOCK) {
try {
File destination = getAssemblyRepositoryLocation();
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(
destination, repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true,
Collections.emptyMap(), Collections.emptyList());
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
File destination = getAssemblyRepositoryLocation();
try (var locking = lockFor(destination)) {
if (!destination.isDirectory()) {
throw new MojoExecutionException(
"Could not update p2 repository, directory does not exist: " + destination);
}
DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor(destination,
repositoryName, true, xzCompress, keepNonXzIndexFiles, false, true, Collections.emptyMap(),
Collections.emptyList());
mirrorApp.recreateArtifactRepository(destinationRepoDescriptor);
} catch (FacadeException e) {
throw new MojoExecutionException("Could not update p2 repository", e);
}
}

Expand Down
10 changes: 6 additions & 4 deletions ...in/java/org/eclipse/tycho/plugins/p2/repository/RemapArtifactToMavenRepositoriesMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*******************************************************************************/
package org.eclipse.tycho.plugins.p2.repository;

import java.io.File;
import java.net.URI;

import org.apache.maven.artifact.repository.ArtifactRepository;
Expand All @@ -25,17 +26,18 @@
* artifacts the can be resolved to Maven repositories so the URL under Maven repository is used for
* fetching and artifact is not duplicated inside this repo.
*/
@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE)
@Mojo(name = "remap-artifacts-to-m2-repo", defaultPhase = LifecyclePhase.PREPARE_PACKAGE, threadSafe = true)
public class RemapArtifactToMavenRepositoriesMojo extends AbstractRepositoryMojo {

@Component()
MirrorApplicationService mirrorApp;

@Override
public void execute() throws MojoExecutionException, MojoFailureException {
try {
mirrorApp.addMavenMappingRules(getAssemblyRepositoryLocation(),
getProject().getRemoteArtifactRepositories().stream() //
File location = getAssemblyRepositoryLocation();
try (var locking = lockFor(location)) {
mirrorApp.addMavenMappingRules( //
location, getProject().getRemoteArtifactRepositories().stream() //
.filter(artifactRepo -> artifactRepo.getLayout().getId().equals("default")) //
.map(ArtifactRepository::getUrl) //
.map(URI::create) //
Expand Down

0 comments on commit c58f295

Please sign in to comment.