Batch Maven project imports to reduce memory consumption

Signed-off-by: Snjezana Peco <[email protected]>
eclipse-jdtls · Apr 10, 2019 · 5f4346f · 5f4346f
1 parent 61d0056
commit 5f4346f
Show file tree

Hide file tree

Showing 16 changed files with 212 additions and 3 deletions.
diff --git a/...dt.ls.core/src/org/eclipse/jdt/ls/core/internal/handlers/WorkspaceDiagnosticsHandler.java b/...dt.ls.core/src/org/eclipse/jdt/ls/core/internal/handlers/WorkspaceDiagnosticsHandler.java
@@ -34,6 +34,7 @@
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.eclipse.jdt.core.ICompilationUnit;
 import org.eclipse.jdt.core.IJavaModelMarker;
+import org.eclipse.jdt.core.IJavaProject;
 import org.eclipse.jdt.core.JavaCore;
 import org.eclipse.jdt.core.JavaModelException;
 import org.eclipse.jdt.ls.core.internal.JDTUtils;
@@ -129,7 +130,8 @@ public boolean visit(IResourceDelta delta) throws CoreException {
 		if (JavaCore.isJavaLikeFileName(file.getName())) {
 			ICompilationUnit cu = (ICompilationUnit) JavaCore.create(file);
 			// Clear the diagnostics for the resource not on the classpath
-			if (cu.getJavaProject() == null || !cu.getJavaProject().isOnClasspath(cu)) {
+			IJavaProject javaProject = cu.getJavaProject();
+			if (javaProject == null || !javaProject.isOnClasspath(cu)) {
 				String uri = JDTUtils.getFileURI(resource);
 				this.connection.publishDiagnostics(new PublishDiagnosticsParams(ResourceUtils.toClientUri(uri), Collections.emptyList()));
 				return false;

diff --git a/...lipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/MavenProjectImporter.java b/...lipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/MavenProjectImporter.java
@@ -13,6 +13,7 @@
 import java.io.File;
 import java.nio.file.FileSystems;
 import java.nio.file.PathMatcher;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
@@ -42,6 +43,7 @@
 import org.eclipse.m2e.core.internal.MavenPluginActivator;
 import org.eclipse.m2e.core.internal.preferences.MavenConfigurationImpl;
 import org.eclipse.m2e.core.internal.preferences.ProblemSeverity;
+import org.eclipse.m2e.core.project.IMavenProjectImportResult;
 import org.eclipse.m2e.core.project.IProjectConfigurationManager;
 import org.eclipse.m2e.core.project.LocalProjectScanner;
 import org.eclipse.m2e.core.project.MavenProjectInfo;
@@ -50,6 +52,8 @@
 @SuppressWarnings("restriction")
 public class MavenProjectImporter extends AbstractProjectImporter {
 
+	private static final int MAX_PROJECTS_TO_IMPORT = 10;
+
 	public static final String IMPORTING_MAVEN_PROJECTS = "Importing Maven project(s)";
 
 	public static final String POM_FILE = "pom.xml";
@@ -149,8 +153,36 @@ public void importToWorkspace(IProgressMonitor monitor) throws CoreException, Op
 			}
 		}
 		if (!toImport.isEmpty()) {
-			ProjectImportConfiguration importConfig = new ProjectImportConfiguration();
-			configurationManager.importProjects(toImport, importConfig, subMonitor.split(75));
+			if (toImport.size() > MAX_PROJECTS_TO_IMPORT) {
+				JavaLanguageServerPlugin.logInfo("Projects size:" + toImport.size());
+				Iterator<MavenProjectInfo> iter = toImport.iterator();
+				List<IMavenProjectImportResult> results = new ArrayList<>(MAX_PROJECTS_TO_IMPORT);
+				SubMonitor monitor2 = SubMonitor.convert(monitor, toImport.size() * 2);
+				int it = 1;
+				while (iter.hasNext()) {
+					int percent = Math.min(100, it++ * 100 * MAX_PROJECTS_TO_IMPORT / (toImport.size() + 1));
+					monitor2.setTaskName(percent + "% " + IMPORTING_MAVEN_PROJECTS);
+					List<MavenProjectInfo> importPartial = new ArrayList<>();
+					int i = 0;
+					while (i++ < MAX_PROJECTS_TO_IMPORT && iter.hasNext()) {
+						importPartial.add(iter.next());
+					}
+					ProjectImportConfiguration importConfig = new ProjectImportConfiguration();
+					List<IMavenProjectImportResult> result = configurationManager.importProjects(importPartial, importConfig, monitor2.split(MAX_PROJECTS_TO_IMPORT));
+					results.addAll(result);
+					monitor2.setWorkRemaining(toImport.size() * 2 - it * MAX_PROJECTS_TO_IMPORT);
+				}
+				List<IProject> imported = new ArrayList<>(results.size());
+				for (IMavenProjectImportResult result : results) {
+					imported.add(result.getProject());
+				}
+				monitor2.setTaskName("Updating Maven project(s)");
+				updateProjects(imported, lastWorkspaceStateSaved, monitor2.split(projects.size()));
+				monitor2.done();
+			} else {
+				ProjectImportConfiguration importConfig = new ProjectImportConfiguration();
+				configurationManager.importProjects(toImport, importConfig, subMonitor.split(75));
+			}
 		}
 		subMonitor.setWorkRemaining(20);
 		updateProjects(projects, lastWorkspaceStateSaved, subMonitor.split(20));

diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch1/batchchild/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch1/batchchild/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>batch1</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batchchild</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch1/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch1/pom.xml
@@ -0,0 +1,17 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>batch</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch1</artifactId>
+	<packaging>pom</packaging>
+
+	<modules>
+		<module>batchchild</module>
+	</modules>
+
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch10/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch10/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch10</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch11/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch11/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch11</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch2/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch2/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch2</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch3/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch3/pom.xml
@@ -0,0 +1,12 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch3</artifactId>
+	<packaging>jar</packaging>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch4/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch4/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch4</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch5/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch5/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch5</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch6/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch6/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch6</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch7/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch7/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch7</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch8/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch8/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch8</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/batch9/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/batch9/pom.xml
@@ -0,0 +1,11 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>foo.bar</groupId>
+		<artifactId>multimodule</artifactId>
+		<version>0.0.1-SNAPSHOT</version>
+	</parent>
+	<artifactId>batch9</artifactId>
+</project>
diff --git a/org.eclipse.jdt.ls.tests/projects/maven/batch/pom.xml b/org.eclipse.jdt.ls.tests/projects/maven/batch/pom.xml
@@ -0,0 +1,22 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>foo.bar</groupId>
+	<artifactId>batch</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<packaging>pom</packaging>
+	<modules>
+		<module>batch1</module>
+		<module>batch2</module>
+		<module>batch3</module>
+		<module>batch4</module>
+		<module>batch5</module>
+		<module>batch6</module>
+		<module>batch7</module>
+		<module>batch8</module>
+		<module>batch9</module>
+		<module>batch10</module>
+		<module>batch11</module>
+	</modules>
+</project>
diff --git a/...pse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/managers/MavenBuildSupportTest.java b/...pse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/managers/MavenBuildSupportTest.java
@@ -28,7 +28,9 @@
 import org.apache.commons.io.FileUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IWorkspaceRoot;
 import org.eclipse.core.resources.IncrementalProjectBuilder;
+import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.jobs.IJobChangeEvent;
 import org.eclipse.core.runtime.jobs.IJobChangeListener;
@@ -41,6 +43,7 @@
 import org.eclipse.jdt.internal.core.BinaryType;
 import org.eclipse.jdt.ls.core.internal.DependencyUtil;
 import org.eclipse.jdt.ls.core.internal.JobHelpers;
+import org.eclipse.jdt.ls.core.internal.ProjectUtils;
 import org.eclipse.jdt.ls.core.internal.SourceContentProvider;
 import org.eclipse.jdt.ls.core.internal.WorkspaceHelper;
 import org.eclipse.jdt.ls.core.internal.managers.ProjectsManager.CHANGE_TYPE;
@@ -174,6 +177,17 @@ public void testDownloadSources() throws Exception {
 		assertNotNull(source);
 	}
 
+	@Test
+	public void testBatchImport() throws Exception {
+		IProject project = importMavenProject("batch");
+		waitForBackgroundJobs();
+		assertTrue(ProjectUtils.isMavenProject(project));
+		IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+		assertEquals(root.getProjects().length, 14);
+		project = root.getProject("batchchild");
+		assertTrue(ProjectUtils.isMavenProject(project));
+	}
+
 	protected void testNonStandardCompilerId(String projectName) throws Exception {
 		IProject project = importMavenProject(projectName);
 		assertIsJavaProject(project);