Merge pull request #170 from aglitchman/dev

Add a parallel compilation for C/C++ files
defold · Aug 28, 2020 · ec04ae5 · ec04ae5
2 parents a5c141c + 2c7b7c5
commit ec04ae5
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 4 deletions.
diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java
@@ -17,6 +17,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.*;
+import java.util.concurrent.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -442,7 +443,7 @@ else if(f.getName().endsWith(".aar")) {
         return allLibJars;
     }
 
-    private File compileFile(int index, File extDir, File src, Map<String, Object> manifestContext) throws IOException, InterruptedException, ExtenderException {
+    private File compileFile(int index, File extDir, File src, Map<String, Object> manifestContext, List<String> commands) throws IOException, InterruptedException, ExtenderException {
         List<String> includes = new ArrayList<>();
         includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(extDir, "include") ) );
         includes.add( ExtenderUtil.getRelativePath(jobDirectory, uploadDirectory) );
@@ -457,7 +458,7 @@ private File compileFile(int index, File extDir, File src, Map<String, Object> m
         context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths));
 
         String command = templateExecutor.execute(platformConfig.compileCmd, context);
-        processExecutor.execute(command);
+        commands.add(command);
         return o;
     }
 
@@ -504,15 +505,44 @@ private void buildExtension(File manifest, Map<String, Object> manifestContext)
         Arrays.sort(srcFiles, NameFileComparator.NAME_INSENSITIVE_COMPARATOR);
 
         List<String> objs = new ArrayList<>();
+        List<String> commands = new ArrayList<>();
 
         // Compile C++ source into object files
         int i = getAndIncreaseNameCount();
         for (File src : srcFiles) {
-            File o = compileFile(i, extDir, src, manifestContext);
+            File o = compileFile(i, extDir, src, manifestContext, commands);
             objs.add(ExtenderUtil.getRelativePath(jobDirectory, o));
             i++;
         }
 
+        ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+        List<Callable<Void>> callables = new ArrayList<>();
+        for (String command : commands) {
+            callables.add(new Callable<Void>() {
+                @Override
+                public Void call() throws Exception {
+                    processExecutor.execute(command);
+                    return null;
+                }
+            });
+        }
+        List<Future<Void>> futures = executor.invokeAll(callables);
+        try {
+            for (Future<Void> future : futures) {
+                future.get(); 
+            }
+        } catch (ExecutionException e) {
+            if (e.getCause() instanceof IOException) {
+                throw (IOException)e.getCause();
+            } else if (e.getCause() instanceof InterruptedException) {
+                throw (InterruptedException)e.getCause();
+            } else {
+                throw new ExtenderException(e.getCause().toString());
+            }
+        } finally {
+            executor.shutdown();
+        }
+
         // Create c++ library
         File lib = null;
         if (platformConfig.writeLibPattern != null) {

diff --git a/server/src/main/java/com/defold/extender/ProcessExecutor.java b/server/src/main/java/com/defold/extender/ProcessExecutor.java
@@ -8,7 +8,7 @@
 import java.util.stream.Collectors;
 
 public class ProcessExecutor {
-    private final StringBuilder output = new StringBuilder();
+    private final StringBuffer output = new StringBuffer();
     private final HashMap<String, String> env = new HashMap<>();
     private File cwd = null;
     private boolean DM_DEBUG_COMMANDS = System.getenv("DM_DEBUG_COMMANDS") != null;