Merge pull request #73 from nmiyake/removeConflictingContainersWithRe…

…factor

Add option to remove containers with conflicting names on startup

circle.yml

@@ -13,7 +13,7 @@ dependencies:
     - sudo pip install docker-compose
   override:
     - ./gradlew resolveConfigurations
-    
+
 test:
   pre:
     - ./gradlew findbugsMain findbugsTest checkstyleMain checkstyleTest javadoc --info

src/main/java/com/palantir/docker/compose/DockerComposeRule.java

@@ -17,11 +17,14 @@
 import com.palantir.docker.compose.connection.waiting.ClusterHealthCheck;
 import com.palantir.docker.compose.connection.waiting.ClusterWait;
 import com.palantir.docker.compose.connection.waiting.HealthCheck;
+import com.palantir.docker.compose.execution.ConflictingContainerRemovingDockerCompose;
 import com.palantir.docker.compose.execution.DefaultDockerCompose;
+import com.palantir.docker.compose.execution.Docker;
 import com.palantir.docker.compose.execution.DockerCompose;
 import com.palantir.docker.compose.execution.DockerComposeExecArgument;
 import com.palantir.docker.compose.execution.DockerComposeExecOption;
 import com.palantir.docker.compose.execution.DockerComposeExecutable;
+import com.palantir.docker.compose.execution.DockerExecutable;
 import com.palantir.docker.compose.execution.RetryingDockerCompose;
 import com.palantir.docker.compose.logging.DoNothingLogCollector;
 import com.palantir.docker.compose.logging.FileLogCollector;
@@ -61,17 +64,29 @@ public ProjectName projectName() {
     }
 
     @Value.Default
-    public DockerComposeExecutable executable() {
+    public DockerComposeExecutable dockerComposeExecutable() {
         return DockerComposeExecutable.builder()
             .dockerComposeFiles(files())
             .dockerConfiguration(machine())
             .projectName(projectName())
             .build();
     }
 
+    @Value.Default
+    public DockerExecutable dockerExecutable() {
+        return DockerExecutable.builder()
+                .dockerConfiguration(machine())
+                .build();
+    }
+
+    @Value.Default
+    public Docker docker() {
+        return new Docker(dockerExecutable());
+    }
+
     @Value.Default
     public DockerCompose dockerCompose() {
-        DockerCompose dockerCompose = new DefaultDockerCompose(executable(), machine());
+        DockerCompose dockerCompose = new DefaultDockerCompose(dockerComposeExecutable(), machine());
         return new RetryingDockerCompose(retryAttempts(), dockerCompose);
     }
 
@@ -93,6 +108,11 @@ protected boolean skipShutdown() {
         return false;
     }
 
+    @Value.Default
+    protected boolean removeConflictingContainersOnStartup() {
+        return true;
+    }
+
     @Value.Default
     protected LogCollector logCollector() {
         return new DoNothingLogCollector();
@@ -102,7 +122,12 @@ protected LogCollector logCollector() {
     public void before() throws IOException, InterruptedException {
         log.debug("Starting docker-compose cluster");
         dockerCompose().build();
-        dockerCompose().up();
+
+        DockerCompose upDockerCompose = dockerCompose();
+        if (removeConflictingContainersOnStartup()) {
+            upDockerCompose = new ConflictingContainerRemovingDockerCompose(upDockerCompose, docker());
+        }
+        upDockerCompose.up();
 
         log.debug("Starting log collection");
 
@@ -187,4 +212,5 @@ public ImmutableDockerComposeRule.Builder waitingForHostNetworkedPort(int port,
             return addClusterWait(new ClusterWait(clusterHealthCheck, timeout));
         }
     }
+
 }

src/main/java/com/palantir/docker/compose/DockerCompositionBuilder.java

@@ -88,6 +88,12 @@ public DockerCompositionBuilder saveLogsTo(String path) {
         return this;
     }
 
+
+    public DockerCompositionBuilder removeConflictingContainersOnStartup(boolean removeConflictingContainersOnStartup) {
+        builder.removeConflictingContainersOnStartup(removeConflictingContainersOnStartup);
+        return this;
+    }
+
     public DockerCompositionBuilder retryAttempts(int retryAttempts) {
         builder.retryAttempts(retryAttempts);
         return this;

src/main/java/com/palantir/docker/compose/execution/SynchronousDockerComposeExecutable.java → src/main/java/com/palantir/docker/compose/execution/Command.java

@@ -24,26 +24,49 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Consumer;
 
-public class SynchronousDockerComposeExecutable {
+public class Command {
     public static final int HOURS_TO_WAIT_FOR_STD_OUT_TO_CLOSE = 12;
     public static final int MINUTES_TO_WAIT_AFTER_STD_OUT_CLOSES = 1;
-    private final DockerComposeExecutable dockerComposeExecutable;
+    private final Executable executable;
     private final Consumer<String> logConsumer;
 
-    public SynchronousDockerComposeExecutable(DockerComposeExecutable dockerComposeExecutable,
-            Consumer<String> logConsumer) {
-        this.dockerComposeExecutable = dockerComposeExecutable;
+    public Command(Executable executable, Consumer<String> logConsumer) {
+        this.executable = executable;
         this.logConsumer = logConsumer;
     }
 
-    public ProcessResult run(String... commands) throws IOException, InterruptedException {
-        Process process = dockerComposeExecutable.execute(commands);
+    public String execute(ErrorHandler errorHandler, String... commands) throws IOException, InterruptedException {
+        ProcessResult result = run(commands);
+
+        if (result.exitCode() != 0) {
+            errorHandler.handle(result.exitCode(), result.output(), executable.commandName(), commands);
+        }
+
+        return result.output();
+    }
+
+    public static ErrorHandler throwingOnError() {
+        return (exitCode, output, commandName, commands) -> {
+            String message =
+                    constructNonZeroExitErrorMessage(exitCode, commandName, commands) + "\nThe output was:\n" + output;
+            throw new DockerExecutionException(message);
+        };
+    }
+
+    private static String constructNonZeroExitErrorMessage(int exitCode, String commandName, String... commands) {
+        return "'" + commandName + " " + Arrays.stream(commands).collect(joining(" ")) + "' returned exit code "
+                + exitCode;
+    }
+
+    private ProcessResult run(String... commands) throws IOException, InterruptedException {
+        Process process = executable.execute(commands);
 
         Future<String> outputProcessing = newSingleThreadExecutor()
                 .submit(() -> processOutputFrom(process));
@@ -57,8 +80,8 @@ public ProcessResult run(String... commands) throws IOException, InterruptedExce
 
     private String processOutputFrom(Process process) {
         return asReader(process.getInputStream()).lines()
-                    .peek(logConsumer)
-                    .collect(joining(System.lineSeparator()));
+                .peek(logConsumer)
+                .collect(joining(System.lineSeparator()));
     }
 
     private String waitForResultFrom(Future<String> outputProcessing) {

src/main/java/com/palantir/docker/compose/execution/ConflictingContainerRemovingDockerCompose.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2016 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.palantir.docker.compose.execution;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ConflictingContainerRemovingDockerCompose extends DelegatingDockerCompose {
+    private static final Logger log = LoggerFactory.getLogger(ConflictingContainerRemovingDockerCompose.class);
+    private static final Pattern NAME_CONFLICT_PATTERN = Pattern.compile("The name \"([^\"]*)\" is already in use");
+
+    private final Docker docker;
+    private final int retryAttempts;
+
+    public ConflictingContainerRemovingDockerCompose(DockerCompose dockerCompose, Docker docker) {
+        this(dockerCompose, docker, 1);
+    }
+
+    public ConflictingContainerRemovingDockerCompose(DockerCompose dockerCompose, Docker docker, int retryAttempts) {
+        super(dockerCompose);
+
+        Preconditions.checkArgument(retryAttempts >= 1, "retryAttempts must be at least 1, was " + retryAttempts);
+        this.docker = docker;
+        this.retryAttempts = retryAttempts;
+    }
+
+    @Override
+    public void up() throws IOException, InterruptedException {
+        for (int currRetryAttempt = 0; currRetryAttempt <= retryAttempts; currRetryAttempt++) {
+            try {
+                getDockerCompose().up();
+                return;
+            } catch (DockerExecutionException e) {
+                Set<String> conflictingContainerNames = getConflictingContainerNames(e.getMessage());
+                if (conflictingContainerNames.isEmpty()) {
+                    // failed due to reason other than conflicting containers, so re-throw
+                    throw e;
+                }
+
+                log.debug("docker-compose up failed due to container name conflicts (container names: {}). "
+                                + "Removing containers and attempting docker-compose up again (attempt {}).",
+                        conflictingContainerNames, currRetryAttempt + 1);
+                removeContainers(conflictingContainerNames);
+            }
+        }
+
+        throw new DockerExecutionException("docker-compose up failed");
+    }
+
+    private void removeContainers(Collection<String> containerNames) throws IOException, InterruptedException {
+        try {
+            docker.rm(containerNames);
+        } catch (DockerExecutionException e) {
+            // there are cases such as in CircleCI where 'docker rm' returns a non-0 exit code and "fails",
+            // but container is still effectively removed as far as conflict resolution is concerned. Because
+            // of this, be permissive and do not fail task even if 'rm' fails.
+            log.debug("docker rm failed, but continuing execution", e);
+        }
+    }
+
+    private static Set<String> getConflictingContainerNames(String output) {
+        ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+        Matcher matcher = NAME_CONFLICT_PATTERN.matcher(output);
+        while (matcher.find()) {
+            builder.add(matcher.group(1));
+        }
+        return builder.build();
+    }
+
+}

src/main/java/com/palantir/docker/compose/execution/DefaultDockerCompose.java

@@ -16,7 +16,6 @@
 package com.palantir.docker.compose.execution;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.stream.Collectors.joining;
 import static org.apache.commons.lang3.Validate.validState;
 import static org.joda.time.Duration.standardMinutes;
 
@@ -31,7 +30,6 @@
 import com.palantir.docker.compose.connection.Ports;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.Arrays;
 import org.apache.commons.io.IOUtils;
 import org.joda.time.Duration;
 import org.slf4j.Logger;
@@ -43,7 +41,7 @@ public class DefaultDockerCompose implements DockerCompose {
     private static final Duration COMMAND_TIMEOUT = standardMinutes(2);
     private static final Logger log = LoggerFactory.getLogger(DefaultDockerCompose.class);
 
-    private final SynchronousDockerComposeExecutable executable;
+    private final Command command;
     private final DockerMachine dockerMachine;
     private final DockerComposeExecutable rawExecutable;
 
@@ -57,49 +55,49 @@ public DefaultDockerCompose(DockerComposeFiles dockerComposeFiles, DockerMachine
 
     public DefaultDockerCompose(DockerComposeExecutable rawExecutable, DockerMachine dockerMachine) {
         this.rawExecutable = rawExecutable;
-        this.executable = new SynchronousDockerComposeExecutable(rawExecutable, log::debug);
+        this.command = new Command(rawExecutable, log::debug);
         this.dockerMachine = dockerMachine;
     }
 
     @Override
     public void build() throws IOException, InterruptedException {
-        executeDockerComposeCommand(throwingOnError(), "build");
+        command.execute(Command.throwingOnError(), "build");
     }
 
     @Override
     public void up() throws IOException, InterruptedException {
-        executeDockerComposeCommand(throwingOnError(), "up", "-d");
+        command.execute(Command.throwingOnError(), "up", "-d");
     }
 
     @Override
     public void down() throws IOException, InterruptedException {
-        executeDockerComposeCommand(swallowingDownCommandDoesNotExist(), "down", "--volumes");
+        command.execute(swallowingDownCommandDoesNotExist(), "down", "--volumes");
     }
 
     @Override
     public void kill() throws IOException, InterruptedException {
-        executeDockerComposeCommand(throwingOnError(), "kill");
+        command.execute(Command.throwingOnError(), "kill");
     }
 
     @Override
     public void rm() throws IOException, InterruptedException {
-        executeDockerComposeCommand(throwingOnError(), "rm", "--force", "-v");
+        command.execute(Command.throwingOnError(), "rm", "--force", "-v");
     }
 
     @Override
     public String exec(DockerComposeExecOption dockerComposeExecOption, String containerName,
             DockerComposeExecArgument dockerComposeExecArgument) throws IOException, InterruptedException {
         verifyDockerComposeVersionAtLeast(VERSION_1_7_0, "You need at least docker-compose 1.7 to run docker-compose exec");
         String[] fullArgs = constructFullDockerComposeExecArguments(dockerComposeExecOption, containerName, dockerComposeExecArgument);
-        return executeDockerComposeCommand(throwingOnError(), fullArgs);
+        return command.execute(Command.throwingOnError(), fullArgs);
     }
 
     private void verifyDockerComposeVersionAtLeast(Version targetVersion, String message) throws IOException, InterruptedException {
         validState(version().greaterThanOrEqualTo(targetVersion), message);
     }
 
     private Version version() throws IOException, InterruptedException {
-        String versionOutput = executeDockerComposeCommand(throwingOnError(), "-v");
+        String versionOutput = command.execute(Command.throwingOnError(), "-v");
         return DockerComposeVersion.parseFromDockerComposeVersion(versionOutput);
     }
 
@@ -115,7 +113,7 @@ private String[] constructFullDockerComposeExecArguments(DockerComposeExecOption
 
     @Override
     public ContainerNames ps() throws IOException, InterruptedException {
-        String psOutput = executeDockerComposeCommand(throwingOnError(), "ps");
+        String psOutput = command.execute(Command.throwingOnError(), "ps");
         return ContainerNames.parseFromDockerComposePs(psOutput);
     }
 
@@ -150,38 +148,15 @@ private Process followLogs(String container) throws IOException, InterruptedExce
 
     @Override
     public Ports ports(String service) throws IOException, InterruptedException {
-        String psOutput = executeDockerComposeCommand(throwingOnError(), "ps", service);
+        String psOutput = command.execute(Command.throwingOnError(), "ps", service);
         validState(!Strings.isNullOrEmpty(psOutput), "No container with name '" + service + "' found");
         return Ports.parseFromDockerComposePs(psOutput, dockerMachine.getIp());
     }
 
-    private ErrorHandler throwingOnError() {
-        return (exitCode, output, commands) -> {
-            String message = constructNonZeroExitErrorMessage(exitCode, commands) + "\nThe output was:\n" + output;
-            throw new DockerComposeExecutionException(message);
-        };
-    }
-
-    private String executeDockerComposeCommand(ErrorHandler errorHandler, String... commands)
-            throws IOException, InterruptedException {
-        ProcessResult result = executable.run(commands);
-
-        if (result.exitCode() != 0) {
-            errorHandler.handle(result.exitCode(), result.output(), commands);
-        }
-
-        return result.output();
-    }
-
-
-    private String constructNonZeroExitErrorMessage(int exitCode, String... commands) {
-        return "'docker-compose " + Arrays.stream(commands).collect(joining(" ")) + "' returned exit code " + exitCode;
-    }
-
     private ErrorHandler swallowingDownCommandDoesNotExist() {
-        return (exitCode, output, commands) -> {
+        return (exitCode, output, commandName, commands) -> {
             if (downCommandWasPresent(output)) {
-                throwingOnError().handle(exitCode, output, commands);
+                Command.throwingOnError().handle(exitCode, output, commandName, commands);
             }
 
             log.warn("It looks like `docker-compose down` didn't work.");