Add retry loop to removing bootstrap etcd service. (#526)

Since the etcd pivot just occurred the apiserver may be temporarily
unavailable. This retries until successful or timeout, but still doesn't
fail the entire process on failure.

pkg/util/etcdutil/migrate.go

@@ -209,7 +209,17 @@ func waitBootEtcdRemoved(etcdServiceIP string) error {
 }
 
 func cleanupBootstrapEtcdService(kubecli kubernetes.Interface) {
-	if err := kubecli.CoreV1().Services(api.NamespaceSystem).Delete(bootstrapEtcdServiceName, &v1.DeleteOptions{}); err != nil {
-		glog.Errorf("failed to remove bootstrap-etcd-service: %v", err)
+	if err := wait.Poll(pollInterval, pollTimeout, func() (bool, error) {
+		if err := kubecli.CoreV1().Services(api.NamespaceSystem).Delete(bootstrapEtcdServiceName, &v1.DeleteOptions{}); err != nil {
+			if apierrors.IsNotFound(err) {
+				glog.Info("bootstrap-etcd-service already removed")
+				return true, nil
+			}
+			glog.Errorf("failed to remove bootstrap-etcd-service: %v", err)
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		glog.Errorf("timed out removing bootstrap-etcd-service: %v", err)
 	}
 }