* fix(ZNTA-1023,ZNTA-1248) use file types from server (#113)

* ZNTA-1248: handle comma-separated file extensions
* Implement push --list-file-types
* Make "file types" options similar in cli and Maven
* Add workaround for FileResource.fileTypeInfoList 404
* Add log message for fallback
* Fall back on defaults when pushing to old server

stub-server/src/main/java/org/zanata/rest/service/MockFileResource.java

@@ -23,12 +23,8 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.ws.rs.Path;
 import javax.ws.rs.WebApplicationException;
@@ -40,19 +36,19 @@
 import org.jboss.resteasy.annotations.providers.multipart.MultipartForm;
 import org.zanata.adapter.po.PoWriter2;
 import org.zanata.common.ContentState;
+import org.zanata.common.FileTypeInfo;
 import org.zanata.common.DocumentType;
 import org.zanata.common.LocaleId;
 import org.zanata.common.ProjectType;
 import org.zanata.rest.DocumentFileUploadForm;
 import org.zanata.rest.StringSet;
 import org.zanata.rest.dto.ChunkUploadResponse;
-import org.zanata.rest.dto.Project;
 import org.zanata.rest.dto.resource.Resource;
 import org.zanata.rest.dto.resource.TextFlow;
 import org.zanata.rest.dto.resource.TextFlowTarget;
 import org.zanata.rest.dto.resource.TranslationsResource;
 
-import static org.zanata.common.ProjectType.Podir;
+import static org.zanata.common.ProjectType.fileProjectSourceDocTypes;
 
 /**
  * @author Patrick Huang <a
@@ -77,9 +73,19 @@ public Response acceptedFileTypes() {
     }
 
     @Override
+    @Deprecated
     public Response acceptedFileTypeList() {
         GenericEntity<List<DocumentType>> genericEntity =
-            new GenericEntity<List<DocumentType>>(ProjectType.fileProjectSourceDocTypes()) {};
+            new GenericEntity<List<DocumentType>>(fileProjectSourceDocTypes()) {};
+        return Response.ok(genericEntity).build();
+    }
+
+    @Override
+    public Response fileTypeInfoList() {
+        List fileTypeInfoList = (List) fileProjectSourceDocTypes().stream().map(
+                DocumentType::toFileTypeInfo).collect(Collectors.toList());
+        GenericEntity<List<FileTypeInfo>> genericEntity =
+                new GenericEntity<List<FileTypeInfo>>(fileTypeInfoList) {};
         return Response.ok(genericEntity).build();
     }
 

zanata-client-commands/src/main/java/org/zanata/client/commands/FileMappingRuleHandler.java

@@ -37,7 +37,6 @@
 import org.zanata.client.config.FileMappingRule;
 import org.zanata.client.config.LocaleMapping;
 import org.zanata.client.util.FileUtil;
-import org.zanata.common.DocumentType;
 import org.zanata.common.ProjectType;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -81,7 +80,7 @@ public static boolean isRuleValid(String rule) {
      */
     public boolean isApplicable(DocNameWithExt docNameWithExt) {
         if (Strings.isNullOrEmpty(mappingRule.getPattern())) {
-            return matchFileExtensionWithProjectType(docNameWithExt);
+            return true;
         }
         PathMatcher matcher =
             FileSystems.getDefault().getPathMatcher("glob:" + mappingRule.getPattern());
@@ -95,18 +94,6 @@ public boolean isApplicable(DocNameWithExt docNameWithExt) {
         return matcher.matches(Paths.get(srcFile.getPath()));
     }
 
-    private boolean matchFileExtensionWithProjectType(
-            DocNameWithExt docNameWithExt) {
-        List<DocumentType> documentTypes = projectType.getSourceFileTypes();
-        for (DocumentType docType: documentTypes) {
-            if (docType.getSourceExtensions().contains(
-                    docNameWithExt.getExtension())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     /**
      * Apply the rule and return relative path of the translation file.
      *

zanata-client-commands/src/main/java/org/zanata/client/commands/StringUtil.java

@@ -1,6 +1,7 @@
 package org.zanata.client.commands;
 
 import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  *
@@ -9,6 +10,8 @@
  *
  */
 public class StringUtil {
+    private static final String newline = System.getProperty("line.separator");
+
     public static String removeFileExtension(String filename, String extension) {
         if (!filename.endsWith(extension))
             throw new IllegalArgumentException("Filename '" + filename
@@ -21,4 +24,13 @@ public static String removeFileExtension(String filename, String extension) {
     public static String indent(int numOfSpaces) {
         return Strings.repeat(" ", numOfSpaces);
     }
+
+    /**
+     * Converts an array of strings into a single string, delimited by newlines
+     * @param lines
+     * @return
+     */
+    public static String multiline(String... lines) {
+        return StringUtils.join(lines, newline);
+    }
 }

zanata-client-commands/src/main/java/org/zanata/client/commands/TransFileResolver.java

@@ -139,7 +139,8 @@ private Optional<File> tryGetTransFileFromProjectMappingRules(
             DocNameWithExt docNameWithExt, LocaleMapping localeMapping,
             Optional<String> translationFileExtension) {
         List<FileMappingRule> fileMappingRules = opts.getFileMappingRules();
-        // TODO may need to sort the rules. put rules without pattern to last
+        // Rules are applied in order. A rule with no pattern always matches,
+        // so such a rule should be the last.
         for (FileMappingRule rule : fileMappingRules) {
             FileMappingRuleHandler handler = new FileMappingRuleHandler(rule,
                     getProjectType(), opts);

zanata-client-commands/src/main/java/org/zanata/client/commands/init/SourceConfigPrompt.java

@@ -24,7 +24,6 @@
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import org.kohsuke.args4j.Option;
@@ -40,7 +39,7 @@
 import org.zanata.client.commands.push.PushOptionsImpl;
 import org.zanata.client.commands.push.RawPushCommand;
 import org.zanata.client.commands.push.RawPushStrategy;
-import org.zanata.common.DocumentType;
+import org.zanata.common.FileTypeInfo;
 import org.zanata.rest.client.FileResourceClient;
 import org.zanata.rest.client.RestClientFactory;
 
@@ -132,10 +131,11 @@ SourceConfigPrompt promptUser() throws Exception {
             console.blankLine();
             console.printfln(Question, get("project.file.type.question"));
 
-            // this answer is not persisted in zanata.xml so user will still need to type it when they do the actual push
+            // FIXME invoke listFileTypes
             console.printfln(Hint, PushOptionsImpl.fileTypeHelp);
             console.printf(Question, get("file.type.prompt"));
             String answer = console.expectAnyNotBlankAnswer();
+            // TODO this answer is not persisted in zanata.xml so user will still need to type it when they do the actual push
             ((PushOptionsImpl) pushOptions).setFileTypes(answer);
         }
 
@@ -255,23 +255,21 @@ private ImmutableList<String> filteredFileExtensions(PushOptions opts) {
                     new RawPushCommand(opts, clientFactory, console);
             FileResourceClient client =
                     clientFactory.getFileResourceClient();
-            List<DocumentType> rawDocumentTypes = client.acceptedFileTypes();
-            Map<DocumentType, Set<String>> filteredDocTypes =
-                    rawPushCommand.validateFileTypes(rawDocumentTypes,
+            List<FileTypeInfo> serverFileTypes = rawPushCommand.fileTypeInfoList(client);
+            ImmutableList<FileTypeInfo> filteredDocTypes =
+                    rawPushCommand.getActualFileTypes(serverFileTypes,
                             opts.getFileTypes());
 
             if (filteredDocTypes.isEmpty()) {
                 log.info("no valid types specified; nothing to do");
                 return ImmutableList.of();
             }
 
-            ImmutableList.Builder<String> sourceFileExtensionsBuilder =
+            ImmutableList.Builder<String> sourceFileExtensions =
                     ImmutableList.builder();
-            for (Set<String> filteredSourceExtensions : filteredDocTypes
-                    .values()) {
-                sourceFileExtensionsBuilder.addAll(filteredSourceExtensions);
-            }
-            return sourceFileExtensionsBuilder.build();
+            filteredDocTypes.forEach(type ->
+                    sourceFileExtensions.addAll(type.getSourceExtensions()));
+            return sourceFileExtensions.build();
         }
     }
 

zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java

@@ -111,6 +111,12 @@ public AbstractPushStrategy getStrategy(PushOptions pushOptions) {
         return strat;
     }
 
+    private void checkOptions() {
+        if (getOpts().getListFileTypes()) {
+            throw new RuntimeException("\"List file types\" is only supported for project type FILE");
+        }
+    }
+
     public static void logOptions(Logger logger, PushOptions opts) {
         if (!logger.isInfoEnabled()) {
             return;
@@ -208,6 +214,7 @@ private boolean pushTrans() {
 
     @Override
     public void run() throws Exception {
+        checkOptions();
         logOptions(log, getOpts());
         pushCurrentModule();
 

zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushOptions.java

@@ -52,6 +52,8 @@ public interface PushOptions extends PushPullOptions {
     // raw file push
     public int getChunkSize();
 
+    boolean getListFileTypes();
+
     public ImmutableList<String> getFileTypes();
 
     @Nonnull

zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushOptionsImpl.java

@@ -53,6 +53,7 @@ public class PushOptionsImpl extends AbstractPushPullOptionsImpl<PushOptions>
     private ImmutableList<String> includes = ImmutableList.of();
     private ImmutableList<String> excludes = ImmutableList.of();
     private ImmutableList<String> fileTypes = ImmutableList.of();
+    private boolean listFileTypes = false;
     private boolean defaultExcludes = DEF_EXCLUDES;
     private String mergeType = DEF_MERGE_TYPE;
     private boolean caseSensitive = DEF_CASE_SENSITIVE;
@@ -194,32 +195,36 @@ public ImmutableList<String> getFileTypes() {
         return fileTypes;
     }
 
-    public static final String fileTypeHelp = "File types to locate and transmit to the server. \n" +
-        "Default file extension will be used unless it is being specified. \n" +
-        "Pattern: TYPE[extension;extension],TYPE[extension] \n" +
-        "Supported types: \n" +
-        "\t XML_DOCUMENT_TYPE_DEFINITION[xml] \n" +
-        "\t PLAIN_TEXT[txt] \n" +
-        "\t IDML[idml] \n" +
-        "\t HTML[html;htm] \n" +
-        "\t OPEN_DOCUMENT_TEXT[odt] \n" +
-        "\t OPEN_DOCUMENT_PRESENTATION[odp] \n" +
-        "\t OPEN_DOCUMENT_GRAPHICS[odg] \n" +
-        "\t OPEN_DOCUMENT_SPREADSHEET[ods] \n" +
-        "\t SUBTITLE[srt;sbt;sub;vtt] \n" +
-        "\t TS[ts] \n" +
-        "\t GETTEXT[pot] \n" +
-        "\t PROPERTIES[properties] \n" +
-        "\t PROPERTIES_UTF8[properties] \n" +
-        "\t XLIFF[xml] \n" +
-        "Usage --file-types \"XML_DOCUMENT_TYPE_DEFINITION,PLAIN_TEXT[md;txt]\"";
+    public static final String fileTypeHelp = "File types to locate and transmit to the server\n" +
+            "when using project type \"file\".\n" +
+            "NOTE: No file types will be pushed unless listed here.\n" +
+            "Pattern: TYPE[extension;extension],TYPE[extension]\n" +
+            "The default file extension(s) for each TYPE will be used unless\n" +
+            "'extension' is specified in square brackets. If overriding extensions,\n" +
+            "please note that most shells require quotes around square brackets and \n" +
+            "semicolons unless they are escaped.\n" +
+            "Example: --file-types \"PROPERTIES,PLAIN_TEXT[md;txt]\"\n" +
+            "Use push with the option --list-file-types to see the server's\n" +
+            "supported types and their default extensions.";
 
     @Option(name = "--file-types", metaVar = "TYPES",
             usage = fileTypeHelp)
     public void setFileTypes(String fileTypes) {
         this.fileTypes = ImmutableList.copyOf(StringUtil.split(fileTypes, ","));
     }
 
+    @Override
+    public boolean getListFileTypes() {
+        return this.listFileTypes;
+    }
+
+    @Option(name = "--list-file-types",
+            usage = "List file types supported by the configured server, " +
+                    "instead of pushing files.")
+    public void setListFileTypes(boolean listFileTypes) {
+        this.listFileTypes = listFileTypes;
+    }
+
     @Override
     public boolean getCaseSensitive() {
         return caseSensitive;