Add support for HTML files in the webapp

Basic support works but
1) the text unit name will change as soon as there is a change to the document which will prevent basic leveraging. It is just using an auto increment for new text unit. We need to look into some different logic for that, eg. md5 hashing of previous content + occurence count
2) if there are image, and maybe some other html element, they just get replaced by a placeholder in the localized file. Need to debug.

cli/pom.xml

@@ -144,7 +144,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
-                    <argLine>-Dspring.config.additional-location=optional:file://${user.home}/.l10n/config/cli/
+                    <!-- TODO revert before final commit -->
+                    <argLine>-Dspring.config.additional-location=optional:file://${user.home}/.l10n/config/webapp/
                         -Dspring.profiles.active=${user.name},test -Xmx1024m
                     </argLine>
                 </configuration>

cli/src/main/java/com/box/l10n/mojito/cli/filefinder/FileFinder.java

@@ -63,6 +63,7 @@ public FileFinder() {
     fileTypes.add(new JSFileType());
     fileTypes.add(new TSFileType());
     fileTypes.add(new YamlFileType());
+    fileTypes.add(new HtmlFileType());
     // TODO(ja) disable for now because this is likely to pickup a lot of files. it might be better
     // to ask for it explicitly
     // fileTypes.add(new JSONFileType());

cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/FileTypes.java

@@ -26,7 +26,8 @@ public enum FileTypes {
   CHROME_EXT_JSON(ChromeExtensionJSONFileType.class),
   I18NEXT_PARSER_JSON(I18NextFileType.class),
   TS(TSFileType.class),
-  YAML(YamlFileType.class);
+  YAML(YamlFileType.class),
+  HTML(HtmlFileType.class);
 
   Class<? extends FileType> clazz;
 

cli/src/main/java/com/box/l10n/mojito/cli/filefinder/file/HtmlFileType.java

@@ -0,0 +1,9 @@
+package com.box.l10n.mojito.cli.filefinder.file;
+
+/** @author jaurambault */
+public class HtmlFileType extends LocaleInNameFileType {
+
+  public HtmlFileType() {
+    this.sourceFileExtension = "html";
+  }
+}

cli/src/test/java/com/box/l10n/mojito/cli/command/PullCommandTest.java

@@ -954,6 +954,51 @@ public void pullPo() throws Exception {
     checkExpectedGeneratedResources();
   }
 
+  @Test
+  public void pullHtml() throws Exception {
+
+    Repository repository = createTestRepoUsingRepoService();
+    System.setProperty("overrideExpectedTestFiles", "true");
+
+    getL10nJCommander()
+        .run(
+            "push",
+            "-r",
+            repository.getName(),
+            "-s",
+            getInputResourcesTestDir("source").getAbsolutePath());
+
+    Asset asset = assetClient.getAssetByPathAndRepositoryId("demo.html", repository.getId());
+    importTranslations(asset.getId(), "source-xliff_", "fr-FR");
+    importTranslations(asset.getId(), "source-xliff_", "ja-JP");
+
+    getL10nJCommander()
+        .run(
+            "pull",
+            "-r",
+            repository.getName(),
+            "-s",
+            getInputResourcesTestDir("source").getAbsolutePath(),
+            "-t",
+            getTargetTestDir("target").getAbsolutePath(),
+            "-lm",
+            "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP");
+
+    getL10nJCommander()
+        .run(
+            "pull",
+            "-r",
+            repository.getName(),
+            "-s",
+            getInputResourcesTestDir("source_modified").getAbsolutePath(),
+            "-t",
+            getTargetTestDir("target_modified").getAbsolutePath(),
+            "-lm",
+            "fr:fr-FR,fr-CA:fr-CA,ja:ja-JP");
+
+    checkExpectedGeneratedResources();
+  }
+
   @Test
   public void removeUntranslated() throws Exception {
 

cli/src/test/java/com/box/l10n/mojito/cli/filefinder/FileFinderTest.java

@@ -152,6 +152,46 @@ public void findXliff() throws IOException, FileFinderException {
     assertFalse(itTargets.hasNext());
   }
 
+  @Test
+  public void findHtml() throws IOException, FileFinderException {
+    FileFinder fileFinder = initFileFinder(false);
+
+    ArrayList<FileMatch> sources = fileFinder.getSources();
+    Collections.sort(sources);
+
+    ArrayList<FileMatch> targets = fileFinder.getTargets();
+    Collections.sort(targets);
+
+    Iterator<FileMatch> itSources = sources.iterator();
+
+    FileMatch next = itSources.next();
+    assertEquals(HtmlFileType.class, next.fileType.getClass());
+    assertEquals(
+        getInputResourcesTestDir("source").toString() + "/filefinder.html",
+        next.getPath().toString());
+    assertEquals("filefinder_fr-FR.html", next.getTargetPath("fr-FR"));
+
+    next = itSources.next();
+    assertEquals(
+        getInputResourcesTestDir("source").toString() + "/sub/filefinder2.html",
+        next.getPath().toString());
+    assertEquals("sub/filefinder2_fr-FR.html", next.getTargetPath("fr-FR"));
+
+    assertFalse(itSources.hasNext());
+
+    Iterator<FileMatch> itTargets = fileFinder.getTargets().iterator();
+    assertEquals(
+        getInputResourcesTestDir("target").toString() + "/filefinder_fr-FR.html",
+        itTargets.next().getPath().toString());
+    assertEquals(
+        getInputResourcesTestDir("target").toString() + "/filefinder_fr.html",
+        itTargets.next().getPath().toString());
+    assertEquals(
+        getInputResourcesTestDir("target").toString() + "/sub/filefinder2_fr.html",
+        itTargets.next().getPath().toString());
+    assertFalse(itTargets.hasNext());
+  }
+
   @Test
   public void findXcodeXliff() throws IOException, FileFinderException {
     FileFinder fileFinder = initFileFinder(false, new XcodeXliffFileType());

cli/src/test/java/com/box/l10n/mojito/cli/filefinder/file/HtmlFileTypeTest.java

@@ -0,0 +1,50 @@
+package com.box.l10n.mojito.cli.filefinder.file;
+
+import static com.box.l10n.mojito.cli.filefinder.FilePattern.BASE_NAME;
+import static com.box.l10n.mojito.cli.filefinder.FilePattern.FILE_EXTENSION;
+import static com.box.l10n.mojito.cli.filefinder.FilePattern.LOCALE;
+import static com.box.l10n.mojito.cli.filefinder.FilePattern.PARENT_PATH;
+
+import com.box.l10n.mojito.cli.filefinder.FilePattern;
+import java.util.regex.Matcher;
+import org.junit.Assert;
+import org.junit.Test;
+
+/** @author jaurambault */
+public class HtmlFileTypeTest {
+
+  @Test
+  public void testSourcePattern() {
+    HtmlFileType htmlFileType = new HtmlFileType();
+    FilePattern sourceFilePattern = htmlFileType.getSourceFilePattern();
+    Matcher matcher = sourceFilePattern.getPattern().matcher("/source/filefinder.html");
+    Assert.assertTrue(matcher.matches());
+    Assert.assertEquals("/source/", matcher.group(PARENT_PATH));
+    Assert.assertEquals("filefinder", matcher.group(BASE_NAME));
+    Assert.assertEquals("html", matcher.group(FILE_EXTENSION));
+  }
+
+  @Test
+  public void testTargetPattern() {
+    HtmlFileType htmlFileType = new HtmlFileType();
+    Matcher matcher =
+        htmlFileType.getTargetFilePattern().getPattern().matcher("/source/filefinder_fr.html");
+    Assert.assertTrue(matcher.matches());
+    Assert.assertEquals("fr", matcher.group(LOCALE));
+    Assert.assertEquals("/source/", matcher.group(PARENT_PATH));
+    Assert.assertEquals("filefinder", matcher.group(BASE_NAME));
+    Assert.assertEquals("html", matcher.group(FILE_EXTENSION));
+  }
+
+  @Test
+  public void testTargetPatternBcp47() {
+    HtmlFileType htmlFileType = new HtmlFileType();
+    Matcher matcher =
+        htmlFileType.getTargetFilePattern().getPattern().matcher("/source/filefinder_fr-FR.html");
+    Assert.assertTrue(matcher.matches());
+    Assert.assertEquals("fr-FR", matcher.group(LOCALE));
+    Assert.assertEquals("/source/", matcher.group(PARENT_PATH));
+    Assert.assertEquals("filefinder", matcher.group(BASE_NAME));
+    Assert.assertEquals("html", matcher.group(FILE_EXTENSION));
+  }
+}

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/expected/target/demo_fr-CA.html

@@ -0,0 +1,8 @@
+<html>
+  <p>100 character description:</p>
+  <ul>
+    <li>15 min</li>
+    <li>1 day</li>
+    <li>1 hour</li>
+    <li>1 month</li>
+  </ul>[#$dp12]</html>

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/expected/target/demo_fr.html

@@ -0,0 +1,8 @@
+<html>
+  <p>100 character description:</p>
+  <ul>
+    <li>15 min</li>
+    <li>1 day</li>
+    <li>1 hour</li>
+    <li>1 month</li>
+  </ul>[#$dp12]</html>

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/expected/target/demo_ja.html

@@ -0,0 +1,8 @@
+<html>
+  <p>100 character description:</p>
+  <ul>
+    <li>15 min</li>
+    <li>1 day</li>
+    <li>1 hour</li>
+    <li>1 month</li>
+  </ul>[#$dp12]</html>

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/input/source/demo.html

@@ -0,0 +1,10 @@
+<html>
+  <p>100 character description:</p>
+  <ul>
+    <li>15 min</li>
+    <li>1 day</li>
+    <li>1 hour</li>
+    <li>1 month</li>
+  </ul>
+  <img src="this-is-causing-issues.jpg">
+</html>

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/input/translations/source-xliff_fr-FR.xliff

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" version="1.2">
+    <file original=""  source-language="en" target-language="fr-FR" datatype="x-undefined">
+        <body>
+<!--          TODO update when it is decided how to process the text unit name and if we support comments -->
+<!--             <trans-unit id="" resname="100 character description:" datatype="php"> -->
+<!--                 <source>100 character description:</source> -->
+<!--                 <target>Description de 100 caractères :</target> -->
+<!--             </trans-unit> -->
+<!--             <trans-unit id="" resname="15 min" datatype="x-javascript+php"> -->
+<!--                 <source>15 min</source> -->
+<!--                 <target>15 min</target> -->
+<!--             </trans-unit> -->
+<!--             <trans-unit id="" resname="1 day" datatype="x-javascript+php"> -->
+<!--                 <source>1 day</source> -->
+<!--                 <target>1 jour</target> -->
+<!--             </trans-unit> -->
+<!--             <trans-unit id="" resname="1 hour" datatype="x-javascript+php"> -->
+<!--                 <source>1 hour</source> -->
+<!--                 <target>1 heure</target> -->
+<!--             </trans-unit> -->
+<!--             <trans-unit id="" resname="1 month" datatype="x-javascript+php"> -->
+<!--                 <source>1 month</source> -->
+<!--                 <target>1 mois</target> -->
+<!--             </trans-unit> -->
+            <trans-unit id="" resname="tu1" datatype="php">
+                <source>100 character description:</source>
+                <target>Description de 100 caractères :</target>
+            </trans-unit>
+            <trans-unit id="" resname="tu2" datatype="x-javascript+php">
+                <source>15 min</source>
+                <target>15 min</target>
+            </trans-unit>
+            <trans-unit id="" resname="tu3" datatype="x-javascript+php">
+                <source>1 day</source>
+                <target>1 jour</target>
+            </trans-unit>
+            <trans-unit id="" resname="tu4" datatype="x-javascript+php">
+                <source>1 hour</source>
+                <target>1 heure</target>
+            </trans-unit>
+            <trans-unit id="" resname="tu5" datatype="x-javascript+php">
+                <source>1 month</source>
+                <target>1 mois</target>¬
+            </trans-unit>
+        </body>
+    </file>
+</xliff>

cli/src/test/resources/com/box/l10n/mojito/cli/command/PullCommandTest_IO/pullHtml/input/translations/source-xliff_ja-JP.xliff

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:okp="okapi-framework:xliff-extensions" version="1.2">
+    <file original=""  source-language="en" target-language="ja-JP" datatype="x-undefined">
+        <body>
+            <trans-unit id="" resname="100 character description:" datatype="php">
+                <source>100 character description:</source>
+                <target>100文字の説明:</target>
+            </trans-unit>
+            <trans-unit id="" resname="15 min" datatype="x-javascript+php">
+                <source>15 min</source>
+                <target>15分</target>
+            </trans-unit>
+            <trans-unit id="" resname="1 day" datatype="x-javascript+php">
+                <source>1 day</source>
+                <target>1日</target>
+            </trans-unit>
+            <trans-unit id="" resname="1 hour" datatype="x-javascript+php">
+                <source>1 hour</source>
+                <target>1時間</target>
+            </trans-unit>
+            <trans-unit id="" resname="1 month" datatype="x-javascript+php">
+                <source>1 month</source>
+                <target>1か月</target>
+            </trans-unit>
+        </body>
+    </file>
+</xliff>

cli/src/test/resources/com/box/l10n/mojito/cli/filefinder/FileFinderTest_IO/findHtml/input/source/filefinder.html

@@ -0,0 +1 @@
+HELLO=Hello {0}!

cli/src/test/resources/com/box/l10n/mojito/cli/filefinder/FileFinderTest_IO/findHtml/input/source/sub/filefinder2.html

@@ -0,0 +1 @@
+HELLO=Hello {0}!

cli/src/test/resources/com/box/l10n/mojito/cli/filefinder/FileFinderTest_IO/findHtml/input/target/filefinder_fr-FR.html

@@ -0,0 +1 @@
+HELLO=Bonjour {0}!

cli/src/test/resources/com/box/l10n/mojito/cli/filefinder/FileFinderTest_IO/findHtml/input/target/filefinder_fr.html

@@ -0,0 +1 @@
+HELLO=Bonjour {0}!

cli/src/test/resources/com/box/l10n/mojito/cli/filefinder/FileFinderTest_IO/findHtml/input/target/sub/filefinder2_fr.html

@@ -0,0 +1 @@
+HELLO=Bonjour {0}!

common/pom.xml

@@ -266,6 +266,29 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>net.sf.okapi.filters</groupId>
+            <artifactId>okapi-filter-html</artifactId>
+            <version>${okapi.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>net.sf.okapi.logbind</groupId>
+                    <artifactId>build-log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-simple</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>net.sf.okapi.steps</groupId>
             <artifactId>okapi-step-common</artifactId>

common/src/main/java/com/box/l10n/mojito/okapi/asset/AssetPathToFilterConfigMapper.java

@@ -33,6 +33,8 @@ public class AssetPathToFilterConfigMapper {
   public static final String XTB_FILTER_CONFIG_ID = XMLFilter.FILTER_CONFIG_ID + "-xtb";
   public static final String JS_FILTER_CONFIG_ID = JSFilter.FILTER_CONFIG_ID + "-js";
 
+  public static final String HTML_FILTER_CONFIG_ID = "okf_html";
+
   private enum AssetFilterType {
     CSV(CSVFilter.FILTER_CONFIG_ID, "csv"),
     XLIFF(XLIFF_FILTER_CONFIG_ID, "xlf", "xliff", "sdlxliff", "mxliff"),
@@ -45,7 +47,8 @@ private enum AssetFilterType {
     XTB(XTB_FILTER_CONFIG_ID, "xtb"),
     JS(JS_FILTER_CONFIG_ID, "ts", "js"),
     JSON(JSONFilter.FILTER_CONFIG_ID, "json"),
-    YAML(YamlFilter.FILTER_CONFIG_ID, "yaml");
+    YAML(YamlFilter.FILTER_CONFIG_ID, "yaml"),
+    HTML(HTML_FILTER_CONFIG_ID, "html");
 
     private String configId;
     private Set<String> supportedExtensions = new HashSet<>();

common/src/main/java/com/box/l10n/mojito/okapi/asset/FilterConfigurationMappers.java

@@ -3,6 +3,7 @@
 import com.box.l10n.mojito.okapi.filters.*;
 import net.sf.okapi.common.filters.DefaultFilters;
 import net.sf.okapi.common.filters.IFilterConfigurationMapper;
+import net.sf.okapi.filters.html.HtmlFilter;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Lazy;
@@ -41,6 +42,7 @@ public IFilterConfigurationMapper getConfiguredFilterConfigurationMapper() {
     mapper.addConfigurations(JSONFilter.class.getName());
     mapper.addConfigurations(XcodeXliffFilter.class.getName());
     mapper.addConfigurations(YamlFilter.class.getName());
+    mapper.addConfigurations(HtmlFilter.class.getName());
 
     return mapper;
   }