Added file checksum handling to ThirdPartyTMSSmartlingWithJSON

webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyFileChecksum.java

@@ -6,12 +6,9 @@
 import javax.persistence.Entity;
 import javax.persistence.Index;
 import javax.persistence.Table;
-import org.hibernate.envers.Audited;
-import org.hibernate.envers.RelationTargetAuditMode;
 
 /** Entity that stores the checksum of a translated file downloaded via a third party sync. */
 @Entity
-@Audited(targetAuditMode = RelationTargetAuditMode.NOT_AUDITED)
 @Table(
     name = "third_party_sync_file_checksum",
     indexes = {

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java

@@ -1,6 +1,7 @@
 package com.box.l10n.mojito.service.thirdparty;
 
 import static com.box.l10n.mojito.android.strings.AndroidPluralQuantity.MANY;
+import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun;
 import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputSourceFile;
 import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.getOutputTargetFile;
 import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile;
@@ -9,11 +10,8 @@
 import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper;
 import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader;
 import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter;
-import com.box.l10n.mojito.entity.Locale;
 import com.box.l10n.mojito.entity.Repository;
 import com.box.l10n.mojito.entity.RepositoryLocale;
-import com.box.l10n.mojito.entity.ThirdPartyFileChecksum;
-import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId;
 import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils;
@@ -45,7 +43,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.regex.Pattern;
@@ -55,7 +52,6 @@
 import java.util.stream.StreamSupport;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
-import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -660,7 +656,11 @@ private void processPullBatch(
                                     "Error with download from Smartling, file content string is not present."));
 
                 if (isFileEqualToPreviousRun(
-                    repository, locale.getLocale(), fileName, fileContent)) {
+                    thirdPartyFileChecksumRepository,
+                    repository,
+                    locale.getLocale(),
+                    fileName,
+                    fileContent)) {
                   logger.info("Checksum match for " + fileName + ", skipping text unit import.");
                   return;
                 }
@@ -690,26 +690,6 @@ private void processPullBatch(
     }
   }
 
-  private boolean isFileEqualToPreviousRun(
-      Repository repository, Locale locale, String fileName, String fileContent) {
-
-    String currentChecksum = DigestUtils.md5Hex(fileContent);
-    ThirdPartyFileChecksumId thirdPartyFileChecksumId =
-        new ThirdPartyFileChecksumId(repository, locale, fileName);
-
-    Optional<ThirdPartyFileChecksum> thirdPartyFileChecksum =
-        thirdPartyFileChecksumRepository.findByThirdPartyFileChecksumId(thirdPartyFileChecksumId);
-    if (thirdPartyFileChecksum.isPresent()
-        && thirdPartyFileChecksum.get().getChecksum().equals(currentChecksum)) {
-      return true;
-    } else {
-      thirdPartyFileChecksumRepository.save(
-          new ThirdPartyFileChecksum(thirdPartyFileChecksumId, currentChecksum));
-    }
-
-    return false;
-  }
-
   @Override
   public void pushTranslations(
       Repository repository,

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java

@@ -1,5 +1,6 @@
 package com.box.l10n.mojito.service.thirdparty;
 
+import static com.box.l10n.mojito.service.thirdparty.ThirdPartyTMSUtils.isFileEqualToPreviousRun;
 import static com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils.isPluralFile;
 
 import com.box.l10n.mojito.entity.Repository;
@@ -58,6 +59,8 @@ public class ThirdPartyTMSSmartlingWithJson {
 
   MeterRegistry meterRegistry;
 
+  ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository;
+
   int batchSize = 5000;
 
   public ThirdPartyTMSSmartlingWithJson(
@@ -66,13 +69,15 @@ public ThirdPartyTMSSmartlingWithJson(
       TextUnitSearcher textUnitSearcher,
       TextUnitBatchImporterService textUnitBatchImporterService,
       SmartlingJsonKeys smartlingJsonKeys,
-      MeterRegistry meterRegistry) {
+      MeterRegistry meterRegistry,
+      ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) {
     this.smartlingClient = smartlingClient;
     this.smartlingJsonConverter = smartlingJsonConverter;
     this.textUnitSearcher = textUnitSearcher;
     this.textUnitBatchImporterService = textUnitBatchImporterService;
     this.smartlingJsonKeys = smartlingJsonKeys;
     this.meterRegistry = meterRegistry;
+    this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository;
   }
 
   void push(
@@ -201,6 +206,19 @@ void pull(Repository repository, String projectId, Map<String, String> localeMap
                     String localizedFileContent =
                         getLocalizedFileContent(projectId, file, smartlingLocale, false);
 
+                    if (isFileEqualToPreviousRun(
+                        thirdPartyFileChecksumRepository,
+                        repository,
+                        repositoryLocale.getLocale(),
+                        file.getFileUri(),
+                        localizedFileContent)) {
+                      logger.info(
+                          "Checksum match for "
+                              + file.getFileUri()
+                              + ", skipping text unit import.");
+                      return;
+                    }
+
                     ImmutableList<TextUnitDTO> textUnitDTOS =
                         smartlingJsonConverter.jsonStringToTextUnitDTOs(
                             localizedFileContent, TextUnitDTO::setTarget);

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSUtils.java

@@ -0,0 +1,35 @@
+package com.box.l10n.mojito.service.thirdparty;
+
+import com.box.l10n.mojito.entity.Locale;
+import com.box.l10n.mojito.entity.Repository;
+import com.box.l10n.mojito.entity.ThirdPartyFileChecksum;
+import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId;
+import java.util.Optional;
+import org.apache.commons.codec.digest.DigestUtils;
+
+public class ThirdPartyTMSUtils {
+
+  public static boolean isFileEqualToPreviousRun(
+      ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository,
+      Repository repository,
+      Locale locale,
+      String fileName,
+      String fileContent) {
+
+    String currentChecksum = DigestUtils.md5Hex(fileContent);
+    ThirdPartyFileChecksumId thirdPartyFileChecksumId =
+        new ThirdPartyFileChecksumId(repository, locale, fileName);
+
+    Optional<ThirdPartyFileChecksum> thirdPartyFileChecksum =
+        thirdPartyFileChecksumRepository.findByThirdPartyFileChecksumId(thirdPartyFileChecksumId);
+    if (thirdPartyFileChecksum.isPresent()
+        && thirdPartyFileChecksum.get().getChecksum().equals(currentChecksum)) {
+      return true;
+    } else {
+      thirdPartyFileChecksumRepository.save(
+          new ThirdPartyFileChecksum(thirdPartyFileChecksumId, currentChecksum));
+    }
+
+    return false;
+  }
+}

webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java

@@ -134,7 +134,7 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase {
 
   @Autowired MeterRegistry meterRegistry;
 
-  @Mock ThirdPartyFileChecksumRepository mockThirdPartyFileChecksumRepository;
+  @Autowired ThirdPartyFileChecksumRepository mockThirdPartyFileChecksumRepository;
 
   StubSmartlingResultProcessor resultProcessor;
 

webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java

@@ -30,6 +30,7 @@
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import org.junit.Assert;
@@ -80,6 +81,8 @@ public class ThirdPartyTMSSmartlingWithJsonTest extends ServiceTestBase {
 
   @Autowired TextUnitSearcher textUnitSearcher;
 
+  @Mock ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepositoryMock;
+
   SmartlingJsonConverter smartlingJsonConverter =
       new SmartlingJsonConverter(ObjectMapper.withIndentedOutput(), new SmartlingJsonKeys());
 
@@ -250,7 +253,8 @@ public void testGetTranslatedUnits() {
         ImmutableList.of(translatedTextUnitDto, untranslatedTextUnitDtoWithOriginalString);
 
     ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson =
-        new ThirdPartyTMSSmartlingWithJson(null, null, null, null, null, meterRegistryMock);
+        new ThirdPartyTMSSmartlingWithJson(
+            null, null, null, null, null, meterRegistryMock, thirdPartyFileChecksumRepositoryMock);
 
     ImmutableList<TextUnitDTO> result =
         thirdPartyTMSSmartlingWithJson.getTranslatedUnits(
@@ -362,12 +366,19 @@ public void testPullWithUntranslatedUnits() throws Exception {
     thirdPartyTMSSmartlingWithJsonMock.smartlingJsonConverter = smartlingJsonConverter;
     thirdPartyTMSSmartlingWithJsonMock.textUnitBatchImporterService =
         textUnitBatchImporterServiceMock;
+    thirdPartyTMSSmartlingWithJsonMock.thirdPartyFileChecksumRepository =
+        thirdPartyFileChecksumRepositoryMock;
 
     Mockito.when(
             thirdPartyTMSSmartlingWithJsonMock.getLocalizedFileContent(
                 projectId, smartlingFile, smartlingLocale, false))
         .thenReturn(smartlingJsonResponseWithOriginalString);
 
+    Mockito.when(
+            thirdPartyFileChecksumRepositoryMock.findByThirdPartyFileChecksumId(
+                isA(ThirdPartyFileChecksumId.class)))
+        .thenReturn(Optional.empty());
+
     // For the first pass, mock a fully translated response
     thirdPartyTMSSmartlingWithJsonMock.pull(repository, projectId, localeMapping);