Skip to content

Commit

Permalink
Short circuit on Smartling pull if no changes to translated files
Browse files Browse the repository at this point in the history
  • Loading branch information
maallen committed Aug 4, 2023
1 parent 92fadd6 commit 872a2a0
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 21 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package com.box.l10n.mojito.entity;

import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.EmbeddedId;
import javax.persistence.Entity;
import javax.persistence.Index;
import javax.persistence.Table;
import org.hibernate.envers.Audited;
import org.hibernate.envers.RelationTargetAuditMode;

/** Entity that stores the checksum of a translated file downloaded via a third party sync. */
@Entity
@Audited(targetAuditMode = RelationTargetAuditMode.NOT_AUDITED)
@Table(
name = "third_party_sync_file_checksum",
indexes = {
@Index(
name = "UK__THIRD_PARTY_FILE_CHECKSUM__REPOSITORY_ID__FILE_NAME",
columnList = "repository_id, file_name",
unique = true),
})
public class ThirdPartyFileChecksum implements Serializable {

@EmbeddedId private ThirdPartyFileChecksumId thirdPartyFileChecksumId;

@Column(name = "checksum")
private String checksum;

public ThirdPartyFileChecksum() {}

public ThirdPartyFileChecksum(
ThirdPartyFileChecksumId thirdPartyFileChecksumId, String checksum) {
this.thirdPartyFileChecksumId = thirdPartyFileChecksumId;
this.checksum = checksum;
}

public ThirdPartyFileChecksumId getThirdPartyFileChecksumId() {
return thirdPartyFileChecksumId;
}

public void setThirdPartyFileChecksumId(ThirdPartyFileChecksumId thirdPartyFileChecksumId) {
this.thirdPartyFileChecksumId = thirdPartyFileChecksumId;
}

public String getChecksum() {
return checksum;
}

public void setChecksum(String checksum) {
this.checksum = checksum;
}

public Locale getLocale() {
return thirdPartyFileChecksumId.getLocale();
}

public Repository getRepository() {
return thirdPartyFileChecksumId.getRepository();
}

public String getFileName() {
return thirdPartyFileChecksumId.getFileName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package com.box.l10n.mojito.entity;

import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Embeddable;
import javax.persistence.ForeignKey;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;

@Embeddable
public class ThirdPartyFileChecksumId implements Serializable {

@ManyToOne(optional = false)
@JoinColumn(
name = "repository_id",
foreignKey = @ForeignKey(name = "FK__THIRD_PARTY_CHECKSUM__REPO__ID"))
private Repository repository;

@Column(name = "file_name")
private String fileName;

@ManyToOne(optional = false)
@JoinColumn(
name = "locale_id",
foreignKey = @ForeignKey(name = "FK__THIRD_PARTY_CHECKSUM__LOCALE__ID"))
private Locale locale;

public ThirdPartyFileChecksumId() {}

public ThirdPartyFileChecksumId(Repository repository, Locale locale, String fileName) {
this.repository = repository;
this.locale = locale;
this.fileName = fileName;
}

public Locale getLocale() {
return locale;
}

public void setLocale(Locale locale) {
this.locale = locale;
}

public Repository getRepository() {
return repository;
}

public void setRepository(Repository repository) {
this.repository = repository;
}

public String getFileName() {
return fileName;
}

public void setFileName(String fileName) {
this.fileName = fileName;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.box.l10n.mojito.service.thirdparty;

import com.box.l10n.mojito.entity.ThirdPartyFileChecksum;
import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.rest.core.annotation.RepositoryRestResource;

@RepositoryRestResource(exported = false)
public interface ThirdPartyFileChecksumRepository
extends JpaRepository<ThirdPartyFileChecksum, ThirdPartyFileChecksumId> {

Optional<ThirdPartyFileChecksum> findByThirdPartyFileChecksumId(
ThirdPartyFileChecksumId thirdPartyFileChecksumId);
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@
import com.box.l10n.mojito.android.strings.AndroidStringDocumentMapper;
import com.box.l10n.mojito.android.strings.AndroidStringDocumentReader;
import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter;
import com.box.l10n.mojito.entity.Locale;
import com.box.l10n.mojito.entity.Repository;
import com.box.l10n.mojito.entity.RepositoryLocale;
import com.box.l10n.mojito.entity.ThirdPartyFileChecksum;
import com.box.l10n.mojito.entity.ThirdPartyFileChecksumId;
import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository;
import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile;
import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFileUtils;
Expand Down Expand Up @@ -42,6 +45,7 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.regex.Pattern;
Expand All @@ -51,6 +55,7 @@
import java.util.stream.StreamSupport;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import org.apache.commons.codec.digest.DigestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -98,6 +103,8 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS {

private final MeterRegistry meterRegistry;

private final ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository;

private final Set<String> supportedImageExtensions =
Sets.newHashSet("png", "jpg", "jpeg", "gif", "tiff");

Expand All @@ -115,7 +122,8 @@ public ThirdPartyTMSSmartling(
ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson,
ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
MeterRegistry meterRegistry) {
MeterRegistry meterRegistry,
ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) {
this(
smartlingClient,
textUnitSearcher,
Expand All @@ -126,7 +134,8 @@ public ThirdPartyTMSSmartling(
thirdPartyTMSSmartlingGlossary,
assetTextUnitToTMTextUnitRepository,
DEFAULT_BATCH_SIZE,
meterRegistry);
meterRegistry,
thirdPartyFileChecksumRepository);
}

public ThirdPartyTMSSmartling(
Expand All @@ -139,7 +148,8 @@ public ThirdPartyTMSSmartling(
ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
int batchSize,
MeterRegistry meterRegistry) {
MeterRegistry meterRegistry,
ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) {
this.smartlingClient = smartlingClient;
this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys;
this.textUnitBatchImporterService = textUnitBatchImporterService;
Expand All @@ -150,6 +160,7 @@ public ThirdPartyTMSSmartling(
this.thirdPartyTMSSmartlingGlossary = thirdPartyTMSSmartlingGlossary;
this.assetTextUnitToTMTextUnitRepository = assetTextUnitToTMTextUnitRepository;
this.meterRegistry = meterRegistry;
this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository;
}

@Override
Expand Down Expand Up @@ -648,6 +659,12 @@ private void processPullBatch(
new SmartlingClientException(
"Error with download from Smartling, file content string is not present."));

if (isFileEqualToPreviousRun(
repository, locale.getLocale(), fileName, fileContent)) {
logger.info("Checksum match for " + fileName + ", skipping text unit import.");
return;
}

List<TextUnitDTO> textUnits;

try {
Expand All @@ -673,6 +690,26 @@ private void processPullBatch(
}
}

private boolean isFileEqualToPreviousRun(
Repository repository, Locale locale, String fileName, String fileContent) {

String currentChecksum = DigestUtils.md5Hex(fileContent);
ThirdPartyFileChecksumId thirdPartyFileChecksumId =
new ThirdPartyFileChecksumId(repository, locale, fileName);

Optional<ThirdPartyFileChecksum> thirdPartyFileChecksum =
thirdPartyFileChecksumRepository.findByThirdPartyFileChecksumId(thirdPartyFileChecksumId);
if (thirdPartyFileChecksum.isPresent()
&& thirdPartyFileChecksum.get().getChecksum().equals(currentChecksum)) {
return true;
} else {
thirdPartyFileChecksumRepository.save(
new ThirdPartyFileChecksum(thirdPartyFileChecksumId, currentChecksum));
}

return false;
}

@Override
public void pushTranslations(
Repository repository,
Expand Down Expand Up @@ -1037,7 +1074,7 @@ private TextUnitSearcherParameters baseParams(
result.setPluralFormsExcluded(pluralFormsExcluded);
result.setSkipTextUnitWithPattern(skipTextUnitsWithPattern);
result.setSkipAssetPathWithPattern(skipAssetsWithPathPattern);

result.setOrdered(true);
if (!Strings.isNullOrEmpty(pluralFormOther)) {
result.setPluralFormOther(pluralFormOther);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,10 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
c.setOffset(searchParameters.getOffset());
}

if (searchParameters.isOrdered()) {
c.setOrder(NativeExps.order().add("tu.id", NativeOrderExp.OrderType.ASC));
}

if (searchParameters instanceof TextUnitSearcherParametersForTesting) {
TextUnitSearcherParametersForTesting textUnitSearcherParametersForTesting =
(TextUnitSearcherParametersForTesting) searchParameters;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public class TextUnitSearcherParameters {
boolean untranslatedOrTranslationNeeded = false;
boolean pluralFormsFiltered = true;
boolean pluralFormsExcluded = false;

boolean isOrdered = false;
Long pluralFormId;
Boolean doNotTranslateFilter;
DateTime tmTextUnitCreatedBefore;
Expand Down Expand Up @@ -308,4 +310,12 @@ public String getSkipAssetPathWithPattern() {
public void setSkipAssetPathWithPattern(String skipAssetPathWithPattern) {
this.skipAssetPathWithPattern = skipAssetPathWithPattern;
}

public boolean isOrdered() {
return isOrdered;
}

public void setOrdered(boolean ordered) {
isOrdered = ordered;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
create table third_party_sync_file_checksum (repository_id bigint not null, locale_id bigint not null, checksum char(32) not null, file_name varchar(255) not null);
alter table third_party_sync_file_checksum add constraint FK__THIRD_PARTY_CHECKSUM__REPO__ID foreign key (repository_id) references repository (id);
alter table third_party_sync_file_checksum add constraint FK__THIRD_PARTY_CHECKSUM__LOCALE__ID foreign key (locale_id) references locale (id);
Loading

0 comments on commit 872a2a0

Please sign in to comment.