diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/batch/io/SubmittedVariantsFileReader.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/batch/io/SubmittedVariantsFileReader.java new file mode 100644 index 000000000..9cb9720dd --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/batch/io/SubmittedVariantsFileReader.java @@ -0,0 +1,149 @@ +/* + * Copyright 2022 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.batch.io; + +import com.mongodb.BasicDBObject; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.Filters; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemStreamException; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.convert.MongoConverter; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Read all SubmittedVariants for a given study whose ids are given in the input file + */ +public class SubmittedVariantsFileReader implements ItemStreamReader { + + private static final Logger logger = LoggerFactory.getLogger(SubmittedVariantsFileReader.class); + + private static final String ASSEMBLY_FIELD = "seq"; + private static final String ACCESSION_FIELD = "accession"; + + private BufferedReader reader; + private boolean endOfFile = false; + private String assembly; + private String variantIdFile; + private MongoCursor evaCursor; + private MongoConverter converter; + private MongoTemplate mongoTemplate; + private int chunkSize; + + public SubmittedVariantsFileReader(String assembly, String variantIdFile, MongoTemplate mongoTemplate, int chunkSize) { + this.assembly = assembly; + this.variantIdFile = variantIdFile; + this.mongoTemplate = mongoTemplate; + this.chunkSize = chunkSize; + } + + @Override + public SubmittedVariantEntity read() { + if (evaCursor == null || !evaCursor.hasNext()) { + if (endOfFile) { + return null; + } + + loadNextBatchAndQuery(); + + if (evaCursor == null || !evaCursor.hasNext()) { + return null; + } + } + + Document nextElement = evaCursor.next(); + return getSubmittedVariantEntity(nextElement); + } + + private SubmittedVariantEntity getSubmittedVariantEntity(Document document) { + return converter.read(SubmittedVariantEntity.class, new BasicDBObject(document)); + } + + @Override + public void open(ExecutionContext executionContext) throws ItemStreamException { + try { + reader = new BufferedReader(new FileReader(variantIdFile)); + } catch (IOException e) { + throw new ItemStreamException("Failed to open the file (" + variantIdFile + ") with variant IDs", e); + } + initializeReader(); + } + + public void initializeReader() { + converter = mongoTemplate.getConverter(); + loadNextBatchAndQuery(); + } + + private void loadNextBatchAndQuery() { + List variantIds = new ArrayList<>(); + String line; + + try { + while (variantIds.size() < chunkSize && (line = reader.readLine()) != null) { + variantIds.add(Integer.parseInt(line.trim())); + } + if (variantIds.isEmpty()) { + endOfFile = true; + return; + } + } catch (IOException e) { + throw new ItemStreamException("Error reading variant IDs from file", e); + } + + Bson query = Filters.and(Filters.in(ACCESSION_FIELD, variantIds), Filters.eq(ASSEMBLY_FIELD, assembly)); + logger.info("Issuing find in EVA collection for a batch of IDs: {}", query); + FindIterable submittedVariantsEVA = getSubmittedVariants(query, SubmittedVariantEntity.class); + evaCursor = submittedVariantsEVA.iterator(); + } + + private FindIterable getSubmittedVariants(Bson query, Class entityClass) { + return mongoTemplate.getCollection(mongoTemplate.getCollectionName(entityClass)) + .find(query) + .noCursorTimeout(true) + .batchSize(chunkSize); + } + + @Override + public void update(ExecutionContext executionContext) throws ItemStreamException { + + } + + @Override + public void close() throws ItemStreamException { + try { + if (evaCursor != null) { + evaCursor.close(); + } + if (reader != null) { + reader.close(); + } + } catch (IOException e) { + throw new ItemStreamException("Exception while closing resources", e); + } + } +} diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/BeanNames.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/BeanNames.java index 523bf3490..9b23bd086 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/BeanNames.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/BeanNames.java @@ -19,12 +19,18 @@ public class BeanNames { public static final String STUDY_SUMITTED_VARIANTS_READER = "STUDY_SUMITTED_VARIANTS_READER"; + public static final String SUBMITTED_VARIANTS_FILE_READER = "SUBMITTED_VARIANTS_FILE_READER"; + public static final String STUDY_DEPRECATION_WRITER = "STUDY_DEPRECATION_WRITER"; public static final String DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP = "DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP"; public static final String DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB = "DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB"; + public static final String DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP = "DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP"; + + public static final String DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB = "DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB"; + public static final String DEPRECATION_PROGRESS_LISTENER = "DEPRECATION_PROGRESS_LISTENER"; } diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsFileReaderConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsFileReaderConfiguration.java new file mode 100644 index 000000000..783afe193 --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/io/StudySubmittedVariantsFileReaderConfiguration.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.configuration.batch.io; + +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.mongodb.core.MongoTemplate; +import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; +import uk.ac.ebi.eva.accession.deprecate.batch.io.SubmittedVariantsFileReader; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.parameters.InputParameters; + +@Configuration +@Import({MongoConfiguration.class}) +public class StudySubmittedVariantsFileReaderConfiguration { + + @Bean(BeanNames.SUBMITTED_VARIANTS_FILE_READER) + @StepScope + SubmittedVariantsFileReader submittedVariantsFileReader(MongoTemplate mongoTemplate, InputParameters parameters) { + return new SubmittedVariantsFileReader(parameters.getAssemblyAccession(), parameters.getVariantIdFile(), + mongoTemplate, parameters.getChunkSize()); + } +} diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java new file mode 100644 index 000000000..95d9cb747 --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/jobs/DeprecateSubmittedVariantsFromFileJobConfiguration.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.BatchConfigurer; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.launch.support.RunIdIncrementer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.commons.batch.configuration.SpringBoot1CompatibilityConfiguration; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; + +@Configuration +@EnableBatchProcessing +public class DeprecateSubmittedVariantsFromFileJobConfiguration { + + @Autowired + @Qualifier(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) + private Step deprecateSubmittedVariantsFromFileStep; + + @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) + public Job deprecateStudySubmittedVariantsFromFileJob(JobBuilderFactory jobBuilderFactory) { + return jobBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) + .incrementer(new RunIdIncrementer()) + .start(deprecateSubmittedVariantsFromFileStep) + .build(); + } + + @Bean + public BatchConfigurer configurer(DataSource dataSource, EntityManagerFactory entityManagerFactory) + throws Exception { + return SpringBoot1CompatibilityConfiguration.getSpringBoot1CompatibleBatchConfigurer(dataSource, + entityManagerFactory); + } +} diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java new file mode 100644 index 000000000..1ef5835fa --- /dev/null +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/configuration/batch/steps/DeprecateSubmittedVariantsFromFileStepConfiguration.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps; + +import org.springframework.batch.core.Step; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.step.tasklet.TaskletStep; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.repeat.policy.SimpleCompletionPolicy; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import uk.ac.ebi.eva.accession.core.model.eva.SubmittedVariantEntity; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; + +@Configuration +@EnableBatchProcessing +public class DeprecateSubmittedVariantsFromFileStepConfiguration { + + @Autowired + @Qualifier(BeanNames.SUBMITTED_VARIANTS_FILE_READER) + private ItemStreamReader submittedVariantsFileReader; + + @Autowired + @Qualifier(BeanNames.STUDY_DEPRECATION_WRITER) + private ItemWriter submittedVariantDeprecationWriter; + + @Autowired + @Qualifier(BeanNames.DEPRECATION_PROGRESS_LISTENER) + private StepExecutionListener progressListener; + + @Bean(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) + public Step deprecateSubmittedVariantsFromFileStep(StepBuilderFactory stepBuilderFactory, + SimpleCompletionPolicy chunkSizeCompletionPolicy) { + TaskletStep step = stepBuilderFactory.get(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP) + .chunk(chunkSizeCompletionPolicy) + .reader(submittedVariantsFileReader) + .writer(submittedVariantDeprecationWriter) + .listener(progressListener) + .build(); + return step; + } +} diff --git a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/parameters/InputParameters.java b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/parameters/InputParameters.java index 395a2e250..6fd352eda 100644 --- a/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/parameters/InputParameters.java +++ b/eva-accession-deprecate/src/main/java/uk/ac/ebi/eva/accession/deprecate/parameters/InputParameters.java @@ -27,6 +27,8 @@ public class InputParameters { private String deprecationReason; + private String variantIdFile; + public int getChunkSize() { return chunkSize; } @@ -66,4 +68,12 @@ public String getDeprecationReason() { public void setDeprecationReason(String deprecationReason) { this.deprecationReason = deprecationReason; } + + public String getVariantIdFile() { + return variantIdFile; + } + + public void setVariantIdFile(String variantIdFile) { + this.variantIdFile = variantIdFile; + } } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java index 584b169ac..56efe9658 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/MongoTestDatabaseSetup.java @@ -83,6 +83,22 @@ public static void populateTestDB(MongoTemplate mongoTemplate) { mongoTemplate.save(rs2, mongoTemplate.getCollectionName(ClusteredVariantEntity.class)); } + public static void populateTestDBForFile(MongoTemplate mongoTemplate) { + // rs1 -> ss1,ss2 + // rs2 -> ss3,ss4 + ss1 = createSS(STUDY1, 5L, 1L, 100L, "C", "T"); + ss2 = createSS(STUDY1, 6L, 1L, 100L, "C", "A"); + ss3 = createSS(STUDY2, 7L, 5L, 102L, "T", "G"); + ss4 = createSS(STUDY2, 8L, 5L, 102L, "T", "A"); + + rs1 = createRS(ss1); + mongoTemplate.save(rs1, mongoTemplate.getCollectionName(DbsnpClusteredVariantEntity.class)); + + mongoTemplate.insert(Arrays.asList(ss1, ss2, ss3, ss4), SubmittedVariantEntity.class); + rs2 = createRS(ss3); + mongoTemplate.save(rs2, mongoTemplate.getCollectionName(ClusteredVariantEntity.class)); + } + public static void assertPostDeprecationDatabaseState(MongoTemplate mongoTemplate) { // ss4 was not deprecated and still remains assertEquals(1, mongoTemplate.findAll(SubmittedVariantEntity.class).size()); diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java index 933c56803..c571c2ffd 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateStudySubmittedVariantsJobConfigurationTest.java @@ -31,6 +31,7 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.batch.test.JobRepositoryTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.annotation.DirtiesContext; @@ -56,6 +57,7 @@ import static org.springframework.data.mongodb.core.query.Query.query; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP; +import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @RunWith(SpringRunner.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @@ -65,7 +67,8 @@ public class DeprecateStudySubmittedVariantsJobConfigurationTest { private static final String TEST_DB = "test-db"; @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; + @Qualifier(JOB_LAUNCHER_FROM_MONGO) + private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @Autowired private JobExplorer jobExplorer; @@ -113,7 +116,7 @@ public void tearDown() { @Test public void basicJobCompletion() throws Exception { - JobExecution jobExecution = jobLauncherTestUtils.launchJob(); + JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); List expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); @@ -131,7 +134,7 @@ private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) @Test @DirtiesContext public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { - JobExecution jobExecution = jobLauncherTestUtils.launchJob(); + JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); List expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); @@ -143,7 +146,7 @@ public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { jobExecution.getJobParameters()) .getJobInstance().getInstanceId(); - jobExecution = jobLauncherTestUtils.launchJob(); + jobExecution = jobLauncherTestUtilsFromMongo.launchJob(); expectedSteps = Collections.singletonList(DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertStepsExecuted(expectedSteps, jobExecution); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java new file mode 100644 index 000000000..2ffdb2aa6 --- /dev/null +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/jobs/DeprecateSubmittedVariantsFromFileJobConfigurationTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2024 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.configuration.jobs; + +import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; +import com.mongodb.MongoClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.explore.JobExplorer; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.batch.test.JobRepositoryTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.eva.accession.core.runner.CommandLineRunnerUtils; +import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; + +import javax.sql.DataSource; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP; +import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") +public class DeprecateSubmittedVariantsFromFileJobConfigurationTest { + + private static final String TEST_DB = "test-db"; + + @Autowired + @Qualifier(JOB_LAUNCHER_FROM_FILE) + private JobLauncherTestUtils jobLauncherTestUtilsFromFile; + + @Autowired + private JobExplorer jobExplorer; + + @Autowired + private JobRepository jobRepository; + + @Autowired + private DataSource datasource; + + @Autowired + private MongoClient mongoClient; + + @Autowired + private MongoTemplate mongoTemplate; + + private JobRepositoryTestUtils jobRepositoryTestUtils; + + //Required by nosql-unit + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( + MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + + @Test + public void contextLoads() { + + } + + @Before + public void setUp() throws Exception { + this.mongoClient.dropDatabase(TEST_DB); + MongoTestDatabaseSetup.populateTestDB(this.mongoTemplate); + jobRepositoryTestUtils = new JobRepositoryTestUtils(jobRepository, datasource); + jobRepositoryTestUtils.removeJobExecutions(); + } + + @After + public void tearDown() { + this.mongoClient.dropDatabase(TEST_DB); + jobRepositoryTestUtils.removeJobExecutions(); + } + + @Test + public void basicJobCompletion() throws Exception { + JobExecution jobExecution = jobLauncherTestUtilsFromFile.launchJob(); + + List expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); + assertStepsExecuted(expectedSteps, jobExecution); + + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); + } + + private void assertStepsExecuted(List expectedSteps, JobExecution jobExecution) { + Collection stepExecutions = jobExecution.getStepExecutions(); + List steps = stepExecutions.stream().map(StepExecution::getStepName).collect(Collectors.toList()); + assertEquals(expectedSteps, steps); + } + + @Test + @DirtiesContext + public void restartCompletedJobThatIsAlreadyInTheRepository() throws Exception { + JobExecution jobExecution = jobLauncherTestUtilsFromFile.launchJob(); + + List expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); + assertStepsExecuted(expectedSteps, jobExecution); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); + + long instanceIdFirstJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); + + jobExecution = jobLauncherTestUtilsFromFile.launchJob(); + expectedSteps = Collections.singletonList(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); + assertStepsExecuted(expectedSteps, jobExecution); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + long instanceIdSecondJob = CommandLineRunnerUtils.getLastJobExecution(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB, + jobExplorer, + jobExecution.getJobParameters()) + .getJobInstance().getInstanceId(); + assertNotEquals(instanceIdSecondJob, instanceIdFirstJob); + MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); + } +} diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java index 705d51850..269b5b925 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateStudySubmittedVariantsStepConfigurationTest.java @@ -27,6 +27,7 @@ import org.springframework.batch.core.JobExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.test.context.ContextConfiguration; @@ -40,6 +41,7 @@ import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_MONGO; @RunWith(SpringRunner.class) @ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) @@ -49,7 +51,8 @@ public class DeprecateStudySubmittedVariantsStepConfigurationTest { private static final String TEST_DB = "test-db"; @Autowired - private JobLauncherTestUtils jobLauncherTestUtils; + @Qualifier(JOB_LAUNCHER_FROM_MONGO) + private JobLauncherTestUtils jobLauncherTestUtilsFromMongo; @Autowired private MongoClient mongoClient; @@ -83,7 +86,7 @@ public void contextLoads() { @Test public void variantsDeprecated() { - JobExecution jobExecution = jobLauncherTestUtils.launchStep(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); + JobExecution jobExecution = jobLauncherTestUtilsFromMongo.launchStep(BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_STEP); assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); } diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java new file mode 100644 index 000000000..aafe0c18d --- /dev/null +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/configuration/steps/DeprecateSubmittedVariantsFromFileStepConfigurationTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2024 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.accession.deprecate.configuration.steps; + +import com.lordofthejars.nosqlunit.mongodb.MongoDbConfigurationBuilder; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; +import com.mongodb.MongoClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.core.BatchStatus; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import uk.ac.ebi.eva.accession.deprecate.MongoTestDatabaseSetup; +import uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration; +import uk.ac.ebi.eva.accession.deprecate.test.configuration.MongoTestConfiguration; +import uk.ac.ebi.eva.accession.deprecate.test.rule.FixSpringMongoDbRule; + +import static org.junit.Assert.assertEquals; +import static uk.ac.ebi.eva.accession.deprecate.test.configuration.BatchTestConfiguration.JOB_LAUNCHER_FROM_FILE; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {BatchTestConfiguration.class, MongoTestConfiguration.class}) +@TestPropertySource("classpath:deprecate-submitted-variants-from-file-test.properties") +public class DeprecateSubmittedVariantsFromFileStepConfigurationTest { + + private static final String TEST_DB = "test-db"; + + @Autowired + @Qualifier(JOB_LAUNCHER_FROM_FILE) + private JobLauncherTestUtils jobLauncherTestUtilsFromFile; + + @Autowired + private MongoClient mongoClient; + + @Autowired + private MongoTemplate mongoTemplate; + + //Required by nosql-unit + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = new FixSpringMongoDbRule( + MongoDbConfigurationBuilder.mongoDb().databaseName(TEST_DB).build()); + + @Before + public void setUp() { + this.mongoClient.dropDatabase(TEST_DB); + MongoTestDatabaseSetup.populateTestDBForFile(this.mongoTemplate); + } + + @After + public void tearDown() { + this.mongoClient.dropDatabase(TEST_DB); + } + + @Test + public void contextLoads() { + + } + + @Test + public void variantsDeprecated() { + JobExecution jobExecution = jobLauncherTestUtilsFromFile.launchStep(BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_STEP); + assertEquals(BatchStatus.COMPLETED, jobExecution.getStatus()); + MongoTestDatabaseSetup.assertPostDeprecationDatabaseState(this.mongoTemplate); + } +} \ No newline at end of file diff --git a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java index a5f85c345..05b7ccab3 100644 --- a/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java +++ b/eva-accession-deprecate/src/test/java/uk/ac/ebi/eva/accession/deprecate/test/configuration/BatchTestConfiguration.java @@ -15,34 +15,67 @@ */ package uk.ac.ebi.eva.accession.deprecate.test.configuration; +import org.springframework.batch.core.Job; import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; - import uk.ac.ebi.eva.accession.core.configuration.nonhuman.MongoConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.InputParametersConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsFileReaderConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.StudySubmittedVariantsReaderConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateStudySubmittedVariantsJobConfiguration; -import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateStudySubmittedVariantsStepConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.io.SubmittedVariantDeprecationWriterConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateSubmittedVariantsFromFileJobConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.jobs.DeprecateStudySubmittedVariantsJobConfiguration; import uk.ac.ebi.eva.accession.deprecate.configuration.batch.listeners.ListenerConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.policies.ChunkSizeCompletionPolicyConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateSubmittedVariantsFromFileStepConfiguration; +import uk.ac.ebi.eva.accession.deprecate.configuration.batch.steps.DeprecateStudySubmittedVariantsStepConfiguration; + +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB; +import static uk.ac.ebi.eva.accession.deprecate.configuration.BeanNames.DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB; @EnableAutoConfiguration @Import({MongoConfiguration.class, InputParametersConfiguration.class, ChunkSizeCompletionPolicyConfiguration.class, StudySubmittedVariantsReaderConfiguration.class, + StudySubmittedVariantsFileReaderConfiguration.class, SubmittedVariantDeprecationWriterConfiguration.class, DeprecateStudySubmittedVariantsStepConfiguration.class, + DeprecateSubmittedVariantsFromFileStepConfiguration.class, DeprecateStudySubmittedVariantsJobConfiguration.class, + DeprecateSubmittedVariantsFromFileJobConfiguration.class, ListenerConfiguration.class }) public class BatchTestConfiguration { - @Bean - public JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); + public static final String JOB_LAUNCHER_FROM_MONGO = "JOB_LAUNCHER_FROM_MONGO"; + public static final String JOB_LAUNCHER_FROM_FILE = "JOB_LAUNCHER_FROM_FILE"; + + @Bean(JOB_LAUNCHER_FROM_MONGO) + public JobLauncherTestUtils jobLauncherTestUtilsFromMongo() { + + return new JobLauncherTestUtils() { + @Override + @Autowired + public void setJob(@Qualifier(DEPRECATE_STUDY_SUBMITTED_VARIANTS_JOB) Job job) { + super.setJob(job); + } + }; + } + + @Bean(JOB_LAUNCHER_FROM_FILE) + public JobLauncherTestUtils jobLauncherTestUtilsFromFile() { + + return new JobLauncherTestUtils() { + @Override + @Autowired + public void setJob(@Qualifier(DEPRECATE_SUBMITTED_VARIANTS_FROM_FILE_JOB) Job job) { + super.setJob(job); + } + }; } } diff --git a/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties new file mode 100644 index 000000000..68229ca2e --- /dev/null +++ b/eva-accession-deprecate/src/test/resources/deprecate-submitted-variants-from-file-test.properties @@ -0,0 +1,40 @@ +spring.batch.job.names=DEPRECATE_STUDY_SUBMITTED_VARIANTS_FROM_FILE_JOB + +parameters.assemblyAccession=GCA_000000001.1 +parameters.deprecationIdSuffix=TEST +parameters.deprecationReason=DEPRECATION_REASON +parameters.chunkSize=2 +parameters.variantIdFile=src/test/resources/variant-ids.csv + +eva.count-stats.url=http://localhost:8080 +eva.count-stats.username=username +eva.count-stats.password=password + +accessioning.instanceId=test-instance-01 +accessioning.submitted.categoryId=test-ss +accessioning.clustered.categoryId=test-rs + +accessioning.monotonic.test-ss.blockSize=100000 +accessioning.monotonic.test-ss.blockStartValue=4 +accessioning.monotonic.test-ss.nextBlockInterval=1000000000 +accessioning.monotonic.test-rs.blockSize=100000 +accessioning.monotonic.test-rs.blockStartValue=4 +accessioning.monotonic.test-rs.nextBlockInterval=1000000000 + +spring.data.mongodb.database=test-db +# This symbolic variable is required because +# this will be replaced with a default value "localhost" when run locally +# (due to the filtering attribute specified in this project's POM) +# (OR) this variable can be overridden in Gitlab with "mongo" (Gitlab services are exposed under their own hostname) +spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.password= +spring.data.mongodb.port=27017 +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true + +human.mongodb.database=human-variants-test +human.mongodb.host=|eva.mongo.host.test| +human.mongodb.password= +human.mongodb.port=27017 + +mongodb.read-preference=primary diff --git a/eva-accession-deprecate/src/test/resources/variant-ids.csv b/eva-accession-deprecate/src/test/resources/variant-ids.csv new file mode 100644 index 000000000..92836bfa3 --- /dev/null +++ b/eva-accession-deprecate/src/test/resources/variant-ids.csv @@ -0,0 +1,3 @@ +5 +6 +7 \ No newline at end of file