Merge pull request #96 from EBIvariation/feature/job-parameters

Architecture improvements - More Spring Batch compliant, independent tests

examples/application.properties

@@ -4,11 +4,8 @@
 spring.profiles.active=production
 
 
-# EXTERNAL APPLICATIONS
+# OpenCGA
 app.opencga.path=/path/to/opencga
-app.vep.path=/path/to/variant_effect_predictor.pl
-app.vep.num-forks=4
-
 
 # STEPS MANAGEMENT
 ## Job repository database
@@ -29,15 +26,7 @@ spring.data.mongodb.port=
 spring.data.mongodb.authentication-database=
 spring.data.mongodb.username=
 spring.data.mongodb.password=
-spring.data.mongodb.database=
 config.db.read-preference=primary
-db.collections.features.name=features
-db.collections.stats.name=populationStatistics
-# TODO The following 2 properties will be used exclusive after removing readers and writers dependency 
-# on OpenCGA. At the moment they need to be specified in both.
-db.collections.files.name=files
-db.collections.variants.name=variants
-
 
 # LOGGING
 # https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-logging.html

examples/initialize-database.properties

@@ -4,3 +4,5 @@ spring.batch.job.names=init-database-job
 # INITIALIZATION PARAMETERS
 input.gtf=/home/cyenyxe/tmp/sample.gtf.gz
 
+# Database configuration
+spring.data.mongodb.database=

examples/load-aggregated-vcf.properties

@@ -17,18 +17,21 @@ output.dir.statistics=/path/to/statistics-files/
 input.fasta=/path/to/homo_sapiens/sequence.fa
 
 # VEP
+app.vep.path=/path/to/variant_effect_predictor.pl
 app.vep.cache.path=/path/to/vep/cache_folder
 app.vep.cache.version=82
 app.vep.cache.species=homo_sapiens
-
+app.vep.num-forks=4
 
 # STEPS MANAGEMENT
 
 ## Skip steps
 statistics.skip=false
 annotation.skip=false
 
-## Repeat steps
-## true: The already COMPLETED steps will be rerun. This is restarting the job from the beginning
-## false(default): if the job was aborted and is relaunched, COMPLETEd steps will NOT be done again
-config.restartability.allow=false
+# Database configuration
+spring.data.mongodb.database=
+# TODO The following 2 properties will be used exclusive after removing readers and writers dependency
+# on OpenCGA. At the moment they need to be specified in both.
+db.collections.files.name=files
+db.collections.variants.name=variants

examples/load-genotyped-vcf.properties

@@ -19,9 +19,11 @@ output.dir.statistics=/path/to/statistics-files/
 
 
 # VEP
+app.vep.path=/path/to/variant_effect_predictor.pl
 app.vep.cache.path=/path/to/vep/cache_folder
 app.vep.cache.version=82
 app.vep.cache.species=homo_sapiens
+app.vep.num-forks=4
 
 
 # STEPS MANAGEMENT
@@ -30,8 +32,9 @@ app.vep.cache.species=homo_sapiens
 statistics.skip=false
 annotation.skip=false
 
-## Repeat steps
-## true: The already COMPLETED steps will be rerun. This is restarting the job from the beginning
-## false(default): if the job was aborted and is relaunched, COMPLETEd steps will NOT be done again
-config.restartability.allow=false
-
+# Database configuration
+spring.data.mongodb.database=
+# TODO The following 2 properties will be used exclusive after removing readers and writers dependency
+# on OpenCGA. At the moment they need to be specified in both.
+db.collections.files.name=files
+db.collections.variants.name=variants

src/main/java/uk/ac/ebi/eva/pipeline/Application.java

@@ -17,6 +17,9 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner;
+import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
+import org.springframework.context.ConfigurableApplicationContext;
 
 
 /**
@@ -28,7 +31,7 @@
  * Append any parameter as needed.
  * TODO document all parameters
  */
-@SpringBootApplication
+@SpringBootApplication(exclude = {MongoDataAutoConfiguration.class, JobLauncherCommandLineRunner.class})
 public class Application {
 
     public static final String VARIANT_WRITER_MONGO_PROFILE = "variant-writer-mongo";
@@ -45,6 +48,7 @@ public class Application {
     public static final String MONGO_EXPERIMENTAL_PROFILE = "experimental";
 
     public static void main(String[] args) throws Exception {
-        SpringApplication.run(Application.class, args);
+        ConfigurableApplicationContext context = SpringApplication.run(Application.class, args);
+        System.exit(SpringApplication.exit(context));
     }
 }

src/main/java/uk/ac/ebi/eva/pipeline/configuration/ChunkSizeCompletionPolicyConfiguration.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2017 EMBL - European Bioinformatics Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.ac.ebi.eva.pipeline.configuration;
+
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.repeat.policy.SimpleCompletionPolicy;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import uk.ac.ebi.eva.pipeline.parameters.ChunkSizeParameters;
+
+/**
+ * Spring configuration to inject a SimplecompletionPolicy that modifies the chunk size with the configured
+ * JobParameters chunk size.
+ */
+@Configuration
+public class ChunkSizeCompletionPolicyConfiguration {
+
+    @Bean
+    @StepScope
+    public SimpleCompletionPolicy chunkSizecompletionPolicy(ChunkSizeParameters chunkSizeParameters) {
+        return new SimpleCompletionPolicy(chunkSizeParameters.getChunkSize());
+    }
+
+}

src/main/java/uk/ac/ebi/eva/pipeline/configuration/JobExecutionApplicationListenerConfiguration.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015-2017 EMBL - European Bioinformatics Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.ac.ebi.eva.pipeline.configuration;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import uk.ac.ebi.eva.pipeline.runner.JobExecutionApplicationListener;
+
+/**
+ * This configuration adds to the application context a JobExecutionApplicationListener that holds the state of the
+ * last job execution. This is used from the runner to decide the return code of the application.
+ */
+@Configuration
+public class JobExecutionApplicationListenerConfiguration {
+
+    @Bean
+    public JobExecutionApplicationListener applicationJobInstanceListener() {
+        return new JobExecutionApplicationListener();
+    }
+
+}

src/main/java/uk/ac/ebi/eva/pipeline/configuration/JobExecutionDeciderConfiguration.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017 EMBL - European Bioinformatics Institute
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.ac.ebi.eva.pipeline.configuration;
+
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.batch.core.job.flow.JobExecutionDecider;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import uk.ac.ebi.eva.pipeline.jobs.deciders.SkipStepDecider;
+import uk.ac.ebi.eva.pipeline.parameters.JobParametersNames;
+
+import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.ANNOTATION_SKIP_STEP_DECIDER;
+import static uk.ac.ebi.eva.pipeline.configuration.BeanNames.STATISTICS_SKIP_STEP_DECIDER;
+
+/**
+ * This class defines the beans for the deciders to skip annotation and statistics step.
+ */
+@Configuration
+@EnableBatchProcessing
+public class JobExecutionDeciderConfiguration {
+
+    @Bean(ANNOTATION_SKIP_STEP_DECIDER)
+    public JobExecutionDecider annotationSkipStepDecider() {
+        return new SkipStepDecider(JobParametersNames.ANNOTATION_SKIP);
+    }
+
+    @Bean(STATISTICS_SKIP_STEP_DECIDER)
+    public JobExecutionDecider statisticsSkipStepDecider() {
+        return new SkipStepDecider(JobParametersNames.STATISTICS_SKIP);
+    }
+
+}