From 63bbbb764d5e430d32ed74824a9631210090f586 Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 13:53:51 +0100 Subject: [PATCH 1/6] Spring 2 upgrade --- .travis.yml | 2 +- database-migration/pom.xml | 14 +-- .../ExtractAnnotationFromVariantTest.java | 75 ++++++++------ dbsnp-importer/pom.xml | 37 +++++-- .../configuration/DBAdaptorConnector.java | 85 ++++++++++++++++ .../ImportVariantsStepConfiguration.java | 18 ++-- .../mongo/MongoConfiguration.java | 28 ++++-- .../mongo/SpringDataMongoDbProperties.java | 98 +++++++++++++++++++ .../dbsnpimporter/io/readers/BatchReader.java | 6 +- .../processors/RenormalizationProcessor.java | 2 +- .../src/main/resources/application.properties | 16 +-- .../ImportSamplesStepConfigurationTest.java | 59 +++++++---- .../ImportVariantsJobConfigurationTest.java | 62 ++++++++---- .../ImportVariantsStepConfigurationTest.java | 58 +++++++---- ...antsStepEvaSubmittedConfigurationTest.java | 57 +++++++---- .../configuration/MongoTestConfiguration.java | 6 +- .../application-eva-submitted.properties | 8 ++ .../{ => properties}/application.properties | 10 +- pom.xml | 16 ++- vcf-dumper/vcf-dumper-lib/pom.xml | 35 ++++--- .../vcfdump/VariantExporterController.java | 5 +- .../VariantToVariantContextConverter.java | 4 +- .../configuration/DBAdaptorConnector.java | 46 +++++---- .../SpringDataMongoDbProperties.java | 22 ++++- .../src/main/resources/eva.properties | 15 +-- .../MongoRepositoryTestConfiguration.java | 19 +++- .../VariantExporterControllerTest.java | 8 +- .../ebi/eva/vcfdump/VariantExporterTest.java | 4 +- .../VariantToVariantContextConverterTest.java | 9 +- .../{ => properties}/evaTest.properties | 12 ++- vcf-dumper/vcf-dumper-ws/pom.xml | 20 +++- .../ebi/eva/vcfdump/server/Application.java | 2 +- .../configuration/MongoConfiguration.java | 14 +++ .../configuration/MultiMongoDbFactory.java | 4 +- .../MultiMongoFactoryConfiguration.java | 3 + .../server/rest/HtsgetVcfController.java | 2 +- .../server/rest/VcfDumperController.java | 2 +- .../src/main/resources/application.properties | 26 +++-- .../server/rest/VcfDumperIntegrationTest.java | 2 + .../configuration/MongoTestConfiguration.java | 5 - .../src/test/resources/application.properties | 11 --- .../test/resources/properties/test.properties | 14 +++ 42 files changed, 687 insertions(+), 254 deletions(-) create mode 100644 dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java create mode 100644 dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java rename dbsnp-importer/src/test/resources/{ => properties}/application-eva-submitted.properties (70%) rename dbsnp-importer/src/test/resources/{ => properties}/application.properties (70%) rename vcf-dumper/{vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server => vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump}/configuration/DBAdaptorConnector.java (76%) rename vcf-dumper/{vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server => vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump}/configuration/SpringDataMongoDbProperties.java (80%) rename vcf-dumper/vcf-dumper-lib/src/test/resources/{ => properties}/evaTest.properties (58%) delete mode 100644 vcf-dumper/vcf-dumper-ws/src/test/resources/application.properties create mode 100644 vcf-dumper/vcf-dumper-ws/src/test/resources/properties/test.properties diff --git a/.travis.yml b/.travis.yml index ec42a028..a66def2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ jdk: - openjdk8 env: - - OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build MONGODB_VERSION=3.0.4 + - OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build MONGODB_VERSION=4.0.18 install: - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-$MONGODB_VERSION.tgz diff --git a/database-migration/pom.xml b/database-migration/pom.xml index a81b2baf..78db72c1 100644 --- a/database-migration/pom.xml +++ b/database-migration/pom.xml @@ -44,19 +44,7 @@ org.mongodb mongo-java-driver - 3.4.0 - - - com.github.fakemongo - fongo - 2.1.0 - test - - - com.fasterxml.jackson.core - jackson-databind - - + 3.8.2 com.fasterxml.jackson.core diff --git a/database-migration/src/test/java/uk/ac/ebi/eva/dbmigration/mongodb/ExtractAnnotationFromVariantTest.java b/database-migration/src/test/java/uk/ac/ebi/eva/dbmigration/mongodb/ExtractAnnotationFromVariantTest.java index 0403a699..6e14946b 100644 --- a/database-migration/src/test/java/uk/ac/ebi/eva/dbmigration/mongodb/ExtractAnnotationFromVariantTest.java +++ b/database-migration/src/test/java/uk/ac/ebi/eva/dbmigration/mongodb/ExtractAnnotationFromVariantTest.java @@ -15,13 +15,14 @@ */ package uk.ac.ebi.eva.dbmigration.mongodb; -import com.github.fakemongo.Fongo; -import com.github.fakemongo.FongoException; +import com.mongodb.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.IndexOptions; + import org.bson.Document; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -30,9 +31,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static junit.framework.TestCase.assertNull; import static org.junit.Assert.assertEquals; @@ -87,18 +91,41 @@ public class ExtractAnnotationFromVariantTest { private ExtractAnnotationFromVariant extractAnnotationFromVariant; + private MongoClient mongoClient; + + private static final Map databaseMap = Stream.of(new String[][]{ + {"VARIANT_WITHOUT_ANNOTATION", "variantWithoutAnnotation"}, + {"VARIANT_WITH_ANNOTATION", "variantWithAnnotation"}, + {"DB_FOR_ANNOTATION_METADATA_CHECK", "DBForAnnotationMetadataCheck"}, + {"DB_FOR_INDEXES_CHECK", "DBForIndexesCheck"}, + {"DEFAULT_ANNOTATION", "defaultAnnotation"} + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + @Rule public final ExpectedException exception = ExpectedException.none(); @Before public void setUp() throws Exception { extractAnnotationFromVariant = new ExtractAnnotationFromVariant(); + mongoClient = new MongoClient(); + } + + @After + public void tearDown() throws Exception { + List mongoDatabases = new ArrayList<>(); + mongoClient.listDatabaseNames().iterator().forEachRemaining(mongoDatabases::add); + for(String usedDatabaseInTest: databaseMap.values()) { + if(mongoDatabases.contains(usedDatabaseInTest)) { + mongoClient.dropDatabase(usedDatabaseInTest); + } + } + mongoClient.close(); } @Test public void variantWithoutAnnotationShouldNotChange() { // given - String dbName = "variantWithoutAnnotation"; + String dbName = databaseMap.get("VARIANT_WITHOUT_ANNOTATION"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -112,7 +139,7 @@ public void variantWithoutAnnotationShouldNotChange() { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase database = new Fongo("testServer").getDatabase(dbName); + MongoDatabase database = mongoClient.getDatabase(dbName); MongoCollection variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME); MongoCollection annotationCollection = database.getCollection(ANNOTATION_COLLECTION_NAME); @@ -141,7 +168,7 @@ public void variantWithoutAnnotationShouldNotChange() { @Test public void variantWithAnnotationShouldMigrate() { // given - String dbName = "variantWithAnnotation"; + String dbName = databaseMap.get("VARIANT_WITH_ANNOTATION"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -155,7 +182,7 @@ public void variantWithAnnotationShouldMigrate() { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName); + MongoDatabase database = mongoClient.getDatabase(dbName); MongoCollection variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME); MongoCollection annotationCollection = database.getCollection(ANNOTATION_COLLECTION_NAME); @@ -163,7 +190,7 @@ public void variantWithAnnotationShouldMigrate() { variantsCollection.insertOne(variantWithAnnot); Document originalVariant = variantsCollection.find().first(); - Document originalAnnotField = (Document) originalVariant.get(ANNOT_FIELD); + Document originalAnnotField = (Document)originalVariant.get(ANNOT_FIELD); createLegacyIndexes(variantsCollection); @@ -189,7 +216,7 @@ public void variantWithAnnotationShouldMigrate() { @Test public void variantWithAnnotationShouldKeepSomeFields() { // given - String dbName = "variantWithAnnotation"; + String dbName = databaseMap.get("VARIANT_WITH_ANNOTATION"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -203,7 +230,7 @@ public void variantWithAnnotationShouldKeepSomeFields() { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName); + MongoDatabase database = mongoClient.getDatabase(dbName); MongoCollection variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME); Document variantWithAnnot = Document.parse(VariantData.VARIANT_WITH_ANNOT_2); @@ -302,7 +329,7 @@ private List computePolyphen(Document originalAnnotField) { @Test public void metadataShouldBeUpdated() throws Exception { // given - String dbName = "DBForAnnotationMetadataCheck"; + String dbName = databaseMap.get("DB_FOR_ANNOTATION_METADATA_CHECK"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -316,7 +343,7 @@ public void metadataShouldBeUpdated() throws Exception { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName); + MongoDatabase database = mongoClient.getDatabase(dbName); MongoCollection variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME); MongoCollection annotationMetadataCollection = database.getCollection( ANNOTATION_METADATA_COLLECTION_NAME); @@ -337,7 +364,7 @@ public void metadataShouldBeUpdated() throws Exception { @Test public void indexesShouldBeCreated() throws Exception { // given - String dbName = "DBForIndexesCheck"; + String dbName = databaseMap.get("DB_FOR_INDEXES_CHECK"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -351,7 +378,7 @@ public void indexesShouldBeCreated() throws Exception { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName); + MongoDatabase database = mongoClient.getDatabase(dbName); MongoCollection variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME); MongoCollection annotationsCollection = database.getCollection(ANNOTATION_COLLECTION_NAME); @@ -362,14 +389,14 @@ public void indexesShouldBeCreated() throws Exception { // when extractAnnotationFromVariant.migrateAnnotation(database); -// extractAnnotationFromVariant.dropIndexes(database); // note we didn't drop the indexes, as fongo doesn't support that + extractAnnotationFromVariant.dropIndexes(database); extractAnnotationFromVariant.reduceAnnotationFromVariants(database); extractAnnotationFromVariant.updateAnnotationMetadata(database); extractAnnotationFromVariant.createIndexes(database); // then ArrayList variantsIndexes = variantsCollection.listIndexes().into(new ArrayList<>()); - assertEquals(8, variantsIndexes.size()); // note we didn't drop the indexes, as fongo doesn't support that + assertEquals(6, variantsIndexes.size()); assertIndexNameExists(variantsIndexes, ANNOT_FIELD + "." + SO_FIELD + "_1"); assertIndexNameExists(variantsIndexes, ANNOT_FIELD + "." + XREFS_FIELD + "_1"); @@ -410,19 +437,9 @@ private void assertIndexNameExists(ArrayList variantsIndexes, String i assertTrue(indexFound); } - @Test - public void testFakemongoFailsToDropIndexes() throws Exception { - MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase("variantWithoutAnnotation"); - MongoCollection variantsCollection = mongoDatabase.getCollection(VARIANT_COLLECTION_NAME); - variantsCollection.createIndex(new Document("annot.ct.so", 1), new IndexOptions().background(true)); - - exception.expect(FongoException.class); - variantsCollection.dropIndex("annot.ct.so_1"); - } - @Test public void testDefaultAnnotationVersion() throws Exception { - String dbName = "defaultAnnotation"; + String dbName = databaseMap.get("DEFAULT_ANNOTATION"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -436,7 +453,7 @@ public void testDefaultAnnotationVersion() throws Exception { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase(dbName); + MongoDatabase mongoDatabase = mongoClient.getDatabase(dbName); MongoCollection collection = mongoDatabase.getCollection(ANNOTATION_METADATA_COLLECTION_NAME); collection.insertOne(buildAnnotationMetadataDocument("79", "78")); @@ -457,7 +474,7 @@ public void testDefaultAnnotationVersion() throws Exception { @Test public void testDefaultAnnotationVersionNotFound() throws Exception { - String dbName = "defaultAnnotation"; + String dbName = databaseMap.get("DEFAULT_ANNOTATION"); Properties properties = new Properties(); properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION); @@ -471,7 +488,7 @@ public void testDefaultAnnotationVersionNotFound() throws Exception { databaseParameters.load(properties); ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters); - MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase(dbName); + MongoDatabase mongoDatabase = mongoClient.getDatabase(dbName); MongoCollection collection = mongoDatabase.getCollection(ANNOTATION_METADATA_COLLECTION_NAME); collection.insertOne(buildAnnotationMetadataDocument("79", "78")); diff --git a/dbsnp-importer/pom.xml b/dbsnp-importer/pom.xml index c4862752..334a9873 100644 --- a/dbsnp-importer/pom.xml +++ b/dbsnp-importer/pom.xml @@ -21,6 +21,7 @@ UTF-8 UTF-8 1.8 + localhost @@ -46,7 +47,7 @@ org.hsqldb hsqldb - 2.3.4 + 2.4.1 test @@ -63,18 +64,12 @@ uk.ac.ebi.eva variation-commons-core - 0.5-SNAPSHOT + ${variation-commons.version} uk.ac.ebi.eva variation-commons-mongodb - 0.5-SNAPSHOT - - - com.github.fakemongo - fongo - test - 2.1.0 + ${variation-commons.version} com.fasterxml.jackson.core @@ -86,6 +81,17 @@ jackson-core 2.8.4 + + com.lordofthejars + nosqlunit-mongodb + test + 1.0.0 + + + javax.validation + validation-api + 2.0.1.Final + @@ -95,6 +101,19 @@ spring-boot-maven-plugin + + + src/test/resources/properties + true + + + src/test/resources/ + false + + src/test/resources/properties/* + + + diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java new file mode 100644 index 00000000..295ac79a --- /dev/null +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java @@ -0,0 +1,85 @@ +/* + * European Variation Archive (EVA) - Open-access database of all types of genetic + * variation data from all species + * + * Copyright 2020 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uk.ac.ebi.eva.dbsnpimporter.configuration; + +import com.mongodb.AuthenticationMechanism; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import com.mongodb.MongoCredential; +import com.mongodb.ReadPreference; +import com.mongodb.ServerAddress; + +import uk.ac.ebi.eva.dbsnpimporter.configuration.mongo.SpringDataMongoDbProperties; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DBAdaptorConnector { + + /** + * Get a MongoClient using the configuration (credentials) in a given Properties. + * + * @param springDataMongoDbProperties can have the next values: + * - eva.mongo.auth.db authentication database + * - eva.mongo.host comma-separated strings of colon-separated host and port strings: host_1:port_1,host_2:port_2 + * - eva.mongo.user + * - eva.mongo.passwd + * - eva.mongo.read-preference string, "secondaryPreferred" if unspecified. one of: + * [primary, primaryPreferred, secondary, secondaryPreferred, nearest] + * @return MongoClient with given credentials + * @throws UnknownHostException + */ + public static MongoClient getMongoClient( + SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { + + String host = springDataMongoDbProperties.getHost(); + int port = springDataMongoDbProperties.getPort(); + port = (port == 0 ? 27017:port); + List servers = new ArrayList<>(); + servers.add(new ServerAddress(host, port)); + + String readPreference = springDataMongoDbProperties.getReadPreference(); + readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference; + + MongoClientOptions options = MongoClientOptions.builder() + .readPreference(ReadPreference.valueOf(readPreference)) + .build(); + + List mongoCredentialList = new ArrayList<>(); + String authenticationDb = springDataMongoDbProperties.getAuthenticationDatabase(); + if (authenticationDb != null && !authenticationDb.isEmpty()) { + MongoCredential mongoCredential = MongoCredential.createCredential( + springDataMongoDbProperties.getUsername(), + authenticationDb, + springDataMongoDbProperties.getPassword().toCharArray()); + String authenticationMechanism = springDataMongoDbProperties.getAuthenticationMechanism(); + if (authenticationMechanism == null) { + return new MongoClient(servers, options); + } + mongoCredential = mongoCredential.withMechanism( + AuthenticationMechanism.fromMechanismName(authenticationMechanism)); + mongoCredentialList = Collections.singletonList(mongoCredential); + } + + return new MongoClient(servers, mongoCredentialList, options); + } +} diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfiguration.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfiguration.java index 562e36c0..2618d74d 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfiguration.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfiguration.java @@ -25,6 +25,7 @@ import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.core.listener.StepListenerSupport; +import org.springframework.batch.core.step.builder.SimpleStepBuilder; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; @@ -88,17 +89,18 @@ public Step importVariantsStep(StepBuilderFactory stepBuilderFactory, SimpleCompletionPolicy chunkSizeCompletionPolicy) { logger.debug("Building '" + IMPORT_VARIANTS_STEP + "'"); - return stepBuilderFactory.get(IMPORT_VARIANTS_STEP) + SimpleStepBuilder builder = stepBuilderFactory.get(IMPORT_VARIANTS_STEP) .chunk(chunkSizeCompletionPolicy) .reader(reader) .processor(processor) - .writer(writer) - .listener((StepExecutionListener) listenerLogger) - .listener((ChunkListener) listenerLogger) - .listener((ItemReadListener) listenerLogger) - .listener((ItemWriteListener) listenerLogger) - .listener(assemblyCheckStepListener) - .build(); + .writer(writer); + builder.listener((StepExecutionListener) listenerLogger); + builder.listener((ChunkListener) listenerLogger); + builder.listener((ItemReadListener) listenerLogger); + builder.listener((ItemWriteListener) listenerLogger); + builder.listener(assemblyCheckStepListener); + + return builder.build(); } } diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/MongoConfiguration.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/MongoConfiguration.java index 557b205d..de0d5e2e 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/MongoConfiguration.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/MongoConfiguration.java @@ -15,28 +15,42 @@ */ package uk.ac.ebi.eva.dbsnpimporter.configuration.mongo; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import com.mongodb.MongoClient; + +import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.MongoDbFactory; -import org.springframework.data.mongodb.core.convert.CustomConversions; +import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.convert.DbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.MappingMongoConverter; -import org.springframework.data.mongodb.core.convert.MongoConverter; import org.springframework.data.mongodb.core.mapping.MongoMappingContext; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; + +import uk.ac.ebi.eva.dbsnpimporter.configuration.DBAdaptorConnector; + +import java.net.UnknownHostException; @Configuration +@EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.commons.mongodb.repositories") +@EntityScan(basePackages = "uk.ac.ebi.eva.commons.mongodb.services") +@EnableMongoAuditing +@Import(SpringDataMongoDbProperties.class) public class MongoConfiguration { @Bean - public MappingMongoConverter mappingMongoConverter(MongoDbFactory factory, MongoMappingContext context, - BeanFactory beanFactory, CustomConversions conversions) { + public MappingMongoConverter mappingMongoConverter(MongoDbFactory factory, MongoMappingContext context) { DbRefResolver dbRefResolver = new DefaultDbRefResolver(factory); MappingMongoConverter mappingConverter = new MappingMongoConverter(dbRefResolver, context); - mappingConverter.setCustomConversions(conversions); mappingConverter.setMapKeyDotReplacement("£"); return mappingConverter; } + + @Bean + public MongoClient mongoClient( + SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { + return DBAdaptorConnector.getMongoClient(springDataMongoDbProperties); + } } diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java new file mode 100644 index 00000000..d73bbbb6 --- /dev/null +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java @@ -0,0 +1,98 @@ +/* + * Copyright 2020 EMBL - European Bioinformatics Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.ac.ebi.eva.dbsnpimporter.configuration.mongo; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import javax.validation.constraints.Size; + + +@ConfigurationProperties(ignoreUnknownFields = false, prefix = "spring.data.mongodb") +@Component +public class SpringDataMongoDbProperties { + + @Size(min = 1) + private String host; + + private int port; + + private String authenticationDatabase; + + private String username; + + private String password; + + private String readPreference; + + private String authenticationMechanism; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getAuthenticationDatabase() { + return authenticationDatabase; + } + + public void setAuthenticationDatabase(String authenticationDatabase) { + this.authenticationDatabase = authenticationDatabase; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getReadPreference() { + return readPreference; + } + + public void setReadPreference(String readPreference) { + this.readPreference = readPreference; + } + + public String getAuthenticationMechanism() { + return authenticationMechanism; + } + + public void setAuthenticationMechanism(String authenticationMechanism) { + this.authenticationMechanism = authenticationMechanism; + } +} diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/io/readers/BatchReader.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/io/readers/BatchReader.java index a52f8c7a..bf137dfa 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/io/readers/BatchReader.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/io/readers/BatchReader.java @@ -140,8 +140,10 @@ public void open(ExecutionContext executionContext) throws ItemStreamException { @Override public void close() throws ItemStreamException { - batchReader.close(); - samplesReader.close(); + if (this.getDataSource() != null) { + batchReader.close(); + samplesReader.close(); + } } @Override diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/jobs/steps/processors/RenormalizationProcessor.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/jobs/steps/processors/RenormalizationProcessor.java index ca47b5fd..1ebd025b 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/jobs/steps/processors/RenormalizationProcessor.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/jobs/steps/processors/RenormalizationProcessor.java @@ -78,7 +78,7 @@ public IVariant process(IVariant variant) throws Exception { */ private boolean isAmbiguous(IVariant variant) { try { - boolean isIndel = variant.getType() == VariantType.INDEL; + boolean isIndel = (variant.getType() == VariantType.INS || variant.getType() == VariantType.DEL); boolean oneAlleleIsEmpty = variant.getReference().isEmpty() ^ variant.getAlternate().isEmpty(); return isIndel && oneAlleleIsEmpty && areContextAndLastNucleotideEqual(variant); } catch (Exception e) { diff --git a/dbsnp-importer/src/main/resources/application.properties b/dbsnp-importer/src/main/resources/application.properties index ae8b50af..40c102ff 100644 --- a/dbsnp-importer/src/main/resources/application.properties +++ b/dbsnp-importer/src/main/resources/application.properties @@ -1,14 +1,16 @@ dbsnp.datasource.driver-class-name=org.postgresql.Driver -dbsnp.datasource.url=@eva.evapro.url@?currentSchema=@eva.dbsnp.schema@ -dbsnp.datasource.username=@eva.evapro.username@ -dbsnp.datasource.password=@eva.evapro.password@ +dbsnp.datasource.url=|eva.evapro.url|?currentSchema=|eva.dbsnp.schema| +dbsnp.datasource.username=|eva.evapro.username| +dbsnp.datasource.password=|eva.evapro.password| spring.jpa.generate-ddl=false ## Job repository database # See JobRepositoryConfiguration, use maven profiles to put the next properties in the environment: -spring.datasource.driver-class-name=@eva.jobrepository.driver-class-name@ -spring.datasource.url=@eva.jobrepository.url@ -spring.datasource.username=@eva.jobrepository.username@ -spring.datasource.password=@eva.jobrepository.password@ +spring.datasource.driver-class-name=|eva.jobrepository.driver-class-name| +spring.datasource.url=|eva.jobrepository.url| +spring.datasource.username=|eva.jobrepository.username| +spring.datasource.password=|eva.jobrepository.password| +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true diff --git a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportSamplesStepConfigurationTest.java b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportSamplesStepConfigurationTest.java index c4f4bbc7..10dfe9ac 100644 --- a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportSamplesStepConfigurationTest.java +++ b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportSamplesStepConfigurationTest.java @@ -15,17 +15,22 @@ */ package uk.ac.ebi.eva.dbsnpimporter.configuration; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; + +import org.bson.Document; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -33,6 +38,7 @@ import org.springframework.test.context.junit4.SpringRunner; import uk.ac.ebi.eva.commons.mongodb.entities.VariantSourceMongo; +import uk.ac.ebi.eva.dbsnpimporter.configuration.mongo.MongoConfiguration; import uk.ac.ebi.eva.dbsnpimporter.jobs.steps.processors.DbsnpBatchToVariantSourceProcessor; import uk.ac.ebi.eva.dbsnpimporter.parameters.Parameters; import uk.ac.ebi.eva.dbsnpimporter.test.DbsnpTestDatasource; @@ -40,7 +46,6 @@ import uk.ac.ebi.eva.dbsnpimporter.test.configuration.MongoTestConfiguration; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -49,8 +54,8 @@ @RunWith(SpringRunner.class) @TestPropertySource({"classpath:application.properties"}) @DirtiesContext -@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoTestConfiguration.class, - JobTestConfiguration.class}) +@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoConfiguration.class, + MongoTestConfiguration.class, JobTestConfiguration.class}) public class ImportSamplesStepConfigurationTest { private static final String DBSNP_BATCH_ID = "11825"; @@ -65,6 +70,8 @@ public class ImportSamplesStepConfigurationTest { private static final String THIRD_SAMPLE = "UNKSEX"; + private static final String TEST_DB = "test-db"; + @Autowired private DbsnpTestDatasource dbsnpTestDatasource; @@ -80,6 +87,22 @@ public class ImportSamplesStepConfigurationTest { @Autowired private MongoOperations mongoOperations; + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = MongoDbRule.MongoDbRuleBuilder.newMongoDbRule().defaultSpringMongoDb(TEST_DB); + + @Before + public void setUp() { + if (mongoOperations.collectionExists(parameters.getFilesCollection())) { + mongoOperations.dropCollection(parameters.getFilesCollection()); + } + if (mongoOperations.collectionExists(parameters.getVariantsCollection())) { + mongoOperations.dropCollection(parameters.getVariantsCollection()); + } + } + @Test public void loadSamples() throws Exception { assertEquals(0, mongoOperations.getCollection(parameters.getFilesCollection()).count()); @@ -87,28 +110,30 @@ public void loadSamples() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(ImportSamplesStepConfiguration.IMPORT_SAMPLES_STEP); assertCompleted(jobExecution); - DBCollection collection = mongoOperations.getCollection(parameters.getFilesCollection()); - List dbObjects = collection.find().toArray(); + MongoCollection collection = mongoOperations.getCollection(parameters.getFilesCollection()); + FindIterable dbObjects = collection.find(); - assertEquals(1, dbObjects.size()); + int[] documentCount = {0}; + dbObjects.iterator().forEachRemaining(document -> {documentCount[0] += 1;}); + assertEquals(1, documentCount[0]); - DBObject dbObject = dbObjects.get(0); + Document document = dbObjects.first(); - assertEquals(DBSNP_BATCH_NAME, dbObject.get(VariantSourceMongo.FILEID_FIELD)); - assertEquals(DBSNP_BATCH_HANDLE + " - " + DBSNP_BATCH_NAME, dbObject.get(VariantSourceMongo.FILENAME_FIELD)); - assertEquals(DBSNP_BATCH_NAME, dbObject.get(VariantSourceMongo.STUDYID_FIELD)); - assertEquals(DBSNP_BATCH_HANDLE + " - " + DBSNP_BATCH_NAME, dbObject.get(VariantSourceMongo.STUDYNAME_FIELD)); + assertEquals(DBSNP_BATCH_NAME, document.get(VariantSourceMongo.FILEID_FIELD)); + assertEquals(DBSNP_BATCH_HANDLE + " - " + DBSNP_BATCH_NAME, document.get(VariantSourceMongo.FILENAME_FIELD)); + assertEquals(DBSNP_BATCH_NAME, document.get(VariantSourceMongo.STUDYID_FIELD)); + assertEquals(DBSNP_BATCH_HANDLE + " - " + DBSNP_BATCH_NAME, document.get(VariantSourceMongo.STUDYNAME_FIELD)); Map expectedSamplesPosition = new HashMap<>(); expectedSamplesPosition.put(FIRST_SAMPLE, 0); expectedSamplesPosition.put(SECOND_SAMPLE, 1); expectedSamplesPosition.put(THIRD_SAMPLE, 2); - assertEquals(expectedSamplesPosition, dbObject.get(VariantSourceMongo.SAMPLES_FIELD)); + assertEquals(expectedSamplesPosition, document.get(VariantSourceMongo.SAMPLES_FIELD)); Map expectedMetadata = new HashMap<>(); expectedMetadata.put(DbsnpBatchToVariantSourceProcessor.DBSNP_BUILD_KEY, String.valueOf(DBSNP_BUILD)); - expectedMetadata.put(DbsnpBatchToVariantSourceProcessor.DBSNP_BATCH_KEY, String.valueOf(DBSNP_BATCH_ID)); - assertEquals(expectedMetadata, dbObject.get(VariantSourceMongo.METADATA_FIELD)); + expectedMetadata.put(DbsnpBatchToVariantSourceProcessor.DBSNP_BATCH_KEY, DBSNP_BATCH_ID); + assertEquals(expectedMetadata, document.get(VariantSourceMongo.METADATA_FIELD)); } public static void assertCompleted(JobExecution jobExecution) { diff --git a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsJobConfigurationTest.java b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsJobConfigurationTest.java index cd9b59f4..d1a5ed2d 100644 --- a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsJobConfigurationTest.java +++ b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsJobConfigurationTest.java @@ -15,11 +15,13 @@ */ package uk.ac.ebi.eva.dbsnpimporter.configuration; -import com.mongodb.BasicDBList; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.BasicDBObject; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import org.bson.Document; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.batch.core.BatchStatus; @@ -29,6 +31,7 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -40,6 +43,7 @@ import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantSourceEntryMongo; import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantStatisticsMongo; import uk.ac.ebi.eva.commons.mongodb.repositories.VariantRepository; +import uk.ac.ebi.eva.dbsnpimporter.configuration.mongo.MongoConfiguration; import uk.ac.ebi.eva.dbsnpimporter.jobs.steps.processors.AssemblyCheckFilterProcessor; import uk.ac.ebi.eva.dbsnpimporter.models.SubSnpCoreFields; import uk.ac.ebi.eva.dbsnpimporter.parameters.Parameters; @@ -60,14 +64,16 @@ @RunWith(SpringRunner.class) @TestPropertySource({"classpath:application.properties"}) -@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoTestConfiguration.class, - JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) +@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoConfiguration.class, + MongoTestConfiguration.class, JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) public class ImportVariantsJobConfigurationTest { private static final int BATCH_ID = 11825; private static final int NON_EXISTENT_BATCH_ID = 1; + private static final String TEST_DB = "test-db"; + @Autowired private DbsnpTestDatasource dbsnpTestDatasource; @@ -90,17 +96,29 @@ public class ImportVariantsJobConfigurationTest { @MockBean private AssemblyCheckFilterProcessor assemblyCheckerMock; + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = MongoDbRule.MongoDbRuleBuilder.newMongoDbRule().defaultSpringMongoDb(TEST_DB); + @Before public void setUp() throws Exception { // the assembly checker mock will filter out one variant when(this.assemblyCheckerMock.process(anyObject())).thenAnswer(invocationOnMock -> { - SubSnpCoreFields inputVariant = invocationOnMock.getArgumentAt(0, SubSnpCoreFields.class); + SubSnpCoreFields inputVariant = invocationOnMock.getArgument(0); if (inputVariant.getRsId() == 3136865) { return null; } else { return inputVariant; } }); + if (mongoOperations.collectionExists(parameters.getFilesCollection())) { + mongoOperations.dropCollection(parameters.getFilesCollection()); + } + if (mongoOperations.collectionExists(parameters.getVariantsCollection())) { + mongoOperations.dropCollection(parameters.getVariantsCollection()); + } assertEquals(0, mongoOperations.getCollection(parameters.getVariantsCollection()).count()); assertEquals(0, mongoOperations.getCollection(parameters.getFilesCollection()).count()); } @@ -114,17 +132,19 @@ public void loadVariantsAndFile() throws Exception { assertCompleted(jobExecution); - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); - List dbObjects = collection.find().toArray(); + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + FindIterable dbObjects = collection.find(); int totalSubsnps = 0; int totalSnps = 0; - for (DBObject dbObject : dbObjects) { - BasicDBList ids = (BasicDBList) dbObject.get("dbsnpIds"); - totalSnps += ids.stream().filter(o -> ((String) o).startsWith("rs")).count(); - totalSubsnps += ids.stream().filter(o -> ((String) o).startsWith("ss")).count(); + int numDocuments = 0; + for (Document dbObject : dbObjects) { + List ids = (List) dbObject.get("dbsnpIds"); + totalSnps += ids.stream().filter(o -> o.startsWith("rs")).count(); + totalSubsnps += ids.stream().filter(o -> o.startsWith("ss")).count(); + numDocuments += 1; } - assertEquals(8, dbObjects.size()); + assertEquals(8, numDocuments); assertEquals(8, totalSnps); assertEquals(11, totalSubsnps); @@ -147,9 +167,9 @@ public void doNotLoadFilefNoVariants() throws Exception { assertCompleted(jobExecution); assertSkipImportSamples(jobExecution); - DBCollection variantsCollection = mongoOperations.getCollection(parameters.getVariantsCollection()); + MongoCollection variantsCollection = mongoOperations.getCollection(parameters.getVariantsCollection()); assertEquals(0, variantsCollection.count()); - DBCollection filesCollection = mongoOperations.getCollection(parameters.getFilesCollection()); + MongoCollection filesCollection = mongoOperations.getCollection(parameters.getFilesCollection()); assertEquals(0, filesCollection.count()); } @@ -167,16 +187,16 @@ private void checkAnInsertion() throws Exception { } private void assertCoordinatesEquals(int subsnpId, long expectedStart, long expectedEnd) throws Exception { - DBObject document = getDocumentBySubsnp(subsnpId); + Document document = getDocumentBySubsnp(subsnpId); assertEquals(expectedStart, document.get("start")); assertEquals(expectedEnd, document.get("end")); } - private DBObject getDocumentBySubsnp(int subsnp) { - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + private Document getDocumentBySubsnp(int subsnp) { + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); String subsnpString = "ss" + subsnp; - List dbObjects = collection.find(new BasicDBObject("dbsnpIds", subsnpString)).toArray(); - return dbObjects.get(0); + FindIterable dbObjects = collection.find(new BasicDBObject("dbsnpIds", subsnpString)); + return dbObjects.first(); } private void checkGenotypes() { @@ -209,7 +229,7 @@ private void checkStatistics() { } private void checkVariantSource() { - DBCollection collection = mongoOperations.getCollection(parameters.getFilesCollection()); + MongoCollection collection = mongoOperations.getCollection(parameters.getFilesCollection()); assertEquals(1, collection.count()); } diff --git a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfigurationTest.java b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfigurationTest.java index e993a238..b94db6f6 100644 --- a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfigurationTest.java +++ b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepConfigurationTest.java @@ -15,22 +15,24 @@ */ package uk.ac.ebi.eva.dbsnpimporter.configuration; -import com.mongodb.BasicDBList; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.BasicDBObject; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import org.bson.Document; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobInstance; -import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -42,6 +44,7 @@ import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantSourceEntryMongo; import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantStatisticsMongo; import uk.ac.ebi.eva.commons.mongodb.repositories.VariantRepository; +import uk.ac.ebi.eva.dbsnpimporter.configuration.mongo.MongoConfiguration; import uk.ac.ebi.eva.dbsnpimporter.jobs.steps.processors.AssemblyCheckFilterProcessor; import uk.ac.ebi.eva.dbsnpimporter.models.SubSnpCoreFields; import uk.ac.ebi.eva.dbsnpimporter.parameters.Parameters; @@ -63,10 +66,12 @@ @RunWith(SpringRunner.class) @TestPropertySource({"classpath:application.properties"}) @DirtiesContext -@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoTestConfiguration.class, - JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) +@ContextConfiguration(classes = {ImportVariantsJobConfiguration.class, MongoConfiguration.class, + MongoTestConfiguration.class, JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) public class ImportVariantsStepConfigurationTest { + private static final String TEST_DB = "test-db"; + @Autowired private DbsnpTestDatasource dbsnpTestDatasource; @@ -89,17 +94,30 @@ public class ImportVariantsStepConfigurationTest { @MockBean private AssemblyCheckFilterProcessor assemblyCheckerMock; + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = MongoDbRule.MongoDbRuleBuilder.newMongoDbRule().defaultSpringMongoDb(TEST_DB); + @Before public void setUp() throws Exception { // the assembly checker mock will filter out one variant when(this.assemblyCheckerMock.process(anyObject())).thenAnswer(invocationOnMock -> { - SubSnpCoreFields inputVariant = invocationOnMock.getArgumentAt(0, SubSnpCoreFields.class); + SubSnpCoreFields inputVariant = invocationOnMock.getArgument(0); if (inputVariant.getRsId() == 3136865) { return null; } else { return inputVariant; } }); + + if (mongoOperations.collectionExists(parameters.getFilesCollection())) { + mongoOperations.dropCollection(parameters.getFilesCollection()); + } + if (mongoOperations.collectionExists(parameters.getVariantsCollection())) { + mongoOperations.dropCollection(parameters.getVariantsCollection()); + } } @Test @@ -111,17 +129,19 @@ public void loadVariants() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(ImportVariantsStepConfiguration.IMPORT_VARIANTS_STEP); assertCompleted(jobExecution); - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); - List dbObjects = collection.find().toArray(); + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + FindIterable documents = collection.find(); int totalSubsnps = 0; int totalSnps = 0; - for (DBObject dbObject : dbObjects) { - BasicDBList ids = (BasicDBList) dbObject.get("dbsnpIds"); - totalSnps += ids.stream().filter(o -> ((String) o).startsWith("rs")).count(); - totalSubsnps += ids.stream().filter(o -> ((String) o).startsWith("ss")).count(); + for (Document document : documents) { + List ids = (List) document.get("dbsnpIds"); + totalSnps += ids.stream().filter(o -> o.startsWith("rs")).count(); + totalSubsnps += ids.stream().filter(o -> o.startsWith("ss")).count(); } - assertEquals(8, dbObjects.size()); + int[] documentCount = {0}; + documents.spliterator().forEachRemaining(document -> documentCount[0]++); + assertEquals(8, documentCount[0]); assertEquals(8, totalSnps); assertEquals(11, totalSubsnps); @@ -146,16 +166,16 @@ private void checkAnInsertion() throws Exception { } private void assertCoordinatesEquals(int subsnpId, long expectedStart, long expectedEnd) throws Exception { - DBObject document = getDocumentBySubsnp(subsnpId); + Document document = getDocumentBySubsnp(subsnpId); assertEquals(expectedStart, document.get("start")); assertEquals(expectedEnd, document.get("end")); } - private DBObject getDocumentBySubsnp(int subsnp) { - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + private Document getDocumentBySubsnp(int subsnp) { + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); String subsnpString = "ss" + subsnp; - List dbObjects = collection.find(new BasicDBObject("dbsnpIds", subsnpString)).toArray(); - return dbObjects.get(0); + FindIterable documents = collection.find(new BasicDBObject("dbsnpIds", subsnpString)); + return documents.first(); } private void checkGenotypes() { diff --git a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepEvaSubmittedConfigurationTest.java b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepEvaSubmittedConfigurationTest.java index 450cbfb4..8246aea6 100644 --- a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepEvaSubmittedConfigurationTest.java +++ b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/configuration/ImportVariantsStepEvaSubmittedConfigurationTest.java @@ -15,11 +15,13 @@ */ package uk.ac.ebi.eva.dbsnpimporter.configuration; -import com.mongodb.BasicDBList; +import com.lordofthejars.nosqlunit.mongodb.MongoDbRule; import com.mongodb.BasicDBObject; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import org.bson.Document; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.batch.core.BatchStatus; @@ -30,6 +32,7 @@ import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; @@ -41,6 +44,7 @@ import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantSourceEntryMongo; import uk.ac.ebi.eva.commons.mongodb.entities.subdocuments.VariantStatisticsMongo; import uk.ac.ebi.eva.commons.mongodb.repositories.VariantRepository; +import uk.ac.ebi.eva.dbsnpimporter.configuration.mongo.MongoConfiguration; import uk.ac.ebi.eva.dbsnpimporter.jobs.steps.processors.AssemblyCheckFilterProcessor; import uk.ac.ebi.eva.dbsnpimporter.models.SubSnpCoreFields; import uk.ac.ebi.eva.dbsnpimporter.parameters.Parameters; @@ -58,10 +62,12 @@ @RunWith(SpringRunner.class) @TestPropertySource({"classpath:application-eva-submitted.properties"}) @DirtiesContext -@ContextConfiguration(classes = {ImportEvaSubmittedVariantsJobConfiguration.class, MongoTestConfiguration.class, - JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) +@ContextConfiguration(classes = {ImportEvaSubmittedVariantsJobConfiguration.class, MongoConfiguration.class, + MongoTestConfiguration.class, JobTestConfiguration.class, EvaRepositoriesConfiguration.class}) public class ImportVariantsStepEvaSubmittedConfigurationTest { + private static final String TEST_DB = "test-db"; + @Autowired private DbsnpTestDatasource dbsnpTestDatasource; @@ -84,17 +90,30 @@ public class ImportVariantsStepEvaSubmittedConfigurationTest { @MockBean private AssemblyCheckFilterProcessor assemblyCheckerMock; + @Autowired + private ApplicationContext applicationContext; + + @Rule + public MongoDbRule mongoDbRule = MongoDbRule.MongoDbRuleBuilder.newMongoDbRule().defaultSpringMongoDb(TEST_DB); + @Before public void setUp() throws Exception { // the assembly checker mock will filter out one variant when(this.assemblyCheckerMock.process(anyObject())).thenAnswer(invocationOnMock -> { - SubSnpCoreFields inputVariant = invocationOnMock.getArgumentAt(0, SubSnpCoreFields.class); + SubSnpCoreFields inputVariant = invocationOnMock.getArgument(0); if (inputVariant.getRsId() == 3136865) { return null; } else { return inputVariant; } }); + + if (mongoOperations.collectionExists(parameters.getFilesCollection())) { + mongoOperations.dropCollection(parameters.getFilesCollection()); + } + if (mongoOperations.collectionExists(parameters.getVariantsCollection())) { + mongoOperations.dropCollection(parameters.getVariantsCollection()); + } } @Test @@ -106,17 +125,19 @@ public void loadVariants() throws Exception { JobExecution jobExecution = jobLauncherTestUtils.launchStep(ImportVariantsStepConfiguration.IMPORT_VARIANTS_STEP); assertCompleted(jobExecution); - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); - List dbObjects = collection.find().toArray(); + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + FindIterable documents = collection.find(); int totalSubsnps = 0; int totalSnps = 0; - for (DBObject dbObject : dbObjects) { - BasicDBList ids = (BasicDBList) dbObject.get("dbsnpIds"); - totalSnps += ids.stream().filter(o -> ((String) o).startsWith("rs")).count(); - totalSubsnps += ids.stream().filter(o -> ((String) o).startsWith("ss")).count(); + for (Document document : documents) { + List ids = (List) document.get("dbsnpIds"); + totalSnps += ids.stream().filter(o -> o.startsWith("rs")).count(); + totalSubsnps += ids.stream().filter(o -> o.startsWith("ss")).count(); } - assertEquals(8, dbObjects.size()); + int[] documentCount = {0}; + documents.spliterator().forEachRemaining(document -> documentCount[0]++); + assertEquals(8, documentCount[0]); assertEquals(8, totalSnps); assertEquals(11, totalSubsnps); @@ -141,16 +162,16 @@ private void checkAnInsertion() throws Exception { } private void assertCoordinatesEquals(int subsnpId, long expectedStart, long expectedEnd) throws Exception { - DBObject document = getDocumentBySubsnp(subsnpId); + Document document = getDocumentBySubsnp(subsnpId); assertEquals(expectedStart, document.get("start")); assertEquals(expectedEnd, document.get("end")); } - private DBObject getDocumentBySubsnp(int subsnp) { - DBCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); + private Document getDocumentBySubsnp(int subsnp) { + MongoCollection collection = mongoOperations.getCollection(parameters.getVariantsCollection()); String subsnpString = "ss" + subsnp; - List dbObjects = collection.find(new BasicDBObject("dbsnpIds", subsnpString)).toArray(); - return dbObjects.get(0); + FindIterable documents = collection.find(new BasicDBObject("dbsnpIds", subsnpString)); + return documents.first(); } private void checkNoSourceEntries() { diff --git a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/test/configuration/MongoTestConfiguration.java b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/test/configuration/MongoTestConfiguration.java index d7056db0..f818984f 100644 --- a/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/test/configuration/MongoTestConfiguration.java +++ b/dbsnp-importer/src/test/java/uk/ac/ebi/eva/dbsnpimporter/test/configuration/MongoTestConfiguration.java @@ -18,8 +18,8 @@ */ package uk.ac.ebi.eva.dbsnpimporter.test.configuration; -import com.github.fakemongo.Fongo; import com.mongodb.MongoClient; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -60,10 +60,6 @@ public MappingMongoConverter mappingMongoConverter() throws IOException { mongoConverter.afterPropertiesSet(); return mongoConverter; } - @Bean - public MongoClient mongoClient() { - return new Fongo("defaultInstance").getMongo(); - } @Bean public MongoTemplate mongoTemplate(MongoDbFactory mongoDbFactory, diff --git a/dbsnp-importer/src/test/resources/application-eva-submitted.properties b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties similarity index 70% rename from dbsnp-importer/src/test/resources/application-eva-submitted.properties rename to dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties index 7de8972c..add0e181 100644 --- a/dbsnp-importer/src/test/resources/application-eva-submitted.properties +++ b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties @@ -16,7 +16,12 @@ spring.datasource.username=SA spring.datasource.password= +spring.data.mongodb.host=|eva.mongo.host.test| spring.data.mongodb.port=27017 +spring.data.mongodb.authentication-database=admin +spring.data.mongodb.username= +spring.data.mongodb.password= +spring.data.mongodb.read-preference=secondaryPreferred job=IMPORT_EVASUBMITTED_VARIANTS_JOB @@ -28,3 +33,6 @@ variants-collection=variants files-collection=files chunk-size=100 reference-fasta-file=src/test/resources/Gallus_gallus-5.0.test.fa + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true diff --git a/dbsnp-importer/src/test/resources/application.properties b/dbsnp-importer/src/test/resources/properties/application.properties similarity index 70% rename from dbsnp-importer/src/test/resources/application.properties rename to dbsnp-importer/src/test/resources/properties/application.properties index d18ea156..22a91e1b 100644 --- a/dbsnp-importer/src/test/resources/application.properties +++ b/dbsnp-importer/src/test/resources/properties/application.properties @@ -15,9 +15,12 @@ spring.datasource.url=jdbc:hsqldb:mem:jobRepositoryDb;sql.syntax_pgs=true;DB_CLO spring.datasource.username=SA spring.datasource.password= - +spring.data.mongodb.host=|eva.mongo.host.test| spring.data.mongodb.port=27017 - +spring.data.mongodb.authentication-database= +spring.data.mongodb.username= +spring.data.mongodb.password= +spring.data.mongodb.read-preference=secondaryPreferred job=IMPORT_VARIANTS_JOB dbsnp-build=150 @@ -28,3 +31,6 @@ variants-collection=variants files-collection=files chunk-size=100 reference-fasta-file=src/test/resources/Gallus_gallus-5.0.test.fa + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true diff --git a/pom.xml b/pom.xml index 91d6a047..54833b12 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.springframework.boot spring-boot-starter-parent - 1.5.10.RELEASE + 2.1.0.RELEASE @@ -23,7 +23,7 @@ 1.8 - 0.5 + 0.7.4-SNAPSHOT @@ -48,6 +48,18 @@ -Xmx1024m + + org.apache.maven.plugins + maven-resources-plugin + 2.6 + + UTF-8 + + + | + + + diff --git a/vcf-dumper/vcf-dumper-lib/pom.xml b/vcf-dumper/vcf-dumper-lib/pom.xml index f6a6a812..b657507d 100644 --- a/vcf-dumper/vcf-dumper-lib/pom.xml +++ b/vcf-dumper/vcf-dumper-lib/pom.xml @@ -27,24 +27,12 @@ com.lordofthejars nosqlunit-mongodb test - 0.14.0 - - - com.github.fakemongo - fongo - test - 2.1.0 - - - com.fasterxml.jackson.core - jackson-databind - - + 1.0.0 com.fasterxml.jackson.core jackson-databind - 2.8.11.3 + 2.9.7 test @@ -63,6 +51,11 @@ + + javax.validation + validation-api + 2.0.1.Final + @@ -84,7 +77,19 @@ true - + + + src/test/resources/properties + true + + + src/test/resources/ + false + + src/test/resources/properties/* + + + diff --git a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantExporterController.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantExporterController.java index ebef3294..446ac5c6 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantExporterController.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantExporterController.java @@ -38,6 +38,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,7 +93,7 @@ public VariantExporterController(String dbName, throws URISyntaxException { this(dbName, variantSourceService, variantService, studies, Collections.emptyList(), evaProperties, queryParameters, WINDOW_SIZE); this.outputStream = outputStream; - LocalDateTime now = LocalDateTime.now(); + String now = LocalDateTime.now().format(DateTimeFormatter.BASIC_ISO_DATE); outputFileName = dbName.replace("eva_", "") + "_exported_" + now + ".vcf"; } @@ -249,7 +250,7 @@ private VariantContextWriter getWriter() { } private VariantContextWriter buildVcfFileWriter() { - LocalDateTime now = LocalDateTime.now(); + String now = LocalDateTime.now().format(DateTimeFormatter.BASIC_ISO_DATE); String fileName = dbName + "_exported_" + now + ".vcf.gz"; outputFilePath = Paths.get(outputDir).resolve(fileName); diff --git a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverter.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverter.java index 64b3632e..b0e98475 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverter.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverter.java @@ -206,14 +206,14 @@ private VariantWithSamplesAndAnnotation addContextNucleotideToVariant(VariantWit // update variant start newVariant = new VariantWithSamplesAndAnnotation(variant.getChromosome(), variant.getStart() - 1, variant.getEnd(), contextNucleotide + variant.getReference(), - contextNucleotide + variant.getAlternate()); + contextNucleotide + variant.getAlternate(), null); newVariant.addSourceEntries(variant.getSourceEntries()); newVariant.setAnnotation(variant.getAnnotation()); } else { // update variant end newVariant = new VariantWithSamplesAndAnnotation(variant.getChromosome(), variant.getStart(), variant.getEnd() + 1, variant.getReference() + contextNucleotide, - variant.getAlternate() + contextNucleotide); + variant.getAlternate() + contextNucleotide, null); newVariant.addSourceEntries(variant.getSourceEntries()); newVariant.setAnnotation(variant.getAnnotation()); } diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/DBAdaptorConnector.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java similarity index 76% rename from vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/DBAdaptorConnector.java rename to vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java index 52f7b76e..3f9d015f 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/DBAdaptorConnector.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package uk.ac.ebi.eva.vcfdump.server.configuration; +package uk.ac.ebi.eva.vcfdump.configuration; import com.mongodb.*; import org.springframework.beans.factory.annotation.Autowired; @@ -30,9 +30,6 @@ @Component public class DBAdaptorConnector { - @Autowired - private DbCollectionsProperties dbCollectionsProperties; - @Autowired private SpringDataMongoDbProperties springDataMongoDbProperties; @@ -49,37 +46,38 @@ public class DBAdaptorConnector { * @return MongoClient with given credentials * @throws UnknownHostException */ - public static MongoClient getMongoClient(SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { + public static MongoClient getMongoClient( + SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { - String[] hosts = springDataMongoDbProperties.getHost().split(","); + String host = springDataMongoDbProperties.getHost(); + int port = springDataMongoDbProperties.getPort(); + port = (port == 0 ? 27017:port); List servers = new ArrayList<>(); + servers.add(new ServerAddress(host, port)); - // Get the list of hosts (optionally including the port number) - for (String host : hosts) { - String[] params = host.split(":"); - if (params.length > 1) { - servers.add(new ServerAddress(params[0], Integer.parseInt(params[1]))); - } else { - servers.add(new ServerAddress(params[0], 27017)); - } - } + String readPreference = springDataMongoDbProperties.getReadPreference(); + readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference; + + MongoClientOptions options = MongoClientOptions.builder() + .readPreference(ReadPreference.valueOf(readPreference)) + .build(); List mongoCredentialList = new ArrayList<>(); String authenticationDb = springDataMongoDbProperties.getAuthenticationDatabase(); if (authenticationDb != null && !authenticationDb.isEmpty()) { - mongoCredentialList = Collections.singletonList(MongoCredential.createCredential( + MongoCredential mongoCredential = MongoCredential.createCredential( springDataMongoDbProperties.getUsername(), authenticationDb, - springDataMongoDbProperties.getPassword().toCharArray())); + springDataMongoDbProperties.getPassword().toCharArray()); + String authenticationMechanism = springDataMongoDbProperties.getAuthenticationMechanism(); + if (authenticationMechanism == null) { + return new MongoClient(servers, options); + } + mongoCredential = mongoCredential.withMechanism( + AuthenticationMechanism.fromMechanismName(authenticationMechanism)); + mongoCredentialList = Collections.singletonList(mongoCredential); } - String readPreference = springDataMongoDbProperties.getReadPreference(); - readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference; - - MongoClientOptions options = MongoClientOptions.builder() - .readPreference(ReadPreference.valueOf(readPreference)) - .build(); - return new MongoClient(servers, mongoCredentialList, options); } diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/SpringDataMongoDbProperties.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java similarity index 80% rename from vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/SpringDataMongoDbProperties.java rename to vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java index 35eaeda5..ad911e92 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/SpringDataMongoDbProperties.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package uk.ac.ebi.eva.vcfdump.server.configuration; +package uk.ac.ebi.eva.vcfdump.configuration; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -27,6 +27,8 @@ public class SpringDataMongoDbProperties { @Size(min = 1) private String host; + private int port; + private String authenticationDatabase; private String username; @@ -35,6 +37,8 @@ public class SpringDataMongoDbProperties { private String readPreference; + private String authenticationMechanism; + public String getHost() { return host; } @@ -43,6 +47,14 @@ public void setHost(String host) { this.host = host; } + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + public String getAuthenticationDatabase() { return authenticationDatabase; } @@ -74,4 +86,12 @@ public String getReadPreference() { public void setReadPreference(String readPreference) { this.readPreference = readPreference; } + + public String getAuthenticationMechanism() { + return authenticationMechanism; + } + + public void setAuthenticationMechanism(String authenticationMechanism) { + this.authenticationMechanism = authenticationMechanism; + } } diff --git a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties index 270b6b04..7f6beb15 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties +++ b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties @@ -1,13 +1,16 @@ -eva.mongo.host=@eva.mongo.host@ -eva.mongo.user=@eva.mongo.user@ -eva.mongo.passwd=@eva.mongo.passwd@ -eva.mongo.auth.db=@eva.mongo.auth.db@ +eva.mongo.host=|eva.mongo.host| +eva.mongo.user=|eva.mongo.user| +eva.mongo.passwd=|eva.mongo.passwd| +eva.mongo.auth.db=|eva.mongo.auth.db| -eva.mongo.collections.variants=@eva.mongo.collections.variants@ -eva.mongo.collections.files=@eva.mongo.collections.files@ +eva.mongo.collections.variants=|eva.mongo.collections.variants| +eva.mongo.collections.files=|eva.mongo.collections.files| ##EVA REST url and version for querying chromosomes eva.rest.url=http://www.ebi.ac.uk/eva/webservices/rest/ eva.rest.version=v1 eva.htsget.blocksize=1000 + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true diff --git a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java index e1c2d4a3..11fbc4f6 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java +++ b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java @@ -18,23 +18,33 @@ */ package uk.ac.ebi.eva.vcfdump; -import com.github.fakemongo.Fongo; import com.mongodb.MongoClient; + import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.SimpleMongoDbFactory; import org.springframework.data.mongodb.core.convert.MappingMongoConverter; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; import org.springframework.util.Assert; import uk.ac.ebi.eva.commons.mongodb.configuration.EvaRepositoriesConfiguration; +import uk.ac.ebi.eva.vcfdump.configuration.DBAdaptorConnector; +import uk.ac.ebi.eva.vcfdump.configuration.SpringDataMongoDbProperties; + +import java.net.UnknownHostException; @Configuration -@Import({EvaRepositoriesConfiguration.class}) +@Import({EvaRepositoriesConfiguration.class, SpringDataMongoDbProperties.class}) @PropertySource({"classpath:evaTest.properties"}) +@EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.commons.mongodb.repositories") +@EntityScan(basePackages = "uk.ac.ebi.eva.commons.mongodb.services") +@EnableMongoAuditing public class MongoRepositoryTestConfiguration { @Bean @@ -72,8 +82,9 @@ public String mongoCollectionsFiles(@Value("${eva.mongo.collections.files:#{null } @Bean - public MongoClient mongoClient() { - return new Fongo("defaultInstance").getMongo(); + public MongoClient mongoClient( + SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { + return DBAdaptorConnector.getMongoClient(springDataMongoDbProperties); } @Bean diff --git a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterControllerTest.java b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterControllerTest.java index 6976e26c..c0fb1e78 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterControllerTest.java +++ b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterControllerTest.java @@ -32,6 +32,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.data.domain.PageRequest; +import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -77,6 +78,9 @@ "/db-dump/eva_hsapiens_grch37/variants_2_0.json"}) public class VariantExporterControllerTest { + @Autowired + private MongoOperations mongoOperations; + @Autowired private ApplicationContext applicationContext; @@ -122,7 +126,7 @@ public class VariantExporterControllerTest { public static void setUpClass() throws IOException{ evaTestProperties = new Properties(); - evaTestProperties.load(VariantExporterControllerTest.class.getResourceAsStream("/evaTest.properties")); + evaTestProperties.load(VariantExporterControllerTest.class.getResourceAsStream("/properties/evaTest.properties")); testOutputFiles = new ArrayList<>(); @@ -402,7 +406,7 @@ private List getVariantsFromOutputFile(String f if (line.charAt(0) != '#') { String[] fields = line.split("\t", 6); VariantWithSamplesAndAnnotation variant = new VariantWithSamplesAndAnnotation(fields[0], Integer.parseInt(fields[1]), Integer.parseInt(fields[1]), - fields[3], fields[4]); + fields[3], fields[4], null); //variant.setEnd(variant.getStart() + variant.getLength() - 1); if (variant.getAlternate().substring(0, 1).equals(variant.getReference().substring(0, 1))) { //variant.setAlternate(variant.getAlternate().substring(1)); diff --git a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterTest.java b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterTest.java index 017fac59..47ccdf01 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterTest.java +++ b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantExporterTest.java @@ -29,6 +29,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.data.domain.PageRequest; +import org.springframework.data.mongodb.core.MongoOperations; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import uk.ac.ebi.eva.commons.core.models.Region; @@ -89,7 +90,8 @@ public class VariantExporterTest { private static final String SHEEP_FILE_2_ID = "ERZ324596"; private static final int NUMBER_OF_SAMPLES_IN_SHEEP_FILES = 453; - + @Autowired + private MongoOperations mongoOperations; @Autowired private ApplicationContext applicationContext; diff --git a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverterTest.java b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverterTest.java index 9fb75d5b..d9dc6728 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverterTest.java +++ b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/VariantToVariantContextConverterTest.java @@ -28,6 +28,7 @@ import uk.ac.ebi.eva.commons.core.models.ConsequenceType; import uk.ac.ebi.eva.commons.core.models.IConsequenceType; import uk.ac.ebi.eva.commons.core.models.VariantSource; +import uk.ac.ebi.eva.commons.core.models.factories.VariantGenotypedVcfFactory; import uk.ac.ebi.eva.commons.core.models.factories.VariantVcfFactory; import uk.ac.ebi.eva.commons.core.models.pipeline.Variant; import uk.ac.ebi.eva.commons.core.models.pipeline.VariantSourceEntry; @@ -69,7 +70,7 @@ public class VariantToVariantContextConverterTest { @BeforeClass public static void setUpBeforeClass() { - variantFactory = new VariantVcfFactory(); + variantFactory = new VariantGenotypedVcfFactory(); // example samples list s1s6SampleList = new ArrayList<>(); @@ -323,7 +324,8 @@ public void severalNucleotidesDeletionInPosition1() { @Test public void twoStudiesNoConflictingNamesSingleVariant() { // create test variant, with two studies and samples with not conflicting names - VariantWithSamplesAndAnnotation variant = new VariantWithSamplesAndAnnotation(CHR_1, 1000, 1000, "T", "G"); + VariantWithSamplesAndAnnotation variant = new VariantWithSamplesAndAnnotation(CHR_1, 1000, 1000, "T", "G", + null); // initialize study 1 metadata and genotypes List source1SampleNames = Arrays.asList("SX_1", "SX_2", "SX_3", "SX_4"); @@ -365,7 +367,8 @@ private void addGenotypes(VariantSourceEntry variantSourceEntry, String ... geno @Test public void twoStudiesConflictingNamesSingleVariant() { // create test variant, with two studies and samples with not conflicting names - VariantWithSamplesAndAnnotation variant = new VariantWithSamplesAndAnnotation(CHR_1, 1000, 1000, "T", "G"); + VariantWithSamplesAndAnnotation variant = new VariantWithSamplesAndAnnotation(CHR_1, 1000, 1000, "T", "G", + null); // studies and samples names String study1 = "study_1"; diff --git a/vcf-dumper/vcf-dumper-lib/src/test/resources/evaTest.properties b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties similarity index 58% rename from vcf-dumper/vcf-dumper-lib/src/test/resources/evaTest.properties rename to vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties index 90325f72..0ab6f6a1 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/resources/evaTest.properties +++ b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties @@ -18,4 +18,14 @@ eva.mongo.collections.files=files eva.mongo.collections.annotation-metadata=annotationMetadata eva.mongo.collections.annotations=annotations eva.mongo.collections.features=features -eva.version= \ No newline at end of file +eva.version= + +spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.port=27017 +spring.data.mongodb.authentication-database= +spring.data.mongodb.username= +spring.data.mongodb.password= +spring.data.mongodb.read-preference=secondaryPreferred + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true diff --git a/vcf-dumper/vcf-dumper-ws/pom.xml b/vcf-dumper/vcf-dumper-ws/pom.xml index 8684c396..7e15912b 100644 --- a/vcf-dumper/vcf-dumper-ws/pom.xml +++ b/vcf-dumper/vcf-dumper-ws/pom.xml @@ -27,7 +27,7 @@ com.fasterxml.jackson.core jackson-databind - 2.8.11.3 + 2.9.7 uk.ac.ebi.eva @@ -48,6 +48,11 @@ + + localhost + ${maven.build.timestamp} + + vcf-dumper src/main/java @@ -65,6 +70,19 @@ + + + src/test/resources/properties + true + + + src/test/resources/ + false + + src/test/resources/properties/* + + + diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/Application.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/Application.java index f7f65b75..8d05eed2 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/Application.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/Application.java @@ -21,9 +21,9 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.web.support.SpringBootServletInitializer; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; @SpringBootApplication @EnableAutoConfiguration(exclude={DataSourceAutoConfiguration.class}) diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java index 186223b9..1beca5d8 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java @@ -15,6 +15,8 @@ */ package uk.ac.ebi.eva.vcfdump.server.configuration; +import com.mongodb.MongoClient; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -22,18 +24,24 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.data.mongodb.MongoDbFactory; +import org.springframework.data.mongodb.config.EnableMongoAuditing; import org.springframework.data.mongodb.core.convert.DbRefResolver; import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; import org.springframework.data.mongodb.core.convert.MappingMongoConverter; import org.springframework.data.mongodb.core.mapping.MongoMappingContext; import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import uk.ac.ebi.eva.vcfdump.configuration.DBAdaptorConnector; +import uk.ac.ebi.eva.vcfdump.configuration.SpringDataMongoDbProperties; + import java.io.IOException; +import java.net.UnknownHostException; @Configuration @Import(DbCollectionsProperties.class) @EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.commons.mongodb.repositories") @ComponentScan(basePackages = "uk.ac.ebi.eva.commons.mongodb.services") +@EnableMongoAuditing public class MongoConfiguration { @Autowired @@ -89,4 +97,10 @@ public MappingMongoConverter mappingMongoConverter() throws IOException { return mongoConverter; } + @Bean + public MongoClient mongoClient( + SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { + return DBAdaptorConnector.getMongoClient(springDataMongoDbProperties); + } + } diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoDbFactory.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoDbFactory.java index 9e03c9a6..71133826 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoDbFactory.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoDbFactory.java @@ -15,8 +15,8 @@ */ package uk.ac.ebi.eva.vcfdump.server.configuration; -import com.mongodb.DB; import com.mongodb.MongoClient; +import com.mongodb.client.MongoDatabase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.mongodb.core.SimpleMongoDbFactory; @@ -64,7 +64,7 @@ public static void clearDatabaseNameForCurrentThread() { } @Override - public DB getDb() { + public MongoDatabase getDb() { final String tlName = dbName.get(); final String dbToUse = (tlName != null ? tlName : this.defaultName); logger.trace("Acquiring database: " + dbToUse); diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoFactoryConfiguration.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoFactoryConfiguration.java index d20c1150..c182ca19 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoFactoryConfiguration.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MultiMongoFactoryConfiguration.java @@ -24,6 +24,9 @@ import org.springframework.context.annotation.Profile; import org.springframework.data.mongodb.MongoDbFactory; +import uk.ac.ebi.eva.vcfdump.configuration.DBAdaptorConnector; +import uk.ac.ebi.eva.vcfdump.configuration.SpringDataMongoDbProperties; + import java.io.IOException; @Configuration diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/HtsgetVcfController.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/HtsgetVcfController.java index f8562080..066a3c1b 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/HtsgetVcfController.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/HtsgetVcfController.java @@ -32,7 +32,7 @@ import uk.ac.ebi.eva.commons.mongodb.services.VariantWithSamplesAndAnnotationsService; import uk.ac.ebi.eva.vcfdump.QueryParams; import uk.ac.ebi.eva.vcfdump.VariantExporterController; -import uk.ac.ebi.eva.vcfdump.server.configuration.DBAdaptorConnector; +import uk.ac.ebi.eva.vcfdump.configuration.DBAdaptorConnector; import uk.ac.ebi.eva.vcfdump.server.configuration.MultiMongoDbFactory; import uk.ac.ebi.eva.vcfdump.server.model.HtsGetError; import uk.ac.ebi.eva.vcfdump.server.model.HtsGetResponse; diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperController.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperController.java index 85929c60..cbc2b410 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperController.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperController.java @@ -32,7 +32,7 @@ import uk.ac.ebi.eva.commons.mongodb.services.VariantWithSamplesAndAnnotationsService; import uk.ac.ebi.eva.vcfdump.QueryParams; import uk.ac.ebi.eva.vcfdump.VariantExporterController; -import uk.ac.ebi.eva.vcfdump.server.configuration.DBAdaptorConnector; +import uk.ac.ebi.eva.vcfdump.configuration.DBAdaptorConnector; import uk.ac.ebi.eva.vcfdump.server.configuration.MultiMongoDbFactory; import javax.servlet.http.HttpServletResponse; diff --git a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties index c5ca4ecb..81d2cc1d 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties +++ b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties @@ -5,14 +5,20 @@ springfox.documentation.swagger.v2.path=/api #spring.profiles.active=oauth2-security #security.oauth2.resource.user-info-uri = ... -spring.data.mongodb.host=@eva.mongo.host@ -spring.data.mongodb.authentication-database=@eva.mongo.auth.db@ -spring.data.mongodb.username=@eva.mongo.user@ -spring.data.mongodb.password=@eva.mongo.passwd@ -spring.data.mongodb.read-preference=@eva.mongo.read-preference@ +spring.data.mongodb.host=|eva.mongo.host| +spring.data.mongodb.port=27017 +spring.data.mongodb.authentication-database=|eva.mongo.auth.db| +spring.data.mongodb.username=|eva.mongo.user| +spring.data.mongodb.password=|eva.mongo.passwd| +spring.data.mongodb.read-preference=secondaryPreferred -db.collection-names.files=@eva.mongo.collections.files@ -db.collection-names.variants=@eva.mongo.collections.variants@ -db.collection-names.annotation-metadata=@eva.mongo.collections.annotation-metadata@ -db.collection-names.features=@eva.mongo.collections.features@ -db.collection-names.annotations=@eva.mongo.collections.annotations@ +db.collection-names.files=|eva.mongo.collections.files| +db.collection-names.variants=|eva.mongo.collections.variants| +db.collection-names.annotation-metadata=|eva.mongo.collections.annotation-metadata| +db.collection-names.features=|eva.mongo.collections.features| +db.collection-names.annotations=|eva.mongo.collections.annotations| + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true + +spring.jmx.default-domain=eva.ebi.ac.uk.|timestamp| diff --git a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperIntegrationTest.java b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperIntegrationTest.java index d0539661..9089dcbb 100644 --- a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperIntegrationTest.java +++ b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/rest/VcfDumperIntegrationTest.java @@ -37,6 +37,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import uk.ac.ebi.eva.vcfdump.server.model.HtsGetResponse; @@ -57,6 +58,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@TestPropertySource({"classpath:test.properties"}) public class VcfDumperIntegrationTest { private static final String EVA_ECABALLUS_20_DB = "eva_ecaballus_20"; diff --git a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java index be452a41..434fdb7d 100644 --- a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java +++ b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java @@ -32,11 +32,6 @@ @AutoConfigureDataMongo public class MongoTestConfiguration { - @Bean - public MongoClient mongoClient() { - return new MongoClient("localhost"); - } - @Bean public MongoDbFactory mongoDbFactory(MongoClient mongoClient) { return new SimpleMongoDbFactory(mongoClient, "vcf-dumper-db-test"); diff --git a/vcf-dumper/vcf-dumper-ws/src/test/resources/application.properties b/vcf-dumper/vcf-dumper-ws/src/test/resources/application.properties deleted file mode 100644 index d3bb65ff..00000000 --- a/vcf-dumper/vcf-dumper-ws/src/test/resources/application.properties +++ /dev/null @@ -1,11 +0,0 @@ -spring.data.mongodb.host=localhost -spring.data.mongodb.authentication-database= -spring.data.mongodb.username= -spring.data.mongodb.password= -spring.data.mongodb.read-preference= - -db.collection-names.files=files -db.collection-names.variants=variants -db.collection-names.annotation-metadata=annotationsMetadata -db.collection-names.annotations=annotations -db.collection-names.features=features \ No newline at end of file diff --git a/vcf-dumper/vcf-dumper-ws/src/test/resources/properties/test.properties b/vcf-dumper/vcf-dumper-ws/src/test/resources/properties/test.properties new file mode 100644 index 00000000..9d563e01 --- /dev/null +++ b/vcf-dumper/vcf-dumper-ws/src/test/resources/properties/test.properties @@ -0,0 +1,14 @@ +spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.authentication-database= +spring.data.mongodb.username= +spring.data.mongodb.password= +spring.data.mongodb.read-preference=secondaryPreferred + +db.collection-names.files=files +db.collection-names.variants=variants +db.collection-names.annotation-metadata=annotationsMetadata +db.collection-names.annotations=annotations +db.collection-names.features=features + +# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding +spring.main.allow-bean-definition-overriding=true From 83e8f5ac0786d6f1cc30b5d50a6c0af4d74bae0b Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 15:01:47 +0100 Subject: [PATCH 2/6] Avoid using separate port parameter --- dbsnp-importer/pom.xml | 2 +- .../configuration/DBAdaptorConnector.java | 15 +++++++++++---- .../mongo/SpringDataMongoDbProperties.java | 10 ---------- .../application-eva-submitted.properties | 1 - .../resources/properties/application.properties | 1 - .../vcfdump/configuration/DBAdaptorConnector.java | 15 +++++++++++---- .../SpringDataMongoDbProperties.java | 10 ---------- .../test/resources/properties/evaTest.properties | 1 - vcf-dumper/vcf-dumper-ws/pom.xml | 2 +- .../src/main/resources/application.properties | 1 - 10 files changed, 24 insertions(+), 34 deletions(-) diff --git a/dbsnp-importer/pom.xml b/dbsnp-importer/pom.xml index 334a9873..fdc4b00e 100644 --- a/dbsnp-importer/pom.xml +++ b/dbsnp-importer/pom.xml @@ -21,7 +21,7 @@ UTF-8 UTF-8 1.8 - localhost + localhost:27017 diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java index 295ac79a..f4ceb07c 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/DBAdaptorConnector.java @@ -51,11 +51,18 @@ public class DBAdaptorConnector { public static MongoClient getMongoClient( SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { - String host = springDataMongoDbProperties.getHost(); - int port = springDataMongoDbProperties.getPort(); - port = (port == 0 ? 27017:port); + String[] hosts = springDataMongoDbProperties.getHost().split(","); List servers = new ArrayList<>(); - servers.add(new ServerAddress(host, port)); + + // Get the list of hosts (optionally including the port number) + for (String host : hosts) { + String[] params = host.split(":"); + if (params.length > 1) { + servers.add(new ServerAddress(params[0], Integer.parseInt(params[1]))); + } else { + servers.add(new ServerAddress(params[0], 27017)); + } + } String readPreference = springDataMongoDbProperties.getReadPreference(); readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference; diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java index d73bbbb6..9adef680 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java @@ -28,8 +28,6 @@ public class SpringDataMongoDbProperties { @Size(min = 1) private String host; - private int port; - private String authenticationDatabase; private String username; @@ -48,14 +46,6 @@ public void setHost(String host) { this.host = host; } - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - public String getAuthenticationDatabase() { return authenticationDatabase; } diff --git a/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties index add0e181..dd76e004 100644 --- a/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties +++ b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties @@ -17,7 +17,6 @@ spring.datasource.password= spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database=admin spring.data.mongodb.username= spring.data.mongodb.password= diff --git a/dbsnp-importer/src/test/resources/properties/application.properties b/dbsnp-importer/src/test/resources/properties/application.properties index 22a91e1b..401ddc02 100644 --- a/dbsnp-importer/src/test/resources/properties/application.properties +++ b/dbsnp-importer/src/test/resources/properties/application.properties @@ -16,7 +16,6 @@ spring.datasource.username=SA spring.datasource.password= spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database= spring.data.mongodb.username= spring.data.mongodb.password= diff --git a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java index 3f9d015f..59d788c0 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/DBAdaptorConnector.java @@ -49,11 +49,18 @@ public class DBAdaptorConnector { public static MongoClient getMongoClient( SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException { - String host = springDataMongoDbProperties.getHost(); - int port = springDataMongoDbProperties.getPort(); - port = (port == 0 ? 27017:port); + String[] hosts = springDataMongoDbProperties.getHost().split(","); List servers = new ArrayList<>(); - servers.add(new ServerAddress(host, port)); + + // Get the list of hosts (optionally including the port number) + for (String host : hosts) { + String[] params = host.split(":"); + if (params.length > 1) { + servers.add(new ServerAddress(params[0], Integer.parseInt(params[1]))); + } else { + servers.add(new ServerAddress(params[0], 27017)); + } + } String readPreference = springDataMongoDbProperties.getReadPreference(); readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference; diff --git a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java index ad911e92..533d65d8 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java @@ -27,8 +27,6 @@ public class SpringDataMongoDbProperties { @Size(min = 1) private String host; - private int port; - private String authenticationDatabase; private String username; @@ -47,14 +45,6 @@ public void setHost(String host) { this.host = host; } - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - public String getAuthenticationDatabase() { return authenticationDatabase; } diff --git a/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties index 0ab6f6a1..7f52300f 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties +++ b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties @@ -21,7 +21,6 @@ eva.mongo.collections.features=features eva.version= spring.data.mongodb.host=|eva.mongo.host.test| -spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database= spring.data.mongodb.username= spring.data.mongodb.password= diff --git a/vcf-dumper/vcf-dumper-ws/pom.xml b/vcf-dumper/vcf-dumper-ws/pom.xml index 7e15912b..60427802 100644 --- a/vcf-dumper/vcf-dumper-ws/pom.xml +++ b/vcf-dumper/vcf-dumper-ws/pom.xml @@ -49,7 +49,7 @@ - localhost + localhost:27017 ${maven.build.timestamp} diff --git a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties index 81d2cc1d..ebd79367 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties +++ b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties @@ -6,7 +6,6 @@ springfox.documentation.swagger.v2.path=/api #security.oauth2.resource.user-info-uri = ... spring.data.mongodb.host=|eva.mongo.host| -spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database=|eva.mongo.auth.db| spring.data.mongodb.username=|eva.mongo.user| spring.data.mongodb.password=|eva.mongo.passwd| From b17b474aab205fa77b065f728aca5b0f74d65efc Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 15:51:08 +0100 Subject: [PATCH 3/6] Fix autowiring issues with SpringDataMongoDbProperties --- vcf-dumper/vcf-dumper-lib/pom.xml | 4 ++++ .../vcf-dumper-lib/src/main/resources/eva.properties | 9 +++++---- .../eva/vcfdump/MongoRepositoryTestConfiguration.java | 2 ++ .../src/test/resources/properties/evaTest.properties | 9 --------- .../vcfdump/server/configuration/MongoConfiguration.java | 2 ++ .../test/configuration/MongoTestConfiguration.java | 1 - 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/vcf-dumper/vcf-dumper-lib/pom.xml b/vcf-dumper/vcf-dumper-lib/pom.xml index b657507d..bb39c86e 100644 --- a/vcf-dumper/vcf-dumper-lib/pom.xml +++ b/vcf-dumper/vcf-dumper-lib/pom.xml @@ -92,4 +92,8 @@ + + localhost:27017 + + diff --git a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties index 7f6beb15..d5bb5bd8 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties +++ b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties @@ -1,7 +1,8 @@ -eva.mongo.host=|eva.mongo.host| -eva.mongo.user=|eva.mongo.user| -eva.mongo.passwd=|eva.mongo.passwd| -eva.mongo.auth.db=|eva.mongo.auth.db| +spring.data.mongodb.host=|eva.mongo.host| +spring.data.mongodb.authentication-database=|eva.mongo.auth.db| +spring.data.mongodb.username=|eva.mongo.user| +spring.data.mongodb.password=|eva.mongo.passwd| +spring.data.mongodb.read-preference=secondaryPreferred eva.mongo.collections.variants=|eva.mongo.collections.variants| eva.mongo.collections.files=|eva.mongo.collections.files| diff --git a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java index 11fbc4f6..344b69c3 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java +++ b/vcf-dumper/vcf-dumper-lib/src/test/java/uk/ac/ebi/eva/vcfdump/MongoRepositoryTestConfiguration.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @@ -45,6 +46,7 @@ @EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.commons.mongodb.repositories") @EntityScan(basePackages = "uk.ac.ebi.eva.commons.mongodb.services") @EnableMongoAuditing +@AutoConfigureDataMongo public class MongoRepositoryTestConfiguration { @Bean diff --git a/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties index 7f52300f..3525e3ec 100644 --- a/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties +++ b/vcf-dumper/vcf-dumper-lib/src/test/resources/properties/evaTest.properties @@ -1,18 +1,9 @@ -#eva.mongo.host=localhost:27017 -#eva.mongo.collections.variants=variants -#eva.mongo.collections.files=files - ##EVA REST url and version for querying chromosomes eva.rest.url= eva.rest.version=v1 eva.htsget.blocksize=1000 -eva.mongo.host=testhost -eva.mongo.user=testuser -eva.mongo.passwd=testpass -eva.mongo.auth.db=testdb -eva.mongo.read-preference= eva.mongo.collections.variants=variants eva.mongo.collections.files=files eva.mongo.collections.annotation-metadata=annotationMetadata diff --git a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java index 1beca5d8..dc41e4f0 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java +++ b/vcf-dumper/vcf-dumper-ws/src/main/java/uk/ac/ebi/eva/vcfdump/server/configuration/MongoConfiguration.java @@ -18,6 +18,7 @@ import com.mongodb.MongoClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -42,6 +43,7 @@ @EnableMongoRepositories(basePackages = "uk.ac.ebi.eva.commons.mongodb.repositories") @ComponentScan(basePackages = "uk.ac.ebi.eva.commons.mongodb.services") @EnableMongoAuditing +@AutoConfigureDataMongo public class MongoConfiguration { @Autowired diff --git a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java index 434fdb7d..ecbec820 100644 --- a/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java +++ b/vcf-dumper/vcf-dumper-ws/src/test/java/uk/ac/ebi/eva/vcfdump/server/test/configuration/MongoTestConfiguration.java @@ -29,7 +29,6 @@ @Configuration @Import({MongoConfiguration.class}) -@AutoConfigureDataMongo public class MongoTestConfiguration { @Bean From 5dd86dc808590114e4458903e7a76525bddd4fa4 Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 17:24:00 +0100 Subject: [PATCH 4/6] Fix autowiring issues with SpringDataMongoDbProperties --- .../mongo/SpringDataMongoDbProperties.java | 10 ++++++++++ .../properties/application-eva-submitted.properties | 1 + .../test/resources/properties/application.properties | 1 + .../configuration/SpringDataMongoDbProperties.java | 10 ++++++++++ 4 files changed, 22 insertions(+) diff --git a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java index 9adef680..d73bbbb6 100644 --- a/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java +++ b/dbsnp-importer/src/main/java/uk/ac/ebi/eva/dbsnpimporter/configuration/mongo/SpringDataMongoDbProperties.java @@ -28,6 +28,8 @@ public class SpringDataMongoDbProperties { @Size(min = 1) private String host; + private int port; + private String authenticationDatabase; private String username; @@ -46,6 +48,14 @@ public void setHost(String host) { this.host = host; } + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + public String getAuthenticationDatabase() { return authenticationDatabase; } diff --git a/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties index dd76e004..add0e181 100644 --- a/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties +++ b/dbsnp-importer/src/test/resources/properties/application-eva-submitted.properties @@ -17,6 +17,7 @@ spring.datasource.password= spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database=admin spring.data.mongodb.username= spring.data.mongodb.password= diff --git a/dbsnp-importer/src/test/resources/properties/application.properties b/dbsnp-importer/src/test/resources/properties/application.properties index 401ddc02..22a91e1b 100644 --- a/dbsnp-importer/src/test/resources/properties/application.properties +++ b/dbsnp-importer/src/test/resources/properties/application.properties @@ -16,6 +16,7 @@ spring.datasource.username=SA spring.datasource.password= spring.data.mongodb.host=|eva.mongo.host.test| +spring.data.mongodb.port=27017 spring.data.mongodb.authentication-database= spring.data.mongodb.username= spring.data.mongodb.password= diff --git a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java index 533d65d8..ad911e92 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java +++ b/vcf-dumper/vcf-dumper-lib/src/main/java/uk/ac/ebi/eva/vcfdump/configuration/SpringDataMongoDbProperties.java @@ -27,6 +27,8 @@ public class SpringDataMongoDbProperties { @Size(min = 1) private String host; + private int port; + private String authenticationDatabase; private String username; @@ -45,6 +47,14 @@ public void setHost(String host) { this.host = host; } + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + public String getAuthenticationDatabase() { return authenticationDatabase; } From 49af400682b9e26956c9c8dae691eaa3f28ea2c9 Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 20:16:37 +0100 Subject: [PATCH 5/6] Default authentication mechanism to SCRAM-SHA-1 --- vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties | 1 + .../vcf-dumper-ws/src/main/resources/application.properties | 1 + 2 files changed, 2 insertions(+) diff --git a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties index d5bb5bd8..262155c2 100644 --- a/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties +++ b/vcf-dumper/vcf-dumper-lib/src/main/resources/eva.properties @@ -3,6 +3,7 @@ spring.data.mongodb.authentication-database=|eva.mongo.auth.db| spring.data.mongodb.username=|eva.mongo.user| spring.data.mongodb.password=|eva.mongo.passwd| spring.data.mongodb.read-preference=secondaryPreferred +spring.data.mongodb.authentication-mechanism=SCRAM-SHA-1 eva.mongo.collections.variants=|eva.mongo.collections.variants| eva.mongo.collections.files=|eva.mongo.collections.files| diff --git a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties index ebd79367..5bfa0566 100644 --- a/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties +++ b/vcf-dumper/vcf-dumper-ws/src/main/resources/application.properties @@ -10,6 +10,7 @@ spring.data.mongodb.authentication-database=|eva.mongo.auth.db| spring.data.mongodb.username=|eva.mongo.user| spring.data.mongodb.password=|eva.mongo.passwd| spring.data.mongodb.read-preference=secondaryPreferred +spring.data.mongodb.authentication-mechanism=SCRAM-SHA-1 db.collection-names.files=|eva.mongo.collections.files| db.collection-names.variants=|eva.mongo.collections.variants| From e710e0c3cd1f0770b65a2d5a12c8d239438592e7 Mon Sep 17 00:00:00 2001 From: Sundar Venkataraman Date: Wed, 10 Jun 2020 23:59:23 +0100 Subject: [PATCH 6/6] Update POM version to a snapshot version --- database-migration/pom.xml | 4 ++-- dbsnp-importer/pom.xml | 4 ++-- pom.xml | 2 +- vcf-dumper/pom.xml | 4 ++-- vcf-dumper/vcf-dumper-cli/pom.xml | 4 ++-- vcf-dumper/vcf-dumper-lib/pom.xml | 2 +- vcf-dumper/vcf-dumper-ws/pom.xml | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/database-migration/pom.xml b/database-migration/pom.xml index 78db72c1..e39b8a49 100644 --- a/database-migration/pom.xml +++ b/database-migration/pom.xml @@ -5,13 +5,13 @@ 4.0.0 database-migration - 0.3.0 + 0.3.1-SNAPSHOT jar uk.ac.ebi.eva eva-tools - 0.3.0 + 0.3.1-SNAPSHOT diff --git a/dbsnp-importer/pom.xml b/dbsnp-importer/pom.xml index fdc4b00e..89a107a8 100644 --- a/dbsnp-importer/pom.xml +++ b/dbsnp-importer/pom.xml @@ -5,7 +5,7 @@ uk.ac.ebi.eva dbsnp-importer - 0.3.0 + 0.3.1-SNAPSHOT jar dbsnp-importer @@ -14,7 +14,7 @@ uk.ac.ebi.eva eva-tools - 0.3.0 + 0.3.1-SNAPSHOT diff --git a/pom.xml b/pom.xml index 54833b12..022cc928 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ uk.ac.ebi.eva eva-tools - 0.3.0 + 0.3.1-SNAPSHOT pom diff --git a/vcf-dumper/pom.xml b/vcf-dumper/pom.xml index c8875fbb..9d97ac21 100644 --- a/vcf-dumper/pom.xml +++ b/vcf-dumper/pom.xml @@ -6,13 +6,13 @@ uk.ac.ebi.eva vcf-dumper - 0.3.0 + 0.3.1-SNAPSHOT pom uk.ac.ebi.eva eva-tools - 0.3.0 + 0.3.1-SNAPSHOT diff --git a/vcf-dumper/vcf-dumper-cli/pom.xml b/vcf-dumper/vcf-dumper-cli/pom.xml index 3901d17d..397c801a 100644 --- a/vcf-dumper/vcf-dumper-cli/pom.xml +++ b/vcf-dumper/vcf-dumper-cli/pom.xml @@ -7,7 +7,7 @@ uk.ac.ebi.eva vcf-dumper - 0.3.0 + 0.3.1-SNAPSHOT vcf-dumper-cli @@ -25,7 +25,7 @@ uk.ac.ebi.eva vcf-dumper-lib - 0.3.0 + 0.3.1-SNAPSHOT diff --git a/vcf-dumper/vcf-dumper-lib/pom.xml b/vcf-dumper/vcf-dumper-lib/pom.xml index bb39c86e..2d390010 100644 --- a/vcf-dumper/vcf-dumper-lib/pom.xml +++ b/vcf-dumper/vcf-dumper-lib/pom.xml @@ -7,7 +7,7 @@ uk.ac.ebi.eva vcf-dumper - 0.3.0 + 0.3.1-SNAPSHOT vcf-dumper-lib diff --git a/vcf-dumper/vcf-dumper-ws/pom.xml b/vcf-dumper/vcf-dumper-ws/pom.xml index 60427802..8d91c62d 100644 --- a/vcf-dumper/vcf-dumper-ws/pom.xml +++ b/vcf-dumper/vcf-dumper-ws/pom.xml @@ -7,7 +7,7 @@ uk.ac.ebi.eva vcf-dumper - 0.3.0 + 0.3.1-SNAPSHOT vcf-dumper-ws @@ -32,7 +32,7 @@ uk.ac.ebi.eva vcf-dumper-lib - 0.3.0 + 0.3.1-SNAPSHOT io.springfox