Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EVA-1903 - Spring Boot 2 upgrade #99

Merged
merged 6 commits into from
Jun 17, 2020
Merged

EVA-1903 - Spring Boot 2 upgrade #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
63bbbb7
Spring 2 upgrade
sundarvenkata-EBI Jun 10, 2020
83e8f5a
Avoid using separate port parameter
sundarvenkata-EBI Jun 10, 2020
b17b474
Fix autowiring issues with SpringDataMongoDbProperties
sundarvenkata-EBI Jun 10, 2020
5dd86dc
Fix autowiring issues with SpringDataMongoDbProperties
sundarvenkata-EBI Jun 10, 2020
49af400
Default authentication mechanism to SCRAM-SHA-1
sundarvenkata-EBI Jun 10, 2020
e710e0c
Update POM version to a snapshot version
sundarvenkata-EBI Jun 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ jdk:
- openjdk8

env:
- OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build MONGODB_VERSION=3.0.4
- OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build MONGODB_VERSION=4.0.18

install:
- wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-$MONGODB_VERSION.tgz
Expand Down
18 changes: 3 additions & 15 deletions database-migration/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>

<artifactId>database-migration</artifactId>
<version>0.3.0</version>
<version>0.3.1-SNAPSHOT</version>
<packaging>jar</packaging>

<parent>
<groupId>uk.ac.ebi.eva</groupId>
<artifactId>eva-tools</artifactId>
<version>0.3.0</version>
<version>0.3.1-SNAPSHOT</version>
</parent>

<dependencies>
Expand Down Expand Up @@ -44,19 +44,7 @@
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.4.0</version>
</dependency>
<dependency>
<groupId>com.github.fakemongo</groupId>
<artifactId>fongo</artifactId>
<version>2.1.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
<version>3.8.2</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
75 changes: 46 additions & 29 deletions ...ion/src/test/java/uk/ac/ebi/eva/dbmigration/mongodb/ExtractAnnotationFromVariantTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,14 @@
*/
package uk.ac.ebi.eva.dbmigration.mongodb;

import com.github.fakemongo.Fongo;
import com.github.fakemongo.FongoException;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.IndexOptions;

import org.bson.Document;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
Expand All @@ -30,9 +31,12 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static junit.framework.TestCase.assertNull;
import static org.junit.Assert.assertEquals;
Expand Down Expand Up @@ -87,18 +91,41 @@ public class ExtractAnnotationFromVariantTest {

private ExtractAnnotationFromVariant extractAnnotationFromVariant;

private MongoClient mongoClient;

private static final Map<String, String> databaseMap = Stream.of(new String[][]{
{"VARIANT_WITHOUT_ANNOTATION", "variantWithoutAnnotation"},
{"VARIANT_WITH_ANNOTATION", "variantWithAnnotation"},
{"DB_FOR_ANNOTATION_METADATA_CHECK", "DBForAnnotationMetadataCheck"},
{"DB_FOR_INDEXES_CHECK", "DBForIndexesCheck"},
{"DEFAULT_ANNOTATION", "defaultAnnotation"}
}).collect(Collectors.toMap(data -> data[0], data -> data[1]));

@Rule
public final ExpectedException exception = ExpectedException.none();

@Before
public void setUp() throws Exception {
extractAnnotationFromVariant = new ExtractAnnotationFromVariant();
mongoClient = new MongoClient();
}

@After
public void tearDown() throws Exception {
List<String> mongoDatabases = new ArrayList<>();
mongoClient.listDatabaseNames().iterator().forEachRemaining(mongoDatabases::add);
for(String usedDatabaseInTest: databaseMap.values()) {
if(mongoDatabases.contains(usedDatabaseInTest)) {
mongoClient.dropDatabase(usedDatabaseInTest);
}
}
mongoClient.close();
}

@Test
public void variantWithoutAnnotationShouldNotChange() {
// given
String dbName = "variantWithoutAnnotation";
String dbName = databaseMap.get("VARIANT_WITHOUT_ANNOTATION");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -112,7 +139,7 @@ public void variantWithoutAnnotationShouldNotChange() {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase database = new Fongo("testServer").getDatabase(dbName);
MongoDatabase database = mongoClient.getDatabase(dbName);
MongoCollection<Document> variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME);
MongoCollection<Document> annotationCollection = database.getCollection(ANNOTATION_COLLECTION_NAME);

Expand Down Expand Up @@ -141,7 +168,7 @@ public void variantWithoutAnnotationShouldNotChange() {
@Test
public void variantWithAnnotationShouldMigrate() {
// given
String dbName = "variantWithAnnotation";
String dbName = databaseMap.get("VARIANT_WITH_ANNOTATION");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -155,15 +182,15 @@ public void variantWithAnnotationShouldMigrate() {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName);
MongoDatabase database = mongoClient.getDatabase(dbName);
MongoCollection<Document> variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME);
MongoCollection<Document> annotationCollection = database.getCollection(ANNOTATION_COLLECTION_NAME);

Document variantWithAnnot = Document.parse(VariantData.VARIANT_WITH_ANNOT_1);
variantsCollection.insertOne(variantWithAnnot);

Document originalVariant = variantsCollection.find().first();
Document originalAnnotField = (Document) originalVariant.get(ANNOT_FIELD);
Document originalAnnotField = (Document)originalVariant.get(ANNOT_FIELD);

createLegacyIndexes(variantsCollection);

Expand All @@ -189,7 +216,7 @@ public void variantWithAnnotationShouldMigrate() {
@Test
public void variantWithAnnotationShouldKeepSomeFields() {
// given
String dbName = "variantWithAnnotation";
String dbName = databaseMap.get("VARIANT_WITH_ANNOTATION");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -203,7 +230,7 @@ public void variantWithAnnotationShouldKeepSomeFields() {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName);
MongoDatabase database = mongoClient.getDatabase(dbName);
MongoCollection<Document> variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME);

Document variantWithAnnot = Document.parse(VariantData.VARIANT_WITH_ANNOT_2);
Expand Down Expand Up @@ -302,7 +329,7 @@ private List<Double> computePolyphen(Document originalAnnotField) {
@Test
public void metadataShouldBeUpdated() throws Exception {
// given
String dbName = "DBForAnnotationMetadataCheck";
String dbName = databaseMap.get("DB_FOR_ANNOTATION_METADATA_CHECK");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -316,7 +343,7 @@ public void metadataShouldBeUpdated() throws Exception {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName);
MongoDatabase database = mongoClient.getDatabase(dbName);
MongoCollection<Document> variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME);
MongoCollection<Document> annotationMetadataCollection = database.getCollection(
ANNOTATION_METADATA_COLLECTION_NAME);
Expand All @@ -337,7 +364,7 @@ public void metadataShouldBeUpdated() throws Exception {
@Test
public void indexesShouldBeCreated() throws Exception {
// given
String dbName = "DBForIndexesCheck";
String dbName = databaseMap.get("DB_FOR_INDEXES_CHECK");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -351,7 +378,7 @@ public void indexesShouldBeCreated() throws Exception {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase database = new Fongo("testServer").getMongo().getDatabase(dbName);
MongoDatabase database = mongoClient.getDatabase(dbName);
MongoCollection<Document> variantsCollection = database.getCollection(VARIANT_COLLECTION_NAME);
MongoCollection<Document> annotationsCollection = database.getCollection(ANNOTATION_COLLECTION_NAME);

Expand All @@ -362,14 +389,14 @@ public void indexesShouldBeCreated() throws Exception {

// when
extractAnnotationFromVariant.migrateAnnotation(database);
// extractAnnotationFromVariant.dropIndexes(database); // note we didn't drop the indexes, as fongo doesn't support that
extractAnnotationFromVariant.dropIndexes(database);
extractAnnotationFromVariant.reduceAnnotationFromVariants(database);
extractAnnotationFromVariant.updateAnnotationMetadata(database);
extractAnnotationFromVariant.createIndexes(database);

// then
ArrayList<Document> variantsIndexes = variantsCollection.listIndexes().into(new ArrayList<>());
assertEquals(8, variantsIndexes.size()); // note we didn't drop the indexes, as fongo doesn't support that
assertEquals(6, variantsIndexes.size());
assertIndexNameExists(variantsIndexes, ANNOT_FIELD + "." + SO_FIELD + "_1");
assertIndexNameExists(variantsIndexes, ANNOT_FIELD + "." + XREFS_FIELD + "_1");

Expand Down Expand Up @@ -410,19 +437,9 @@ private void assertIndexNameExists(ArrayList<Document> variantsIndexes, String i
assertTrue(indexFound);
}

@Test
public void testFakemongoFailsToDropIndexes() throws Exception {
MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase("variantWithoutAnnotation");
MongoCollection<Document> variantsCollection = mongoDatabase.getCollection(VARIANT_COLLECTION_NAME);
variantsCollection.createIndex(new Document("annot.ct.so", 1), new IndexOptions().background(true));

exception.expect(FongoException.class);
variantsCollection.dropIndex("annot.ct.so_1");
}

@Test
public void testDefaultAnnotationVersion() throws Exception {
String dbName = "defaultAnnotation";
String dbName = databaseMap.get("DEFAULT_ANNOTATION");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -436,7 +453,7 @@ public void testDefaultAnnotationVersion() throws Exception {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase(dbName);
MongoDatabase mongoDatabase = mongoClient.getDatabase(dbName);
MongoCollection<Document> collection = mongoDatabase.getCollection(ANNOTATION_METADATA_COLLECTION_NAME);

collection.insertOne(buildAnnotationMetadataDocument("79", "78"));
Expand All @@ -457,7 +474,7 @@ public void testDefaultAnnotationVersion() throws Exception {

@Test
public void testDefaultAnnotationVersionNotFound() throws Exception {
String dbName = "defaultAnnotation";
String dbName = databaseMap.get("DEFAULT_ANNOTATION");

Properties properties = new Properties();
properties.put(DatabaseParameters.VEP_VERSION, VEP_VERSION);
Expand All @@ -471,7 +488,7 @@ public void testDefaultAnnotationVersionNotFound() throws Exception {
databaseParameters.load(properties);
ExtractAnnotationFromVariant.setDatabaseParameters(databaseParameters);

MongoDatabase mongoDatabase = new Fongo("testServer").getDatabase(dbName);
MongoDatabase mongoDatabase = mongoClient.getDatabase(dbName);
MongoCollection<Document> collection = mongoDatabase.getCollection(ANNOTATION_METADATA_COLLECTION_NAME);

collection.insertOne(buildAnnotationMetadataDocument("79", "78"));
Expand Down
41 changes: 30 additions & 11 deletions dbsnp-importer/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

<groupId>uk.ac.ebi.eva</groupId>
<artifactId>dbsnp-importer</artifactId>
<version>0.3.0</version>
<version>0.3.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>dbsnp-importer</name>
Expand All @@ -14,13 +14,14 @@
<parent>
<groupId>uk.ac.ebi.eva</groupId>
<artifactId>eva-tools</artifactId>
<version>0.3.0</version>
<version>0.3.1-SNAPSHOT</version>
</parent>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<eva.mongo.host.test>localhost:27017</eva.mongo.host.test>
</properties>

<dependencies>
Expand All @@ -46,7 +47,7 @@
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>2.3.4</version>
<version>2.4.1</version>
<scope>test</scope>
</dependency>

Expand All @@ -63,18 +64,12 @@
<dependency>
<groupId>uk.ac.ebi.eva</groupId>
<artifactId>variation-commons-core</artifactId>
<version>0.5-SNAPSHOT</version>
<version>${variation-commons.version}</version>
</dependency>
<dependency>
<groupId>uk.ac.ebi.eva</groupId>
<artifactId>variation-commons-mongodb</artifactId>
<version>0.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.github.fakemongo</groupId>
<artifactId>fongo</artifactId>
<scope>test</scope>
<version>2.1.0</version>
<version>${variation-commons.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand All @@ -86,6 +81,17 @@
<artifactId>jackson-core</artifactId>
<version>2.8.4</version>
</dependency>
<dependency>
<groupId>com.lordofthejars</groupId>
<artifactId>nosqlunit-mongodb</artifactId>
<scope>test</scope>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
</dependencies>

<build>
Expand All @@ -95,6 +101,19 @@
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
<testResources>
<testResource>
<directory>src/test/resources/properties</directory>
<filtering>true</filtering>
</testResource>
<testResource>
<directory>src/test/resources/</directory>
<filtering>false</filtering>
<excludes>
<exclude>src/test/resources/properties/*</exclude>
</excludes>
</testResource>
</testResources>
</build>


Expand Down
Loading