Merge pull request #137 from sundarvenkata-EBI/ops/spring2_upgrade

EVA-1902 - Spring Boot 2 upgrade - Replace Fongo with real MongoDB tests because Fongo is not supported in Spring Boot 2.1 - see [here](fakemongo/fongo#357) - Update Travis and Gitlab scripts to include Mongo installations - Update POMs to support "|" delimiters so that EVA mongo host for tests can be parametrized for both Travis and Gitlab - javax.validation API dependency need no longer be explicitly pulled in since Spring Boot 2 contains it - Upgrade jackson-databind version to 2.9.7 to work with Spring Boot 2 - JPA repository query methods should be marked with "Query" annotation
EBIvariation · May 29, 2020 · fb1d77d · fb1d77d
2 parents a271299 + 7dcbd65
commit fb1d77d
Show file tree

Hide file tree

Showing 41 changed files with 412 additions and 179 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -12,8 +12,11 @@ variables:
 test:
     stage: test
     image: maven:3.6.1-jdk-8-alpine
+    services:
+        - mongo:4.0.18
     script:
-        - mvn clean test --projects 'eva-lib,eva-server'
+        # Gitlab exposes services under their own hostnames. So test host should be "mongo" instead of "localhost".
+        - mvn clean test --projects 'eva-lib,eva-server' -Deva.mongo.host.test=mongo
     only:
         - master
         - tags

diff --git a/.travis.yml b/.travis.yml
@@ -1,16 +1,18 @@
 sudo: false
-
-language: java
-
-jdk:
-  - openjdk8
-
-env:
-  - OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build
-
-install:
-  - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V -PTravis
-
-script:
-  - mvn test -s .travis.settings.xml -PTravis
-
+matrix:
+  include:
+    - language: "java"
+      jdk: "openjdk8"
+      env:
+        - OPENCGA_HOME=$TRAVIS_BUILD_DIR/opencga/opencga-app/build
+        - MONGODB_VERSION=4.0.18
+      install:
+        - wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-$MONGODB_VERSION.tgz
+        - tar xfz mongodb-linux-x86_64-$MONGODB_VERSION.tgz
+        - export PATH=`pwd`/mongodb-linux-x86_64-$MONGODB_VERSION/bin:$PATH
+        - mkdir -p data/db
+        - mongod --dbpath=data/db &
+        - mongod --version
+        - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V -PTravis
+      script:
+        - mvn test -s .travis.settings.xml -PTravis
diff --git a/dbsnp-import/pom.xml b/dbsnp-import/pom.xml
@@ -64,12 +64,6 @@
             <artifactId>hsqldb</artifactId>
            <!-- <scope>test</scope>-->
         </dependency>
-        <dependency>
-            <groupId>com.github.fakemongo</groupId>
-            <artifactId>fongo</artifactId>
-            <version>2.0.12</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
     <build>

diff --git a/dbsnp-import/src/main/java/uk/ac/ebi/eva/server/Application.java b/dbsnp-import/src/main/java/uk/ac/ebi/eva/server/Application.java
@@ -19,7 +19,7 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.support.SpringBootServletInitializer;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
 
 @SpringBootApplication
 public class Application extends SpringBootServletInitializer {

diff --git a/dbsnp-import/src/main/resources/application.properties b/dbsnp-import/src/main/resources/application.properties
@@ -1,10 +1,13 @@
 spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=@eva.evapro.jdbc.url@?currentSchema=@eva.evapro.dbsnp-progress.schema@
-spring.datasource.username=@eva.evapro.user@
-spring.datasource.password=@eva.evapro.password@
+spring.datasource.url=|eva.evapro.jdbc.url|?currentSchema=|eva.evapro.dbsnp-progress.schema|
+spring.datasource.username=|eva.evapro.user|
+spring.datasource.password=|eva.evapro.password|
 
 spring.jpa.generate-ddl=false
 spring.data.rest.base-path=/v1/
 
 management.endpoints.web.exposure.include=info,health
 management.info.git.mode=full
+
+# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding
+spring.main.allow-bean-definition-overriding=true
diff --git a/...-import/src/test/java/uk/ac/ebi/eva/server/repositories/ProgressReportRepositoryTest.java b/...-import/src/test/java/uk/ac/ebi/eva/server/repositories/ProgressReportRepositoryTest.java
@@ -28,10 +28,11 @@
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 @RunWith(SpringRunner.class)
 @DataJpaTest
@@ -47,13 +48,14 @@ public void testCountRecords() {
 
     @Test
     public void testFindById() {
-        ProgressReport report = progressReportRepository.findOne(new ProgressReportPK("fruitfly_7227",
-                                                                                      "GCA_000001215.4"));
+        Optional<ProgressReport> report = progressReportRepository.findById(new ProgressReportPK("fruitfly_7227",
+                                                                                                 "GCA_000001215.4"));
+        assertTrue(report.isPresent());
         ProgressReport expected = new ProgressReport("fruitfly_7227", 7227, "Drosophila melanogaster", "Fruit fly",
                                                      "GCA_000001215.4", 149, true, false, false, Status.pending,
                                                      Status.pending, Status.pending, null, null, null, 0L, 0L, 0L, 0L,
                                                      0L, 0L);
-        assertEquals(expected, report);
+        assertEquals(expected, report.get());
     }
 
     @Test
@@ -70,24 +72,25 @@ public void testFindMultipleAssemblies() {
 
     @Test
     public void testVariantWithEvidenceImportFields() {
-        ProgressReport report = progressReportRepository.findOne(new ProgressReportPK("arabidopsis_3702",
-                                                                                      "GCA_000001735.1"));
-        assertEquals(Status.done, report.getVariantsWithEvidenceImported());
+        Optional<ProgressReport> report = progressReportRepository.findById(new ProgressReportPK("arabidopsis_3702",
+                                                                                                 "GCA_000001735.1"));
+        assertTrue(report.isPresent());
+        assertEquals(Status.done, report.get().getVariantsWithEvidenceImported());
         Calendar cal = Calendar.getInstance();
         cal.set(2018, Calendar.MAY, 30, 0, 0, 0);
         cal.set(Calendar.MILLISECOND, 0);
         Date date = cal.getTime();
-        assertEquals(date.getTime(), report.getVariantsWithEvidenceImportedDate().getTime());
+        assertEquals(date.getTime(), report.get().getVariantsWithEvidenceImportedDate().getTime());
     }
 
     @Test
     public void testVariantWithoutGenbankAccession() {
         String databaseName = "orangutan_9600";
         String genbankAssemblyAccession = "";
-        ProgressReport report = progressReportRepository.findOne(new ProgressReportPK(databaseName,
-                                                                                      genbankAssemblyAccession));
-        assertNotNull(report);
-        assertEquals(genbankAssemblyAccession, report.getGenbankAssemblyAccession());
-        assertEquals(databaseName, report.getDatabaseName());
+        Optional<ProgressReport> report = progressReportRepository.findById(new ProgressReportPK(databaseName,
+                                                                                                 genbankAssemblyAccession));
+        assertTrue(report.isPresent());
+        assertEquals(genbankAssemblyAccession, report.get().getGenbankAssemblyAccession());
+        assertEquals(databaseName, report.get().getDatabaseName());
     }
 }
diff --git a/dbsnp-import/src/test/resources/application.properties b/dbsnp-import/src/test/resources/application.properties
@@ -9,3 +9,6 @@ spring.datasource.generate-unique-name=true
 
 spring.jpa.generate-ddl=true
 spring.jpa.show-sql=true
+
+# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding
+spring.main.allow-bean-definition-overriding=true
diff --git a/dgva-server/src/main/java/uk/ac/ebi/dgva/server/Application.java b/dgva-server/src/main/java/uk/ac/ebi/dgva/server/Application.java
@@ -19,7 +19,7 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.boot.web.support.SpringBootServletInitializer;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
 
 @SpringBootApplication
 public class Application extends SpringBootServletInitializer {

diff --git a/dgva-server/src/main/resources/application.properties b/dgva-server/src/main/resources/application.properties
@@ -15,3 +15,6 @@ springfox.documentation.swagger.v2.path=/webservices/api
 
 management.endpoints.web.exposure.include=info,health
 management.info.git.mode=full
+
+# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding
+spring.main.allow-bean-definition-overriding=true
diff --git a/dgva-server/src/test/resources/application.properties b/dgva-server/src/test/resources/application.properties
@@ -11,3 +11,6 @@ spring.jpa.show-sql=true
 
 logging.level.com.bytestree.restful=DEBUG
 logging.level.org.springframework.web.client.RestTemplate=DEBUG
+
+# See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.1-Release-Notes#bean-overriding
+spring.main.allow-bean-definition-overriding=true
diff --git a/eva-lib/pom.xml b/eva-lib/pom.xml
@@ -13,6 +13,7 @@
 
     <properties>
         <compileSource>1.8</compileSource>
+        <eva.mongo.host.test>localhost</eva.mongo.host.test>
     </properties>
 
     <dependencies>
@@ -32,24 +33,13 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-jpa</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.github.fakemongo</groupId>
-            <artifactId>fongo</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>com.lordofthejars</groupId>
             <artifactId>nosqlunit-core</artifactId>
         </dependency>
         <dependency>
             <groupId>com.lordofthejars</groupId>
             <artifactId>nosqlunit-mongodb</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.github.fakemongo</groupId>
-                    <artifactId>fongo</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>com.h2database</groupId>
@@ -60,10 +50,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-validation</artifactId>
         </dependency>
-        <dependency>
-            <groupId>javax.validation</groupId>
-            <artifactId>validation-api</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
@@ -77,6 +63,19 @@
 
     <build>
         <finalName>${project.artifactId}-${project.version}-${git.commit.id.abbrev}</finalName>
+        <testResources>
+            <testResource>
+                <directory>src/test/resources/properties</directory>
+                <filtering>true</filtering>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources/</directory>
+                <filtering>false</filtering>
+                <excludes>
+                    <exclude>src/test/resources/properties/*</exclude>
+                </excludes>
+            </testResource>
+        </testResources>
     </build>
 
 </project>
diff --git a/eva-lib/src/main/java/uk/ac/ebi/eva/lib/MongoConfiguration.java b/eva-lib/src/main/java/uk/ac/ebi/eva/lib/MongoConfiguration.java
@@ -15,6 +15,13 @@
  */
 package uk.ac.ebi.eva.lib;
 
+import com.mongodb.AuthenticationMechanism;
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
+import com.mongodb.MongoCredential;
+import com.mongodb.ReadPreference;
+import com.mongodb.ServerAddress;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
@@ -29,8 +36,13 @@
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
 import uk.ac.ebi.eva.lib.configuration.DbCollectionsProperties;
+import uk.ac.ebi.eva.lib.configuration.SpringDataMongoDbProperties;
 
 import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 @Configuration
 @Import(DbCollectionsProperties.class)
@@ -47,6 +59,9 @@ public class MongoConfiguration {
     @Autowired
     private DbCollectionsProperties dbCollectionsProperties;
 
+    @Autowired
+    private SpringDataMongoDbProperties springDataMongoDbProperties;
+
     @Bean
     public String mongoCollectionsVariants() {
         return dbCollectionsProperties.getVariants();
@@ -91,4 +106,57 @@ public MappingMongoConverter mappingMongoConverter() throws IOException {
         return mongoConverter;
     }
 
+    /**
+     * Get a MongoClient using the configuration (credentials) in a given Properties.
+     *
+     * @param springDataMongoDbProperties can have the next values:
+     *                   - eva.mongo.auth.db authentication database
+     *                   - eva.mongo.host comma-separated strings of colon-separated host and port strings: host_1:port_1,host_2:port_2
+     *                   - eva.mongo.user
+     *                   - eva.mongo.passwd
+     *                   - eva.mongo.read-preference string, "secondaryPreferred" if unspecified. one of:
+     *                          [primary, primaryPreferred, secondary, secondaryPreferred, nearest]
+     * @return MongoClient with given credentials
+     * @throws UnknownHostException
+     */
+    public static MongoClient getMongoClient(SpringDataMongoDbProperties springDataMongoDbProperties) throws UnknownHostException {
+
+        String[] hosts = springDataMongoDbProperties.getHost().split(",");
+        List<ServerAddress> servers = new ArrayList<>();
+
+        // Get the list of hosts (optionally including the port number)
+        for (String host : hosts) {
+            String[] params = host.split(":");
+            if (params.length > 1) {
+                servers.add(new ServerAddress(params[0], Integer.parseInt(params[1])));
+            } else {
+                servers.add(new ServerAddress(params[0], 27017));
+            }
+        }
+
+        String readPreference = springDataMongoDbProperties.getReadPreference();
+        readPreference = readPreference == null || readPreference.isEmpty()? "secondaryPreferred" : readPreference;
+
+        MongoClientOptions options = MongoClientOptions.builder()
+                                                       .readPreference(ReadPreference.valueOf(readPreference))
+                                                       .build();
+
+        List<MongoCredential> mongoCredentialList = new ArrayList<>();
+        String authenticationDb = springDataMongoDbProperties.getAuthenticationDatabase();
+        if (authenticationDb != null && !authenticationDb.isEmpty()) {
+            MongoCredential mongoCredential = MongoCredential.createCredential(
+                    springDataMongoDbProperties.getUsername(),
+                    authenticationDb,
+                    springDataMongoDbProperties.getPassword().toCharArray());
+            String authenticationMechanism = springDataMongoDbProperties.getAuthenticationMechanism();
+            if (authenticationMechanism == null) {
+                return new MongoClient(servers, options);
+            }
+            mongoCredential = mongoCredential.withMechanism(
+                    AuthenticationMechanism.fromMechanismName(authenticationMechanism));
+            mongoCredentialList = Collections.singletonList(mongoCredential);
+        }
+
+        return new MongoClient(servers, mongoCredentialList, options);
+    }
 }
diff --git a/eva-lib/src/main/java/uk/ac/ebi/eva/lib/configuration/SpringDataMongoDbProperties.java b/eva-lib/src/main/java/uk/ac/ebi/eva/lib/configuration/SpringDataMongoDbProperties.java
@@ -24,21 +24,33 @@
 @Component
 public class SpringDataMongoDbProperties {
 
+    private String database;
+
     @Size(min = 1)
     private String host;
 
     private String authenticationDatabase;
 
+    private String authenticationMechanism;
+
     private String username;
 
     private String password;
 
     private String readPreference;
 
+    public String getDatabase() {
+        return database;
+    }
+
     public String getHost() {
         return host;
     }
 
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
     public void setHost(String host) {
         this.host = host;
     }
@@ -51,6 +63,14 @@ public void setAuthenticationDatabase(String authenticationDatabase) {
         this.authenticationDatabase = authenticationDatabase;
     }
 
+    public String getAuthenticationMechanism() {
+        return authenticationMechanism;
+    }
+
+    public void setAuthenticationMechanism(String authenticationMechanism) {
+        this.authenticationMechanism = authenticationMechanism;
+    }
+
     public String getUsername() {
         return username;
     }