Merge remote-tracking branch 'origin/develop'

# Conflicts:
#	build.gradle

build.gradle

@@ -11,7 +11,7 @@ buildscript {
     }
 }
 
-version "2.0.0"
+version "2.1.0"
 group "au.org.ala"
 
 apply plugin:"eclipse"
@@ -111,7 +111,7 @@ dependencies {
     }
     implementation "com.google.guava:guava:19.0"
     implementation 'dk.glasius:external-config:3.1.1'
-    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.1.0"
+    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.3.0"
     implementation "org.grails.plugins:ala-ws-security-plugin:${alaSecurityLibsVersion}"
     implementation "org.grails.plugins:ala-ws-plugin:${alaSecurityLibsVersion}"
     implementation "org.grails.plugins:ala-auth:${alaSecurityLibsVersion}"
@@ -127,8 +127,8 @@ dependencies {
 bootRun {
     ignoreExitValue true
     jvmArgs(
-        '-Dspring.output.ansi.enabled=always', 
-        '-noverify', 
+        '-Dspring.output.ansi.enabled=always',
+        '-noverify',
         '-XX:TieredStopAtLevel=1',
         '-Xmx1024m')
     sourceResources sourceSets.main

docker/solr8/Dockerfile

@@ -6,10 +6,10 @@ COPY zoo.cfg zoo.cfg
 #COPY --chown=solr:solr lib /opt/solr/server/solr-webapp/webapp/WEB-INF/lib/
 RUN mkdir -p bie bie/conf bie/data bie/suggest-infix
 COPY conf bie/conf
-COPY bie.core.properties bie/core.properties
+COPY --chown=solr:solr bie.core.properties bie/core.properties
 RUN mkdir -p bie-offline bie-offline/conf bie-offline/data bie-offline/suggest-infix
 COPY conf bie-offline/conf
-COPY bie-offline.core.properties bie-offline/core.properties
+COPY --chown=solr:solr bie-offline.core.properties bie-offline/core.properties
 COPY solr.in.sh /etc/default/solr.in.sh
 
 

gradle.properties

@@ -10,4 +10,4 @@ webdriverBinariesVersion=2.6
 chromeDriverVersion=2.45.0
 geckodriverVersion=0.24.0
 seleniumSafariDriverVersion=3.14.0
-alaSecurityLibsVersion=6.0.4
+alaSecurityLibsVersion=6.1.0

grails-app/conf/application.yml

@@ -19,6 +19,8 @@ grails:
       # Whether to translate GORM events into Reactor events
       # Disabled by default for performance reasons
       events: false
+  cors:
+    enabled: true
 info:
   app:
     name: '@info.app.name@'
@@ -150,6 +152,12 @@ images:
     small: https://images.ala.org.au/image/proxyImageThumbnailLarge?imageId={0}
     metadata: https://images.ala.org.au/ws/image/{0}
   index: true
+hiddenImages:
+  # URL of hidden images configuration file
+  config: /default-hidden-images-lists.json
+wiki:
+  # URL of wiki configuration file
+  config: /default-wiki-lists.json
 wordPress:
   service: https://www.ala.org.au
   sitemap: /xmlsitemap.xml
@@ -174,6 +182,8 @@ knowledgeBase:
 lists:
   service: https://lists.ala.org.au/ws
   items: /speciesListItems/{0}?includeKVP=true&max={1}&offset={2}
+  add: /createItem
+  remove: /deleteItem
 # If attribution entry is null, the default is used
 naming:
   service: https://namematching-ws.ala.org.au
@@ -214,6 +224,12 @@ additionalResultFields: ""
 
 # SOLR additional params and connections
 solr:
+  admin:
+    type: HTTP
+    connection: http://localhost:8983/solr
+    queueSize: 10
+    threadCount: 4
+    timeout: 120000
   live:
     type: HTTP
     connection: http://localhost:8983/solr/bie
@@ -257,7 +273,15 @@ skin:
   orgNameLong: Atlas of Living Australia
 useLegacyAuto: false
 import:
-  sequence: collectory,taxonomy-all,vernacular,denormalise,layers,regions,localities,conservation-lists,wordpress,knowledgebase,favourites,weights,link-identifiers,images,occurrences,suggest-index
+  sequence: collectory,taxonomy-all,vernacular,denormalise,layers,regions,localities,conservation-lists,wordpress,knowledgebase,favourites,weights,link-identifiers,images,occurrences,hidden-images,wiki-urls,suggest-index,swap
+  sequenceDaily: conservation-lists,wordpress,knowledgebase,favourites,suggest-index,swap
+  sequenceWeekly: images,hidden-images,wiki-urls,occurrences,layers,regions,localities,suggest-index,swap
+  # enable daily and weekly tasks
+  enableTasks: false
+  # run daily sequence on this hour
+  dailyRunHour: 2
+  # run weekly sequence on this day (6 == Saturday)
+  weeklyRunDay: 6
   taxonomy:
     dir: /data/bie/import
   priority:
@@ -275,4 +299,4 @@ import:
 
 headerAndFooter:
   baseURL: "https://www.ala.org.au/commonui-bs3-2019"
-  version: "2"
+  version: "2"

grails-app/conf/default-hidden-images-lists.json

@@ -0,0 +1,8 @@
+{
+  "lists": [
+    {
+      "uid": "dr21953",
+      "imageId": "image Id"
+    }
+  ]
+}

grails-app/conf/default-image-lists.json

@@ -35,6 +35,10 @@
     }
   ],
   "lists": [
+    {
+      "uid": "dr4778",
+      "imageId": "imageId"
+    }
   ],
   "imageFields":"taxon_concept_lsid,kingdom,phylum,class,order,family,genus,species,taxon_name,image_url,data_resource_uid"
-}
+}

grails-app/conf/default-wiki-lists.json

@@ -0,0 +1,8 @@
+{
+  "lists": [
+    {
+      "uid": "dr21955",
+      "wikiUrl": "url"
+    }
+  ]
+}

grails-app/conf/logback.xml

@@ -11,7 +11,11 @@
         </encoder>
     </appender>
 
+    <logger name="org.grails.config.NavigableMap" level="ERROR"/>
+
+    <logger name="au.org.ala.bie" level="INFO"/>
+
     <root level="WARN">
         <appender-ref ref="STDOUT" />
     </root>
-</configuration>
+</configuration>

grails-app/conf/spring/resources.groovy

@@ -1,5 +1,6 @@
 import au.org.ala.bie.solr.SolrClientBean
 import au.org.ala.bie.util.ConservationListsSource
+import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler
 
 // Place your Spring DSL code here
 beans = {
@@ -24,5 +25,14 @@ beans = {
             application.config.solr.updatingLive.threadCount as Integer,
             application.config.solr.updatingLive.timeout as Integer
     )
+    adminSolrClient(SolrClientBean,
+            application.config.solr.admin.type,
+            application.config.solr.admin.connection,
+            application.config.solr.admin.queueSize as Integer,
+            application.config.solr.admin.threadCount as Integer,
+            application.config.solr.admin.timeout as Integer
+    )
     conservationListsSource(ConservationListsSource, application.config.conservationListsUrl)
+
+    threadPoolTaskScheduler(ThreadPoolTaskScheduler)
 }

grails-app/controllers/au/org/ala/bie/AdminController.groovy

@@ -3,11 +3,14 @@ import au.org.ala.web.AlaSecured
 
 @AlaSecured(value = "ROLE_ADMIN", redirectUri = "/")
 class AdminController {
+    def indexService
 
-    def index() {} // GSP only index
+    def index() {
+        [info: indexService.info()]
+    }
 
     // Documented in openapi.yml
     def indexFields() {
         redirect(controller: "misc", action: "indexFields") // shouldn't get triggered due UrlMappings
     }
-}
+}

grails-app/controllers/au/org/ala/bie/ImportController.groovy

@@ -14,16 +14,8 @@
 package au.org.ala.bie
 
 import au.org.ala.bie.util.Job
-import au.org.ala.plugins.openapi.Path
 import grails.converters.JSON
 import au.org.ala.web.AlaSecured
-import io.swagger.v3.oas.annotations.Operation
-import io.swagger.v3.oas.annotations.headers.Header
-import io.swagger.v3.oas.annotations.media.Schema
-import io.swagger.v3.oas.annotations.responses.ApiResponse
-import io.swagger.v3.oas.annotations.security.SecurityRequirement
-
-import javax.ws.rs.Produces
 
 
 /**
@@ -32,7 +24,7 @@ import javax.ws.rs.Produces
 @AlaSecured(value = "ROLE_ADMIN", redirectUri = "/")
 class ImportController {
 
-    def importService, bieAuthService
+    def importService, bieAuthService, indexService
     def brokerMessagingTemplate
     def jobService
 
@@ -46,7 +38,11 @@ class ImportController {
         [filePaths: filePaths]
     }
 
-    def all(){}
+    def all(){
+        [info: indexService.info()]
+    }
+
+    def daily(){}
 
     def collectory(){}
 
@@ -66,6 +62,10 @@ class ImportController {
 
     def occurrences(){}
 
+    def swap() {
+        indexService.swap()
+    }
+
     /**
      * Import a DwC-A into this system.
      *
@@ -83,21 +83,34 @@ class ImportController {
         def dwcDir = params.dwca_dir
 
         if(new File(dwcDir).exists()){
-            def job = execute("importDwca", "admin.button.importdwca", { importService.importDwcA(dwcDir, clearIndex) })
+            def job = execute("importDwca", "admin.button.importdwca", { importService.importDwcA(dwcDir, clearIndex, false) })
             asJson (job.status())
         } else {
             asJson ([success: false, message: 'Supplied directory path is not accessible'])
         }
     }
 
-
     def importAll(){
-        def job = execute(
-                "importDwca,importCollectory,deleteDanglingSynonyms,importLayers,importLocalities,importRegions,importHabitats,importHabitats," +
-                        "importWordPressPages,importOccurrences,importConsevationSpeciesLists,buildVernacularSpeciesLists,buildLinkIdentifiers" +
-                        "denormaliseTaxa,loadImages,importKnowledgeBase",
+        boolean online = params.getBoolean('online', false)
+        def job = execute(importService.importSequence.join(','),
                 "admin.button.importall",
-                { importService.importAll() })
+                { importService.importAll(importService.importSequence, online) })
+        asJson(job.status())
+    }
+
+    def importDaily(){
+        boolean online = params.getBoolean('online', false)
+        def job = execute(importService.importDailySequence.join(','),
+                "admin.button.daily",
+                { importService.importAll(importService.importDailySequence, online) })
+        asJson(job.status())
+    }
+
+    def importWeekly(){
+        boolean online = params.getBoolean('online', false)
+        def job = execute(importService.importWeeklySequence.join(','),
+                "admin.button.weekly",
+                { importService.importAll(importService.importWeeklySequence, online) })
         asJson(job.status())
     }
 
@@ -107,8 +120,9 @@ class ImportController {
      * @return
      */
     def importAllDwcA() {
+        boolean online = params.getBoolean('online', false)
         if(new File(grailsApplication.config..getProperty('import.taxonomy.dir')).exists()){
-            def job = execute("importDwca", "admin.button.importalldwca", { importService.importAllDwcA() })
+            def job = execute("importDwca", "admin.button.importalldwca", { importService.importAllDwcA(online) })
             asJson(job.status())
         } else {
             asJson ([success: false, message: 'Supplied directory path is not accessible'])
@@ -117,7 +131,8 @@ class ImportController {
 
     // Documented in openapi.yml
     def deleteDanglingSynonyms(){
-        def job = execute("deleteDanglingSynonyms", "admin.button.deletedanglingsynonyms", { importService.clearDanglingSynonyms() })
+        boolean online = params.getBoolean('online', false)
+        def job = execute("deleteDanglingSynonyms", "admin.button.deletedanglingsynonyms", { importService.clearDanglingSynonyms(online) })
         asJson(job.status())
     }
 
@@ -192,7 +207,8 @@ class ImportController {
      */
     // Documented in openapi.yml
     def importHabitats(){
-        def job = execute("importHabitats", "admin.button.importhabitats", { importService.importHabitats() })
+        boolean online = params.getBoolean('online', false)
+        def job = execute("importHabitats", "admin.button.importhabitats", { importService.importHabitats(online) })
         asJson(job.status())
     }
 
@@ -239,7 +255,8 @@ class ImportController {
      */
     // Documented in openapi.yml
     def importConservationSpeciesLists(){
-        def job = execute("importConsevationSpeciesLists", "admin.button.importlistconservatioon", { importService.importConservationSpeciesLists() })
+        boolean online = params.getBoolean('online', false)
+        def job = execute("importConsevationSpeciesLists", "admin.button.importlistconservatioon", { importService.importConservationSpeciesLists(online) })
         asJson(job.status())
     }
 
@@ -250,7 +267,21 @@ class ImportController {
      */
     // Documented in openapi.yml
     def importVernacularSpeciesLists(){
-        def job = execute("buildVernacularSpeciesLists", "admin.button.importlistvernacular", { importService.importVernacularSpeciesLists() })
+        boolean online = params.getBoolean('online', false)
+        def job = execute("buildVernacularSpeciesLists", "admin.button.importlistvernacular", { importService.importVernacularSpeciesLists(online) })
+        asJson (job.status())
+
+    }
+
+    def importWikiUrls(){
+        boolean online = params.getBoolean('online', false)
+        def job = execute("buildWikiUrls", "admin.button.importlistwiki", { importService.loadWikiUrls(online) })
+        asJson (job.status())
+    }
+
+    def importHiddenImages(){
+        boolean online = params.getBoolean('online', false)
+        def job = execute("buildHiddenImages", "admin.button.importlisthiddenimages", { importService.loadHiddenImages(online) })
         asJson (job.status())
 
     }

grails-app/controllers/au/org/ala/bie/ImportServicesController.groovy

@@ -55,7 +55,7 @@ class ImportServicesController {
                         "importWordPressPages,importOccurrences,importConsevationSpeciesLists,buildVernacularSpeciesLists,buildLinkIdentifiers" +
                         "denormaliseTaxa,loadImages,",
                 "admin.button.importall",
-                { importService.importAll() })
+                { importService.importAll(importService.importSequence, false) })
         asJson(job.status())
     }