Merge pull request #147 from CCI-Tools/forman-xxx-xcube_intergr_2

@dzelge we continue w.o. your approval.

.env

@@ -14,8 +14,8 @@ REACT_APP_KEYCLOAK_CLIENT_ID=cate-webui
 # REACT_APP_MAX_NUM_USERS=50
 
 # TODO: document me
-# REACT_APP_CATEHUB_ENDPOINT=https://stage.catehub.brockmann-consult.de
-REACT_APP_CATEHUB_ENDPOINT=http://localhost:8000
-REACT_APP_CATEHUB_WEBAPI_MANAG_PATH=/user/{username}/webapi
-REACT_APP_CATEHUB_WEBAPI_CLOSE_PATH=/user/{username}/webapi/shutdown
-REACT_APP_CATEHUB_WEBAPI_COUNT_PATH=/webapi/count
+REACT_APP_CATEHUB_ENDPOINT=https://stage.catehub.climate.esa.int/api/v2
+#REACT_APP_CATEHUB_ENDPOINT=http://localhost:8080/api/v2
+REACT_APP_CATEHUB_WEBAPI_MANAG_PATH=/users/{username}/webapis
+REACT_APP_CATEHUB_WEBAPI_CLOSE_PATH=/users/{username}/webapis
+REACT_APP_CATEHUB_WEBAPI_COUNT_PATH=/webapis

CHANGES.md

@@ -1,3 +1,30 @@
+### Changes 3.0 (in development)
+
+* Cate App 3.0 now requires the Web API service of the `cate 3.0+` 
+  Python package.
+
+* In order to keep alive the connection to the Web API service,
+  Cate App now sends a keepalive signal every 2.5 seconds. (#150)
+
+* Optimisations in the DATA SOURCES panel (that have been enabled by 
+  using [xcube](https://xcube.readthedocs.io/) in the backend):
+  - Initalising the "CCI Open Data Portal" data store
+    is now accellerated by a magnitude.
+  - Local caching of remote data sources when opening datasets 
+    is now much faster and more reliable.
+  - Added new experimental store "CCI Zarr Store" that offers
+    selected CCI datasets that have been converted to Zarr format 
+    and are read from JASMIN object storage.
+  - Ability to add more data stores has been greatly improved.
+
+* We now obtain Cate Hub's status information from a dedicated GitHub 
+  repository [cate-status](https://github.com/CCI-Tools/cate-status).
+
+* Adapted to changed cate-hub API. (An API response no longer has 
+  `status` and `result` properties, instead a response _is_ the result
+  and the response status is represented by the HTTP response code.)
+
+
 ### Changes 2.2.3
 
 * Fixed a problem that prevented using Matomo Analytics service.

Dockerfile

@@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps
 
 LABEL maintainer="[email protected]"
 LABEL name="Cate App"
-LABEL version="2.2.3"
+LABEL version="3.0.0-dev.0"
 
 RUN apt-get -y update && apt-get install -y git apt-utils wget vim
 

appveyor.yml

@@ -1,4 +1,4 @@
-version: 2.2.3-{build}
+version: 3.0.0-dev.0-{build}
 image: Ubuntu
 stack: node 12
 install:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cate-app",
-  "version": "2.2.3",
+  "version": "3.0.0-dev.0",
   "private": true,
   "dependencies": {
     "@blueprintjs/core": "^3.30.1",
@@ -50,6 +50,7 @@
     "@types/dom4": "^1.5.20",
     "@types/geojson": "^1.0.6",
     "@types/jest": "^24.0.0",
+    "@types/json-schema": "^7.0.7",
     "@types/mocha": "^2.2.41",
     "@types/node": "^12.0.0",
     "@types/oboe": "^2.0.28",

src/renderer/actions.spec.ts

@@ -122,7 +122,7 @@ describe('Actions', () => {
                 ]);
         });
 
-        it('updateDataSourceTemporalCoverage', () => {
+        it('updateDataSourceMetaInfo', () => {
             dispatch(actions.updateDataStores(
                 [
                     {id: 'local-1'},
@@ -133,17 +133,19 @@ describe('Actions', () => {
                                                    {id: 'fileset-1'},
                                                    {id: 'fileset-2'}
                                                ] as any));
-            dispatch(actions.updateDataSourceTemporalCoverage('local-2',
-                                                              'fileset-1',
-                                                              ['2010-01-01', '2014-12-30']));
+            dispatch(actions.updateDataSourceMetaInfo('local-2',
+                                                      'fileset-1',
+                                                      {'data_id': 'x', 'type_specifier': 'y'},
+                                                      'ok'));
             expect(getState().data.dataStores).to.deep.equal(
                 [
                     {id: 'local-1'},
                     {
                         id: 'local-2', dataSources: [
                             {
                                 id: 'fileset-1',
-                                temporalCoverage: ['2010-01-01', '2014-12-30']
+                                metaInfo: {'data_id': 'x', 'type_specifier': 'y'},
+                                metaInfoStatus: 'ok',
                             },
                             {id: 'fileset-2'}
                         ]

src/renderer/actions.ts

@@ -41,10 +41,11 @@ import * as selectors from './selectors';
 import {
     BackendConfigState,
     ColorMapCategoryState,
-    ControlState,
+    ControlState, DatasetDescriptor,
     DataSourceState,
     DataStoreState,
-    GeographicPosition,
+    GeographicPosition, 
+    HubStatus,
     ImageStatisticsState,
     LayerState,
     MessageState,
@@ -67,6 +68,7 @@ import {
 } from './state';
 import {
     AUTO_LAYER_ID,
+    findDataSource,
     findResourceByName,
     genSimpleId,
     getCsvUrl,
@@ -137,6 +139,7 @@ export const SET_WEBAPI_STATUS = 'SET_WEBAPI_STATUS';
 export const SET_WEBAPI_CLIENT = 'SET_WEBAPI_CLIENT';
 export const SET_WEBAPI_SERVICE_URL = 'SET_WEBAPI_SERVICE_URL';
 export const SET_WEBAPI_SERVICE_INFO = 'SET_WEBAPI_SERVICE_INFO';
+export const UPDATE_HUB_STATUS = 'UPDATE_HUB_STATUS';
 export const UPDATE_DIALOG_STATE = 'UPDATE_DIALOG_STATE';
 export const UPDATE_TASK_STATE = 'UPDATE_TASK_STATE';
 export const REMOVE_TASK_STATE = 'REMOVE_TASK_STATE';
@@ -312,29 +315,47 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction {
 
         const webAPIClient = newWebAPIClient(selectors.apiWebSocketsUrlSelector(getState()));
 
+        const formatMessage = (message: string, event: any): string => {
+            if (event.message) {
+                return `${message} (${event.message})`;
+            } else {
+                return message;
+            }
+        };
+
+        /**
+         * Called to inform backend we are still alive.
+         * Hopefully avoids closing WebSocket connection.
+         */
+        const keepAlive = () => {
+            if (webAPIClient.isOpen) {
+                console.debug("calling keep_alive()");
+                webAPIClient.call('keep_alive', [])
+            }
+        };
+
+        let keepAliveTimer = null;
+
         webAPIClient.onOpen = () => {
             dispatch(setWebAPIClient(webAPIClient));
             dispatch(loadBackendConfig());
             dispatch(loadColorMaps());
             dispatch(loadPreferences());
             dispatch(loadDataStores());
             dispatch(loadOperations());
-        };
-
-        const formatMessage = (message: string, event: any): string => {
-            if (event.message) {
-                return `${message} (${event.message})`;
-            } else {
-                return message;
-            }
+            keepAliveTimer = setInterval(keepAlive, 2500);
         };
 
         webAPIClient.onClose = (event) => {
+            if (keepAliveTimer !== null) {
+                clearInterval(keepAliveTimer);
+            }
             const webAPIStatus = getState().communication.webAPIStatus;
             if (webAPIStatus === 'shuttingDown' || webAPIStatus === 'loggingOut') {
                 // When we are logging off, the webAPIClient is expected to close.
                 return;
             }
+            // When we end up here, the connection closed unintentionally.
             console.error('webAPIClient.onClose:', event);
             dispatch(setWebAPIStatus('closed'));
             showToast({type: 'notification', text: formatMessage('Connection to Cate service closed', event)});
@@ -353,6 +374,10 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction {
     };
 }
 
+export function updateHubStatus(hubStatus: HubStatus): Action {
+    return {type: UPDATE_HUB_STATUS, payload: hubStatus};
+}
+
 export function updateInitialState(initialState: Object): Action {
     return {type: UPDATE_INITIAL_STATE, payload: initialState};
 }
@@ -885,7 +910,7 @@ export function setSelectedPlacemarkId(selectedPlacemarkId: string | null): Acti
 
 export const UPDATE_DATA_STORES = 'UPDATE_DATA_STORES';
 export const UPDATE_DATA_SOURCES = 'UPDATE_DATA_SOURCES';
-export const UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE = 'UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE';
+export const UPDATE_DATA_SOURCE_META_INFO = 'UPDATE_DATA_SOURCE_META_INFO';
 
 /**
  * Asynchronously load the available Cate data stores.
@@ -987,33 +1012,54 @@ export function setSelectedDataStoreIdImpl(selectedDataStoreId: string | null) {
     return updateSessionState({selectedDataStoreId});
 }
 
-export function setSelectedDataSourceId(selectedDataSourceId: string | null) {
-    return updateSessionState({selectedDataSourceId});
+export function setSelectedDataSourceId(selectedDataSourceId: string): ThunkAction {
+    return (dispatch: Dispatch, getState: GetState) => {
+        dispatch(updateSessionState({selectedDataSourceId}));
+        const dataStoreId = getState().session.selectedDataStoreId;
+        if (dataStoreId && selectedDataSourceId) {
+            dispatch(loadDataSourceMetaInfo(dataStoreId, selectedDataSourceId));
+        }
+    }
 }
 
 export function setDataSourceFilterExpr(dataSourceFilterExpr: string) {
     return updateSessionState({dataSourceFilterExpr});
 }
 
-export function loadTemporalCoverage(dataStoreId: string, dataSourceId: string): ThunkAction {
+export function loadDataSourceMetaInfo(dataStoreId: string, dataSourceId: string): ThunkAction {
     return (dispatch: Dispatch, getState: GetState) => {
+        const dataStores = getState().data.dataStores;
+        if (!dataStores) {
+            return;
+        }
+        const dataSource = findDataSource(dataStores, dataStoreId, dataSourceId);
+        if (!dataSource || dataSource.metaInfoStatus !== 'init') {
+            return;
+        }
+
+        dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, undefined, 'loading'));
 
         function call(onProgress) {
-            return selectors.datasetAPISelector(getState()).getDataSourceTemporalCoverage(dataStoreId, dataSourceId, onProgress);
+            return selectors.datasetAPISelector(getState()).getDataSourceMetaInfo(dataStoreId, dataSourceId, onProgress);
         }
 
-        function action(temporalCoverage) {
-            dispatch(updateDataSourceTemporalCoverage(dataStoreId, dataSourceId, temporalCoverage));
+        function action(metaInfo: DatasetDescriptor) {
+            dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, metaInfo, 'ok'));
         }
 
-        callAPI({title: `Load temporal coverage for ${dataSourceId}`, dispatch, call, action});
+        function planB() {
+            dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, undefined, 'error'));
+        }
+
+        callAPI({title: `Loading meta data for ${dataSourceId}`, dispatch, call, action, planB});
     };
 }
 
-export function updateDataSourceTemporalCoverage(dataStoreId: string,
-                                                 dataSourceId: string,
-                                                 temporalCoverage: [string, string] | null): Action {
-    return {type: UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE, payload: {dataStoreId, dataSourceId, temporalCoverage}};
+export function updateDataSourceMetaInfo(dataStoreId: string,
+                                         dataSourceId: string,
+                                         metaInfo: DatasetDescriptor | undefined,
+                                         metaInfoStatus: 'loading' | 'ok' | 'error' = 'ok'): Action {
+    return {type: UPDATE_DATA_SOURCE_META_INFO, payload: {dataStoreId, dataSourceId, metaInfo, metaInfoStatus}};
 }
 
 export function openDataset(dataSourceId: string, args: any, updateLocalDataSources: boolean): ThunkAction {
@@ -2712,4 +2758,3 @@ function readDroppedFile(file: File, dispatch: Dispatch) {
         console.warn('Dropped file of unrecognized type: ', file.name);
     }
 }
-

src/renderer/components/DataSourceDetails.tsx

@@ -23,20 +23,24 @@ const DataSourceDetails: React.FC<IDataSourceDetailsProps> = ({dataSource}) => {
     if (!dataSource) {
         return null;
     }
+
     let metaInfoKeys;
-    if (dataSource.metaInfo) {
-        metaInfoKeys = Object.keys(dataSource.metaInfo).filter(key => key !== 'variables');
-    }
     let variables;
-    if (dataSource.metaInfo.variables) {
-        variables = dataSource.metaInfo.variables;
+
+    const metaInfo = dataSource.metaInfo;
+
+    if (metaInfo) {
+        metaInfoKeys = Object.keys(metaInfo).filter(key => key !== 'variables');
+        if (metaInfo.variables) {
+            variables = metaInfo.variables;
+        }
     }
 
     const details: DetailPart[] = [
         renderAbstract(dataSource),
         renderVariablesTable(variables),
-        renderMetaInfoTable(dataSource.metaInfo, metaInfoKeys),
-        renderMetaInfoLicences(dataSource.metaInfo),
+        renderMetaInfoTable(metaInfo, metaInfoKeys),
+        renderMetaInfoLicences(metaInfo),
     ];
 
     return (
@@ -84,18 +88,26 @@ function renderAbstract(dataSource: DataSourceState): DetailPart {
                 </div>
             );
         }
-        if (dataSource.temporalCoverage) {
+        if (dataSource.metaInfo && dataSource.metaInfo.time_range) {
+            let [start, end] = dataSource.metaInfo.time_range;
+            if (!start && !end) {
+                start = end = 'unknown';
+            } else if (!start) {
+                start = 'unknown';
+            } else if (!end) {
+                end = 'today';
+            }
             temporalCoverage = (
                 <div><h5>Temporal coverage</h5>
                     <table>
                         <tbody>
                         <tr>
                             <td>Start</td>
-                            <td className="user-selectable">{dataSource.temporalCoverage[0]}</td>
+                            <td className="user-selectable">{start}</td>
                         </tr>
                         <tr>
                             <td>End</td>
-                            <td className="user-selectable">{dataSource.temporalCoverage[1]}</td>
+                            <td className="user-selectable">{end}</td>
                         </tr>
                         </tbody>
                     </table>