From 9758ebfa650cf298099ca03a042e3518a134841d Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 12 Nov 2020 18:13:38 +0100 Subject: [PATCH 01/17] More generic generation of data source labels --- src/renderer/containers/DataSourcesPanel.tsx | 45 ++++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/renderer/containers/DataSourcesPanel.tsx b/src/renderer/containers/DataSourcesPanel.tsx index d16a78e..79674b4 100644 --- a/src/renderer/containers/DataSourcesPanel.tsx +++ b/src/renderer/containers/DataSourcesPanel.tsx @@ -552,19 +552,7 @@ class DataSourcesList extends React.PureComponent { // noinspection JSMethodCanBeStatic private renderTextIcon(dataSource: DataSourceState) { - const ecvId = ((dataSource.meta_info && dataSource.meta_info.cci_project) || '').toLowerCase(); - const ecvMetaItem = ECV_META.ecvs[ecvId]; - let backgroundColor, label; - if (ecvMetaItem) { - backgroundColor = ECV_META.colors[ecvMetaItem.color] || ecvMetaItem.color; - label = ecvMetaItem.label; - } - if (!backgroundColor) { - backgroundColor = ECV_META.colors["default"] || "#0BB7A0"; - } - if (!label) { - label = ecvId.substr(0, 3).toUpperCase() || '?'; - } + let {backgroundColor, label} = dataSourceToTextIconProps(dataSource); return
{label}
; } @@ -812,3 +800,34 @@ class DataSourceDetails extends React.PureComponent 1) { + ecvId = idParts[1].toLowerCase(); + } + } + const ecvMetaItem = ecvId && ECV_META.ecvs[ecvId]; + let backgroundColor; + if (ecvMetaItem) { + backgroundColor = ECV_META.colors[ecvMetaItem.color] || ecvMetaItem.color; + label = ecvMetaItem.label || label; + } + if (!backgroundColor) { + backgroundColor = ECV_META.colors["default"] || "#0BB7A0"; + } + if (!label) { + label = (ecvId && ecvId.substr(0, 3).toUpperCase()) || '?'; + } + return {backgroundColor, label}; +} + From bc4531db7658782410b98757692edf4c2465a01e Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 18 Mar 2021 13:49:20 +0100 Subject: [PATCH 02/17] Started 2.3.0-dev.0 --- CHANGES.md | 4 +++ Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 3 ++- src/renderer/state.ts | 61 ++++++++++++++++++++++++++++++++++++++++++- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- yarn.lock | 5 ++++ 8 files changed, 75 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d0dcf45..1d852a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +### Changes 2.3.0 (in development) + +* ... + ### Changes 2.2.3 * Fixed a problem that prevented using Matomo Analytics service. diff --git a/Dockerfile b/Dockerfile index 461f6b6..70fa56d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.2.3" +LABEL version="2.3.0-dev.0" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index 13020f5..bff6ab1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.2.3-{build} +version: 2.3.0-dev.0-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index c6f109f..d21a7a2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.2.3", + "version": "2.3.0-dev.0", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", @@ -50,6 +50,7 @@ "@types/dom4": "^1.5.20", "@types/geojson": "^1.0.6", "@types/jest": "^24.0.0", + "@types/json-schema": "^7.0.7", "@types/mocha": "^2.2.41", "@types/node": "^12.0.0", "@types/oboe": "^2.0.28", diff --git a/src/renderer/state.ts b/src/renderer/state.ts index 44e8d7c..0832377 100644 --- a/src/renderer/state.ts +++ b/src/renderer/state.ts @@ -1,5 +1,6 @@ import { IconName } from '@blueprintjs/core'; import { Feature, FeatureCollection, GeoJsonObject, Point } from 'geojson'; +import { JSONSchema7 } from 'json-schema'; import { KeycloakProfile } from 'keycloak-js'; import { SimpleStyle } from '../common/geojson-simple-style'; @@ -85,10 +86,68 @@ export interface DataStoreState { export type DataSourceVerificationFlags = "open" | "cache" | "map"; + +export interface DatasetDescriptor { + data_id: string; + type_specifier: string; + crs?: string; + bbox?: [number, number, number, number]; + spatial_res?: number; + time_range?: [string | null, string | null]; + time_period?: string; + variables?: { [var_name: string]: VariableDescriptor }; + + [attr_name: string]: any; +} + +export interface VariableDescriptor { + name: string; + units: string; + long_name: string; + standard_name: string; +} + +// TODO: (forman) use this in the future instead of DatasetDescriptor/VariableDescriptor +// {{{{{{{{{{{{ + +export interface DataDescriptor2 { + data_id: string; + type_specifier: string; + crs?: string | null; + bbox?: [number, number, number, number] | null; + time_range?: [string | null, string | null] | null; + time_period?: string | null; + open_params_schema?: JSONSchema7 | null; +} + +export interface GeoDataFrameDescriptor2 extends DataDescriptor2 { +} + +export interface DatasetDescriptor2 extends DataDescriptor2 { + spatial_res?: number | null; + dims?: { [dim_name: string]: number } | null; + coords?: { [var_name: string]: VariableDescriptor2 } | null; + data_vars?: { [var_name: string]: VariableDescriptor2 } | null; + attrs?: { [attr_name: string]: any } | null; +} + +export interface VariableDescriptor2 { + name: string; + dtype: string; + dims: string[]; + chunks?: number[] | null; + attrs?: { [attr_name: string]: any } | null; +} +// }}}}}}}}}}}} + + export interface DataSourceState { id: string; title?: string; - metaInfo: { [key: string]: any } | null; + // TODO: (forman) replace by descriptor in the future + metaInfo?: DatasetDescriptor; + // TODO: (forman) use this in the future instead of metaInfo + descriptor?: DatasetDescriptor2 | GeoDataFrameDescriptor2; typeSpecifier?: string | null; verificationFlags?: DataSourceVerificationFlags[] | null; temporalCoverage?: [string, string] | null; diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index dc8c19e..f97d469 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.2.3"; +const CATE_PWA_VERSION = "2.3.0-dev.0"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index b6e2b4a..b773ffb 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.2.3"; +export const CATE_APP_VERSION = "2.3.0-dev.0"; diff --git a/yarn.lock b/yarn.lock index a423bd9..9623094 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1772,6 +1772,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.5.tgz#dcce4430e64b443ba8945f0290fb564ad5bac6dd" integrity sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ== +"@types/json-schema@^7.0.7": + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" + integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA== + "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" From 8e5ae51fdbfd5b65f6955ab0c2a11e1e3e82d512 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 18 Mar 2021 13:52:59 +0100 Subject: [PATCH 03/17] FIX --- src/renderer/state.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/state.ts b/src/renderer/state.ts index 0832377..2346e2e 100644 --- a/src/renderer/state.ts +++ b/src/renderer/state.ts @@ -95,7 +95,7 @@ export interface DatasetDescriptor { spatial_res?: number; time_range?: [string | null, string | null]; time_period?: string; - variables?: { [var_name: string]: VariableDescriptor }; + variables?: VariableDescriptor[]; [attr_name: string]: any; } From b08f2550d48e739ff5bc80b75d9d92c389b20a91 Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Thu, 18 Mar 2021 16:03:23 +0100 Subject: [PATCH 04/17] describe additional attributes --- src/renderer/state.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/renderer/state.ts b/src/renderer/state.ts index 2346e2e..81ff631 100644 --- a/src/renderer/state.ts +++ b/src/renderer/state.ts @@ -96,15 +96,23 @@ export interface DatasetDescriptor { time_range?: [string | null, string | null]; time_period?: string; variables?: VariableDescriptor[]; + abstract?: string; + catalog_url?: string; + catalogue_url?: string; + cci_project?: string; + info_url?: string; + licences?: [string]; + title?: string; + uuid?: string; [attr_name: string]: any; } export interface VariableDescriptor { name: string; - units: string; - long_name: string; - standard_name: string; + units?: string; + long_name?: string; + standard_name?: string; } // TODO: (forman) use this in the future instead of DatasetDescriptor/VariableDescriptor From ba0c9cee658e00f1561fc8ad9109d45c18552c43 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 18 Mar 2021 18:08:26 +0100 Subject: [PATCH 05/17] Lazy loading data source metadata --- src/renderer/actions.spec.ts | 12 +-- src/renderer/actions.ts | 48 ++++++++--- src/renderer/components/DataSourceDetails.tsx | 32 ++++--- src/renderer/containers/DataSourcesPanel.tsx | 10 +-- src/renderer/reducers.ts | 7 +- src/renderer/selectors.ts | 9 +- src/renderer/state-util.ts | 14 ++++ src/renderer/state.ts | 6 +- src/renderer/webapi/apis/DatasetAPI.ts | 83 +++++++------------ 9 files changed, 121 insertions(+), 100 deletions(-) diff --git a/src/renderer/actions.spec.ts b/src/renderer/actions.spec.ts index 0e9debc..5c000a0 100644 --- a/src/renderer/actions.spec.ts +++ b/src/renderer/actions.spec.ts @@ -122,7 +122,7 @@ describe('Actions', () => { ]); }); - it('updateDataSourceTemporalCoverage', () => { + it('updateDataSourceMetaInfo', () => { dispatch(actions.updateDataStores( [ {id: 'local-1'}, @@ -133,9 +133,10 @@ describe('Actions', () => { {id: 'fileset-1'}, {id: 'fileset-2'} ] as any)); - dispatch(actions.updateDataSourceTemporalCoverage('local-2', - 'fileset-1', - ['2010-01-01', '2014-12-30'])); + dispatch(actions.updateDataSourceMetaInfo('local-2', + 'fileset-1', + {'data_id': 'x', 'type_specifier': 'y'}, + 'ok')); expect(getState().data.dataStores).to.deep.equal( [ {id: 'local-1'}, @@ -143,7 +144,8 @@ describe('Actions', () => { id: 'local-2', dataSources: [ { id: 'fileset-1', - temporalCoverage: ['2010-01-01', '2014-12-30'] + metaInfo: {'data_id': 'x', 'type_specifier': 'y'}, + metaInfoStatus: 'ok', }, {id: 'fileset-2'} ] diff --git a/src/renderer/actions.ts b/src/renderer/actions.ts index be19198..d6f24c5 100644 --- a/src/renderer/actions.ts +++ b/src/renderer/actions.ts @@ -41,7 +41,7 @@ import * as selectors from './selectors'; import { BackendConfigState, ColorMapCategoryState, - ControlState, + ControlState, DatasetDescriptor, DataSourceState, DataStoreState, GeographicPosition, @@ -67,6 +67,7 @@ import { } from './state'; import { AUTO_LAYER_ID, + findDataSource, findResourceByName, genSimpleId, getCsvUrl, @@ -885,7 +886,7 @@ export function setSelectedPlacemarkId(selectedPlacemarkId: string | null): Acti export const UPDATE_DATA_STORES = 'UPDATE_DATA_STORES'; export const UPDATE_DATA_SOURCES = 'UPDATE_DATA_SOURCES'; -export const UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE = 'UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE'; +export const UPDATE_DATA_SOURCE_META_INFO = 'UPDATE_DATA_SOURCE_META_INFO'; /** * Asynchronously load the available Cate data stores. @@ -987,33 +988,54 @@ export function setSelectedDataStoreIdImpl(selectedDataStoreId: string | null) { return updateSessionState({selectedDataStoreId}); } -export function setSelectedDataSourceId(selectedDataSourceId: string | null) { - return updateSessionState({selectedDataSourceId}); +export function setSelectedDataSourceId(selectedDataSourceId: string): ThunkAction { + return (dispatch: Dispatch, getState: GetState) => { + dispatch(updateSessionState({selectedDataSourceId})); + const dataStoreId = getState().session.selectedDataStoreId; + if (dataStoreId && selectedDataSourceId) { + dispatch(loadDataSourceMetaInfo(dataStoreId, selectedDataSourceId)); + } + } } export function setDataSourceFilterExpr(dataSourceFilterExpr: string) { return updateSessionState({dataSourceFilterExpr}); } -export function loadTemporalCoverage(dataStoreId: string, dataSourceId: string): ThunkAction { +export function loadDataSourceMetaInfo(dataStoreId: string, dataSourceId: string): ThunkAction { return (dispatch: Dispatch, getState: GetState) => { + const dataStores = getState().data.dataStores; + if (!dataStores) { + return; + } + const dataSource = findDataSource(dataStores, dataStoreId, dataSourceId); + if (!dataSource || dataSource.metaInfoStatus !== 'init') { + return; + } + + dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, undefined, 'loading')); function call(onProgress) { - return selectors.datasetAPISelector(getState()).getDataSourceTemporalCoverage(dataStoreId, dataSourceId, onProgress); + return selectors.datasetAPISelector(getState()).getDataSourceMetaInfo(dataStoreId, dataSourceId, onProgress); } - function action(temporalCoverage) { - dispatch(updateDataSourceTemporalCoverage(dataStoreId, dataSourceId, temporalCoverage)); + function action(metaInfo: DatasetDescriptor) { + dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, metaInfo, 'ok')); + } + + function planB() { + dispatch(updateDataSourceMetaInfo(dataStoreId, dataSourceId, undefined, 'error')); } - callAPI({title: `Load temporal coverage for ${dataSourceId}`, dispatch, call, action}); + callAPI({title: `Loading meta data for ${dataSourceId}`, dispatch, call, action, planB}); }; } -export function updateDataSourceTemporalCoverage(dataStoreId: string, - dataSourceId: string, - temporalCoverage: [string, string] | null): Action { - return {type: UPDATE_DATA_SOURCE_TEMPORAL_COVERAGE, payload: {dataStoreId, dataSourceId, temporalCoverage}}; +export function updateDataSourceMetaInfo(dataStoreId: string, + dataSourceId: string, + metaInfo: DatasetDescriptor | undefined, + metaInfoStatus: 'loading' | 'ok' | 'error' = 'ok'): Action { + return {type: UPDATE_DATA_SOURCE_META_INFO, payload: {dataStoreId, dataSourceId, metaInfo, metaInfoStatus}}; } export function openDataset(dataSourceId: string, args: any, updateLocalDataSources: boolean): ThunkAction { diff --git a/src/renderer/components/DataSourceDetails.tsx b/src/renderer/components/DataSourceDetails.tsx index 4c19138..c6d9685 100644 --- a/src/renderer/components/DataSourceDetails.tsx +++ b/src/renderer/components/DataSourceDetails.tsx @@ -23,20 +23,24 @@ const DataSourceDetails: React.FC = ({dataSource}) => { if (!dataSource) { return null; } + let metaInfoKeys; - if (dataSource.metaInfo) { - metaInfoKeys = Object.keys(dataSource.metaInfo).filter(key => key !== 'variables'); - } let variables; - if (dataSource.metaInfo.variables) { - variables = dataSource.metaInfo.variables; + + const metaInfo = dataSource.metaInfo; + + if (metaInfo) { + metaInfoKeys = Object.keys(metaInfo).filter(key => key !== 'variables'); + if (metaInfo.variables) { + variables = metaInfo.variables; + } } const details: DetailPart[] = [ renderAbstract(dataSource), renderVariablesTable(variables), - renderMetaInfoTable(dataSource.metaInfo, metaInfoKeys), - renderMetaInfoLicences(dataSource.metaInfo), + renderMetaInfoTable(metaInfo, metaInfoKeys), + renderMetaInfoLicences(metaInfo), ]; return ( @@ -84,18 +88,26 @@ function renderAbstract(dataSource: DataSourceState): DetailPart { ); } - if (dataSource.temporalCoverage) { + if (dataSource.metaInfo && dataSource.metaInfo.time_range) { + let [start, end] = dataSource.metaInfo.time_range; + if (!start && !end) { + start = end = 'unknown'; + } else if (!start) { + start = 'unknown'; + } else if (!end) { + end = 'today'; + } temporalCoverage = (
Temporal coverage
- + - +
Start{dataSource.temporalCoverage[0]}{start}
End{dataSource.temporalCoverage[1]}{end}
diff --git a/src/renderer/containers/DataSourcesPanel.tsx b/src/renderer/containers/DataSourcesPanel.tsx index 7b8e00a..78fecdd 100644 --- a/src/renderer/containers/DataSourcesPanel.tsx +++ b/src/renderer/containers/DataSourcesPanel.tsx @@ -97,7 +97,7 @@ interface IDataSourcesPanelDispatch { updateSessionState(sessionState: any): void; - loadTemporalCoverage(dataStoreId: string, dataSourceId: string): void; + loadDataSourceMetaInfo(dataStoreId: string, dataSourceId: string): void; showDialog(dialogId: string): void; @@ -111,7 +111,6 @@ const mapDispatchToProps = { setSessionState: actions.setSessionProperty, setControlState: actions.setControlProperty, updateSessionState: actions.updateSessionState, - loadTemporalCoverage: actions.loadTemporalCoverage, showDialog: actions.showDialog, hideDialog: actions.hideDialog, }; @@ -260,16 +259,9 @@ class DataSourcesPanel extends React.Component { const newDataSources = [...dataStore.dataSources]; - const dataSourceId = action.payload.dataSourceId; - const temporalCoverage = action.payload.temporalCoverage; + const {dataSourceId, metaInfo, metaInfoStatus} = action.payload; const dataSourceIndex = newDataSources.findIndex(dataSource => dataSource.id === dataSourceId); if (dataSourceIndex < 0) { throw Error('illegal data source ID: ' + dataSourceId); } const oldDataSource = newDataSources[dataSourceIndex]; - newDataSources[dataSourceIndex] = {...oldDataSource, temporalCoverage}; + newDataSources[dataSourceIndex] = {...oldDataSource, metaInfo, metaInfoStatus}; return newDataSources; }); } diff --git a/src/renderer/selectors.ts b/src/renderer/selectors.ts index fe45d73..db34967 100644 --- a/src/renderer/selectors.ts +++ b/src/renderer/selectors.ts @@ -440,12 +440,15 @@ export const selectedDataSourceSelector = createSelector( selectedDataSourceSelector, - (selectedDataSource: DataSourceState): [string, string] | null => { - return selectedDataSource ? selectedDataSource.temporalCoverage : null; + (selectedDataSource: DataSourceState): [string | null, string | null] | null => { + return (selectedDataSource + && selectedDataSource.metaInfo + && selectedDataSource.metaInfo.time_range) || null; } ); diff --git a/src/renderer/state-util.ts b/src/renderer/state-util.ts index 6e1869e..eee2b60 100644 --- a/src/renderer/state-util.ts +++ b/src/renderer/state-util.ts @@ -251,6 +251,20 @@ export function findOperation(operations: OperationState[], name: string): Opera return operations && operations.find(op => op.qualifiedName === name || op.name === name); } +export function findDataSource(dataStores: DataStoreState[], + dataStoreId: string, + dataSourceId: string): DataSourceState | null { + const dataStore = dataStores && dataStores.find(dataStore => dataStore.id === dataStoreId); + if (dataStore) { + const dataSource = dataStore.dataSources + && dataStore.dataSources.find(dataSource => dataSource.id === dataSourceId); + if (dataSource) { + return dataSource; + } + } + return null; +} + export function findVariableIndexCoordinates(resources: ResourceState[], ref: VariableDataRefState): any[] { const resource = findResourceById(resources, ref.resId); if (!resource) { diff --git a/src/renderer/state.ts b/src/renderer/state.ts index 81ff631..d0adaea 100644 --- a/src/renderer/state.ts +++ b/src/renderer/state.ts @@ -104,7 +104,7 @@ export interface DatasetDescriptor { licences?: [string]; title?: string; uuid?: string; - + // Anything else: [attr_name: string]: any; } @@ -154,8 +154,10 @@ export interface DataSourceState { title?: string; // TODO: (forman) replace by descriptor in the future metaInfo?: DatasetDescriptor; + metaInfoStatus: 'init' | 'loading' | 'ok' | 'error'; // TODO: (forman) use this in the future instead of metaInfo - descriptor?: DatasetDescriptor2 | GeoDataFrameDescriptor2; + // descriptor?: DatasetDescriptor2 | GeoDataFrameDescriptor2; + // descriptorStatus: 'init' | 'loading' | 'ok' | 'error'; typeSpecifier?: string | null; verificationFlags?: DataSourceVerificationFlags[] | null; temporalCoverage?: [string, string] | null; diff --git a/src/renderer/webapi/apis/DatasetAPI.ts b/src/renderer/webapi/apis/DatasetAPI.ts index fd3e5e6..e59d77c 100644 --- a/src/renderer/webapi/apis/DatasetAPI.ts +++ b/src/renderer/webapi/apis/DatasetAPI.ts @@ -1,51 +1,8 @@ -import { DataSourceState, DataStoreState, DimSizes } from '../../state'; +import { DatasetDescriptor, DataSourceState, DataStoreState, DimSizes } from '../../state'; import { JobProgress, JobPromise } from '../Job'; import { WebAPIClient } from '../WebAPIClient'; -function responseToTemporalCoverage(response: any): [string, string] | null { - if (response && response.temporal_coverage_start && response.temporal_coverage_end) { - return [response.temporal_coverage_start, response.temporal_coverage_end]; - } - return null; -} - -function addVerificationFlagsForTesting(dataSources: DataSourceState[]): DataSourceState[] { - return dataSources.map((ds, i) => { - // console.debug(`dataSources[${i}]:`, ds); - return ds; - /* - if (i === 0) { - return { - ...ds, - typeSpecifier: 'dataset', - verificationFlags: null - } - } else if (i === 1) { - return { - ...ds, - typeSpecifier: 'dataset', - verificationFlags: ['open'] - } - } else if (i === 2) { - return { - ...ds, - typeSpecifier: 'dataset', - verificationFlags: ['open', 'map'] - } - } else if (i === 3) { - return { - ...ds, - typeSpecifier: 'dataset', - verificationFlags: ['open', 'map', 'cache'] - } - } else { - return ds; - } - */ - }); -} - export class DatasetAPI { private readonly webAPIClient: WebAPIClient; @@ -59,19 +16,18 @@ export class DatasetAPI { getDataSources(dataStoreId: string, onProgress: (progress: JobProgress) => void): JobPromise { - return this.webAPIClient - .call('get_data_sources', - [dataStoreId], - onProgress, - addVerificationFlagsForTesting); + return this.webAPIClient.call('get_data_sources', + [dataStoreId], + onProgress, + DatasetAPI.responseToDataSources); } - getDataSourceTemporalCoverage(dataStoreId: string, dataSourceId: string, - onProgress: (progress: JobProgress) => void): JobPromise<[string, string] | null> { - return this.webAPIClient.call('get_data_source_temporal_coverage', + getDataSourceMetaInfo(dataStoreId: string, dataSourceId: string, + onProgress: (progress: JobProgress) => void): JobPromise { + return this.webAPIClient.call('get_data_source_meta_info', [dataStoreId, dataSourceId], - onProgress, responseToTemporalCoverage - ); + onProgress, + DatasetAPI.responseToMetaInfo); } addLocalDataSource(dataSourceId: string, filePathPattern: string, @@ -94,4 +50,23 @@ export class DatasetAPI { indexers: DimSizes): JobPromise<{ [varName: string]: number } | null> { return this.webAPIClient.call('extract_pixel_values', [baseDir, source, point, indexers]); } + + static responseToDataSources(dataSources: any[]): DataSourceState[] { + // noinspection JSUnusedLocalSymbols + return dataSources.map((dataSource, i): DataSourceState => { + console.debug(`dataSources[${i}]:`, dataSource); + return { + id: dataSource['id'], + title: dataSource['title'], + typeSpecifier: dataSource['type_specifier'] || 'dataset', + verificationFlags: dataSource['verification_flags'], + metaInfoStatus: 'init', + }; + }); + } + + static responseToMetaInfo(response: any): DatasetDescriptor { + return response as DatasetDescriptor; + } } + From bb9beddb5f43521c51590a3b8a6fbc11f953e361 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 18 Mar 2021 18:14:55 +0100 Subject: [PATCH 06/17] Now generating correct ECV avatars --- src/renderer/components/DataSourceItem.tsx | 48 +++++++++++++++------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/renderer/components/DataSourceItem.tsx b/src/renderer/components/DataSourceItem.tsx index 6b52c09..40b71a6 100644 --- a/src/renderer/components/DataSourceItem.tsx +++ b/src/renderer/components/DataSourceItem.tsx @@ -36,20 +36,7 @@ interface DataSourceItemProps { const DataSourceItem: React.FC = ({dataSource, showDataSourceIDs}) => { const metaInfo = dataSource.metaInfo; - // TODO: get rid of render logic here - const ecvId = ((metaInfo && metaInfo.cci_project) || '').toLowerCase(); - const ecvMetaItem = ECV_META.ecvs[ecvId]; - let backgroundColor, label; - if (ecvMetaItem) { - backgroundColor = ECV_META.colors[ecvMetaItem.color] || ecvMetaItem.color; - label = ecvMetaItem.label; - } - if (!backgroundColor) { - backgroundColor = ECV_META.colors["default"] || "#0BB7A0"; - } - if (!label) { - label = ecvId.substr(0, 3).toUpperCase() || '?'; - } + const {backgroundColor, label} = dataSourceToTextIconProps(dataSource); const icon =
{label}
; const title = dataSource.title || (metaInfo && metaInfo.title); @@ -81,4 +68,35 @@ const DataSourceItem: React.FC = ({dataSource, showDataSour } -export default DataSourceItem; \ No newline at end of file +export default DataSourceItem; + + +function dataSourceToTextIconProps(dataSource: DataSourceState) { + let ecvId; + let label; + if (dataSource.title) { + ecvId = dataSource.title.split(' ', 1)[0].toLowerCase(); + label = dataSource.title.substr(0, 3).toUpperCase(); + } + if (!ecvId || !ECV_META.ecvs[ecvId]) { + // This is a CCI-store specific hack + const idParts = dataSource.id.split('.', 2); + if (idParts.length > 1) { + ecvId = idParts[1].toLowerCase(); + } + } + const ecvMetaItem = ecvId && ECV_META.ecvs[ecvId]; + let backgroundColor; + if (ecvMetaItem) { + backgroundColor = ECV_META.colors[ecvMetaItem.color] || ecvMetaItem.color; + label = ecvMetaItem.label || label; + } + if (!backgroundColor) { + backgroundColor = ECV_META.colors["default"] || "#0BB7A0"; + } + if (!label) { + label = (ecvId && ecvId.substr(0, 3).toUpperCase()) || '?'; + } + return {backgroundColor, label}; +} + From ad50c47d4f9e52e8fbb651d95883e2c6d43cc93e Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 18 Mar 2021 18:44:16 +0100 Subject: [PATCH 07/17] Update --- CHANGES.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1d852a0..e0487c1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,12 @@ ### Changes 2.3.0 (in development) -* ... +* Optimisations in the DATA SOURCES panel (that have been enabled by + using [xcube](https://xcube.readthedocs.io/) in the backend): + - Initalising the "ESA CCI Open Data Portal" data store + is now accellerated by a magnitude. + - Local caching of remote data sources when opening datasets + is now much faster and more reliable. + - Ability to add more data stores has been greatly improved. ### Changes 2.2.3 From e48ff9a1e593c53a21ed3b76f17e6e7b00a63f3e Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 19 Mar 2021 14:42:05 +0100 Subject: [PATCH 08/17] We now obtain Cate Hub's status information from GitHub --- CHANGES.md | 3 + src/renderer/actions.ts | 7 +- src/renderer/containers/AppModePage.tsx | 42 +++++++----- src/renderer/containers/AppRouter.tsx | 43 +++++++++++++ src/renderer/initial-state.ts | 1 + src/renderer/main.tsx | 64 ++++++++----------- src/renderer/reducers.ts | 6 ++ src/renderer/state.ts | 7 ++ .../webapi/apis/ServiceProvisionAPI.ts | 3 +- 9 files changed, 118 insertions(+), 58 deletions(-) create mode 100644 src/renderer/containers/AppRouter.tsx diff --git a/CHANGES.md b/CHANGES.md index e0487c1..12d6a45 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ is now much faster and more reliable. - Ability to add more data stores has been greatly improved. +* We now obtain Cate Hub's status information from a dedicated GitHub + repository [cate-status](https://github.com/CCI-Tools/cate-status). + ### Changes 2.2.3 * Fixed a problem that prevented using Matomo Analytics service. diff --git a/src/renderer/actions.ts b/src/renderer/actions.ts index d6f24c5..8f74d31 100644 --- a/src/renderer/actions.ts +++ b/src/renderer/actions.ts @@ -44,7 +44,7 @@ import { ControlState, DatasetDescriptor, DataSourceState, DataStoreState, - GeographicPosition, + GeographicPosition, HubStatus, ImageStatisticsState, LayerState, MessageState, @@ -138,6 +138,7 @@ export const SET_WEBAPI_STATUS = 'SET_WEBAPI_STATUS'; export const SET_WEBAPI_CLIENT = 'SET_WEBAPI_CLIENT'; export const SET_WEBAPI_SERVICE_URL = 'SET_WEBAPI_SERVICE_URL'; export const SET_WEBAPI_SERVICE_INFO = 'SET_WEBAPI_SERVICE_INFO'; +export const UPDATE_HUB_STATUS = 'UPDATE_HUB_STATUS'; export const UPDATE_DIALOG_STATE = 'UPDATE_DIALOG_STATE'; export const UPDATE_TASK_STATE = 'UPDATE_TASK_STATE'; export const REMOVE_TASK_STATE = 'REMOVE_TASK_STATE'; @@ -354,6 +355,10 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction { }; } +export function updateHubStatus(hubStatus: HubStatus): Action { + return { type: UPDATE_HUB_STATUS, payload: hubStatus}; +} + export function updateInitialState(initialState: Object): Action { return {type: UPDATE_INITIAL_STATE, payload: initialState}; } diff --git a/src/renderer/containers/AppModePage.tsx b/src/renderer/containers/AppModePage.tsx index a48990a..7ed274a 100644 --- a/src/renderer/containers/AppModePage.tsx +++ b/src/renderer/containers/AppModePage.tsx @@ -9,13 +9,10 @@ import GdprBanner from './GdprBanner'; import { TermsAndConditions } from '../components/TermsAndConditions'; import { DEFAULT_SERVICE_URL } from '../initial-state'; import { State } from '../state'; - import cateIcon from '../resources/cate-icon-512.png'; import VersionTags from './VersionTags'; -const maintenanceReason: string | undefined = process.env.REACT_APP_CATEHUB_MAINTENANCE; - const CENTER_DIV_STYLE: CSSProperties = { display: 'flex', alignItems: 'center', @@ -40,14 +37,24 @@ interface IDispatch { } interface IAppModePageProps { + hubOk?: boolean; + hubMessage?: string | null; } // noinspection JSUnusedLocalSymbols function mapStateToProps(state: State): IAppModePageProps { - return {}; + const hubStatus = state.communication.hubStatus; + return { + hubOk: !!hubStatus && hubStatus.status === 'ok', + hubMessage: hubStatus && hubStatus.message, + }; } -const _AppModePage: React.FC = () => { +const _AppModePage: React.FC = ( + { + hubOk, + hubMessage + }) => { const history = useHistory(); const [, keycloakInitialized] = useKeycloak(); @@ -96,21 +103,24 @@ const _AppModePage: React.FC = () => { {'Cate
- {maintenanceReason ? ( -
-  {maintenanceReason} + {hubOk && ( +
+ Please select a Cate service provision mode
- ) : ( -
- Please select a Cate service provision mode -
- )} + )} + + {hubMessage && ( +
+  {hubMessage} +
+ )} +
}> @@ -121,7 +131,7 @@ const _AppModePage: React.FC = () => { I agree to the  diff --git a/src/renderer/containers/AppRouter.tsx b/src/renderer/containers/AppRouter.tsx new file mode 100644 index 0000000..d708830 --- /dev/null +++ b/src/renderer/containers/AppRouter.tsx @@ -0,0 +1,43 @@ +import * as React from 'react'; +import { connect } from 'react-redux'; +import { BrowserRouter as Router, Redirect, Route, Switch } from "react-router-dom"; + +import { State, HubStatus } from '../state'; +import AppMainPageForHub from '../containers/AppMainPageForHub'; +import AppMainPageForSA from './AppMainPageForSA'; +import AppModePage from './AppModePage'; + + +interface IAppRouterProps { + hubStatus: HubStatus | null; +} + +function mapStateToProps(state: State): IAppRouterProps { + return { + hubStatus: state.communication.hubStatus + }; +} + +const AppRouter: React.FC = ({hubStatus}) => { + return ( + + + + + + + + + + { + (hubStatus === null || hubStatus.status !== 'ok') + ? () + : () + } + + + + ); +} + +export default connect(mapStateToProps)(AppRouter); diff --git a/src/renderer/initial-state.ts b/src/renderer/initial-state.ts index f8f9e7b..4130edc 100644 --- a/src/renderer/initial-state.ts +++ b/src/renderer/initial-state.ts @@ -143,6 +143,7 @@ export const INITIAL_COMMUNICATION_STATE: CommunicationState = { webAPIStatus: null, webAPIServiceInfo: null, webAPIClient: null, + hubStatus: null, userProfile: null, tasks: {}, }; diff --git a/src/renderer/main.tsx b/src/renderer/main.tsx index 55797ca..ceeeaea 100644 --- a/src/renderer/main.tsx +++ b/src/renderer/main.tsx @@ -1,12 +1,5 @@ import * as React from 'react'; import * as ReactDOM from 'react-dom'; -import { - BrowserRouter as Router, - // HashRouter as Router, - Switch, - Redirect, - Route, -} from "react-router-dom"; import { Provider as StoreProvider } from 'react-redux'; import { applyMiddleware, createStore, Middleware, Store } from 'redux'; import { createLogger } from 'redux-logger'; @@ -15,16 +8,14 @@ import Keycloak, { KeycloakInitOptions, KeycloakConfig } from 'keycloak-js' import { KeycloakProvider } from '@react-keycloak/web' import * as actions from './actions' -import AppMainPageForHub from './containers/AppMainPageForHub' -import AppMainPageForSA from './containers/AppMainPageForSA' -import AppModePage from './containers/AppModePage'; +import AppRouter from './containers/AppRouter'; import { stateReducer } from './reducers'; import { State } from './state'; import { isElectron } from './electron'; +import { getEndpointUrl } from './webapi/apis/ServiceProvisionAPI'; const keycloak = Keycloak(getKeycloakConfig()); -const maintenanceReason: string | undefined = process.env.REACT_APP_CATEHUB_MAINTENANCE; const keycloakProviderInitConfig: KeycloakInitOptions = { onLoad: 'check-sso', @@ -63,40 +54,36 @@ export function main() { console.debug('onKeycloakTokens', tokens); } + // Fetch hub status from GitHub + const deployment = getEndpointUrl().includes('stage') ? 'development' : 'production'; + fetch(`https://raw.githubusercontent.com/CCI-Tools/cate-status/main/${deployment}.json`, + {mode: 'cors'}) + .then(response => + response.json()) + .then(hubStatus => + store.dispatch(actions.updateHubStatus( + {...hubStatus, deployment}))) + .catch(e => console.error(e)); + ReactDOM.render( ( - - - - - - - - - { - maintenanceReason - ? () - : () - } - - - - - - - - + + + + + ), document.getElementById('root') ); if (!isElectron()) { - // Dektop-PWA app install, see https://web.dev/customize-install/ + // + // Desktop-PWA app install, see https://web.dev/customize-install/ // window.addEventListener('beforeinstallprompt', (event: Event) => { // Update UI notify the user they can install the PWA @@ -115,4 +102,3 @@ function getKeycloakConfig(): KeycloakConfig { } return {realm, url, clientId}; } - diff --git a/src/renderer/reducers.ts b/src/renderer/reducers.ts index 207cb9b..b22a14c 100644 --- a/src/renderer/reducers.ts +++ b/src/renderer/reducers.ts @@ -841,6 +841,12 @@ const communicationReducer = (state: CommunicationState = INITIAL_COMMUNICATION_ const webAPIServiceInfo = action.payload.webAPIServiceInfo; return {...state, webAPIServiceInfo}; } + case actions.UPDATE_HUB_STATUS: { + return { + ...state, + hubStatus: action.payload + } + } case actions.UPDATE_TASK_STATE: return { ...state, diff --git a/src/renderer/state.ts b/src/renderer/state.ts index d0adaea..a39e71d 100644 --- a/src/renderer/state.ts +++ b/src/renderer/state.ts @@ -67,6 +67,12 @@ export interface WebAPIServiceInfo { hostOS?: HostOS; } +export interface HubStatus { + status: "ok" | "offline"; + message?: string; + deployment: "development" | "production"; +} + export interface DataStoreNotice { id: string; title: string; @@ -727,6 +733,7 @@ export interface CommunicationState { webAPIStatus: WebAPIStatus | null; webAPIClient: WebAPIClient | null; userProfile: KeycloakProfile | null; + hubStatus: HubStatus | null; // A map that stores the current state of any tasks (e.g. data fetch jobs from remote API) given a jobId tasks: { [jobId: number]: TaskState; }; } diff --git a/src/renderer/webapi/apis/ServiceProvisionAPI.ts b/src/renderer/webapi/apis/ServiceProvisionAPI.ts index 09411f0..91c8991 100644 --- a/src/renderer/webapi/apis/ServiceProvisionAPI.ts +++ b/src/renderer/webapi/apis/ServiceProvisionAPI.ts @@ -89,8 +89,7 @@ export class ServiceProvisionAPI { } } - -function getEndpointUrl(): string { +export function getEndpointUrl(): string { if (process.env.REACT_APP_CATEHUB_ENDPOINT) { return process.env.REACT_APP_CATEHUB_ENDPOINT; } else if (window.location.host.indexOf('stage') >= 0) { From 6c9e795d3a58ea89b97b6f3b1bcf7ee6f98f870d Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 19 Mar 2021 17:52:27 +0100 Subject: [PATCH 09/17] Fix --- Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 2 +- src/renderer/containers/AppRouter.tsx | 5 ++++- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70fa56d..75300e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.3.0-dev.0" +LABEL version="2.3.0-dev.1" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index bff6ab1..d4a8342 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.3.0-dev.0-{build} +version: 2.3.0-dev.1-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index d21a7a2..dc8dfe2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.3.0-dev.0", + "version": "2.3.0-dev.1", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", diff --git a/src/renderer/containers/AppRouter.tsx b/src/renderer/containers/AppRouter.tsx index d708830..a6e76e0 100644 --- a/src/renderer/containers/AppRouter.tsx +++ b/src/renderer/containers/AppRouter.tsx @@ -30,7 +30,10 @@ const AppRouter: React.FC = ({hubStatus}) => { { - (hubStatus === null || hubStatus.status !== 'ok') + // It should read + // (hubStatus === null || hubStatus.status !== 'ok') + // but this will always bring us back to "/" after login :( + (hubStatus !== null && hubStatus.status !== 'ok') ? () : () } diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index f97d469..ad0547f 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.3.0-dev.0"; +const CATE_PWA_VERSION = "2.3.0-dev.1"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index b773ffb..5edc18a 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.3.0-dev.0"; +export const CATE_APP_VERSION = "2.3.0-dev.1"; From 247f10498d038182ad62c1766d4b8fcbf80c6898 Mon Sep 17 00:00:00 2001 From: dzelge Date: Mon, 22 Mar 2021 14:37:23 +0100 Subject: [PATCH 10/17] Corrected performServiceOp. The hub's response object has changed. --- .env | 10 +++++----- src/renderer/webapi/apis/ServiceProvisionAPI.ts | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 17934f5..807e3ca 100644 --- a/.env +++ b/.env @@ -14,8 +14,8 @@ REACT_APP_KEYCLOAK_CLIENT_ID=cate-webui # REACT_APP_MAX_NUM_USERS=50 # TODO: document me -# REACT_APP_CATEHUB_ENDPOINT=https://stage.catehub.brockmann-consult.de -REACT_APP_CATEHUB_ENDPOINT=http://localhost:8000 -REACT_APP_CATEHUB_WEBAPI_MANAG_PATH=/user/{username}/webapi -REACT_APP_CATEHUB_WEBAPI_CLOSE_PATH=/user/{username}/webapi/shutdown -REACT_APP_CATEHUB_WEBAPI_COUNT_PATH=/webapi/count \ No newline at end of file +REACT_APP_CATEHUB_ENDPOINT=https://stage.catehub.climate.esa.int/api/v2 +#REACT_APP_CATEHUB_ENDPOINT=http://localhost:8080/api/v2 +REACT_APP_CATEHUB_WEBAPI_MANAG_PATH=/users/{username}/webapis +REACT_APP_CATEHUB_WEBAPI_CLOSE_PATH=/users/{username}/webapis +REACT_APP_CATEHUB_WEBAPI_COUNT_PATH=/webapis \ No newline at end of file diff --git a/src/renderer/webapi/apis/ServiceProvisionAPI.ts b/src/renderer/webapi/apis/ServiceProvisionAPI.ts index 91c8991..ddb25c3 100644 --- a/src/renderer/webapi/apis/ServiceProvisionAPI.ts +++ b/src/renderer/webapi/apis/ServiceProvisionAPI.ts @@ -82,10 +82,8 @@ export class ServiceProvisionAPI { throw HttpError.fromResponse(response); } const jsonObject = await response.json(); - if (jsonObject.status !== 'ok') { - throw new Error(jsonObject.message); - } - return jsonObject.result as T; + + return jsonObject as T; } } From 8d616794cc484b790a6af99b841edb8d9119f312 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Mon, 22 Mar 2021 15:31:45 +0100 Subject: [PATCH 11/17] Adapted to changed cate-hub API. --- CHANGES.md | 5 +++++ src/renderer/webapi/apis/ServiceProvisionAPI.ts | 9 ++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 12d6a45..a78b46b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,11 @@ * We now obtain Cate Hub's status information from a dedicated GitHub repository [cate-status](https://github.com/CCI-Tools/cate-status). +* Adapted to changed cate-hub API. (An API response no longer has + `status` and `result` properties, instead a response _is_ the result + and the response status is represented by the HTTP response code.) + + ### Changes 2.2.3 * Fixed a problem that prevented using Matomo Analytics service. diff --git a/src/renderer/webapi/apis/ServiceProvisionAPI.ts b/src/renderer/webapi/apis/ServiceProvisionAPI.ts index 91c8991..c4accb4 100644 --- a/src/renderer/webapi/apis/ServiceProvisionAPI.ts +++ b/src/renderer/webapi/apis/ServiceProvisionAPI.ts @@ -34,6 +34,9 @@ interface CountResult { running_pods: number; } +/** + * Represents the cate-hub API. + */ export class ServiceProvisionAPI { /** @@ -81,11 +84,7 @@ export class ServiceProvisionAPI { if (!response.ok) { throw HttpError.fromResponse(response); } - const jsonObject = await response.json(); - if (jsonObject.status !== 'ok') { - throw new Error(jsonObject.message); - } - return jsonObject.result as T; + return response.json(); } } From 0a7a7e2700844b4b342832aee18113e62c199536 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Mon, 22 Mar 2021 15:32:51 +0100 Subject: [PATCH 12/17] Increased version. --- Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 2 +- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 75300e9..06d6848 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.3.0-dev.1" +LABEL version="2.3.0-dev.2" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index d4a8342..b0645a3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.3.0-dev.1-{build} +version: 2.3.0-dev.2-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index dc8dfe2..41fb604 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.3.0-dev.1", + "version": "2.3.0-dev.2", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index ad0547f..9753dab 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.3.0-dev.1"; +const CATE_PWA_VERSION = "2.3.0-dev.2"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index 5edc18a..82f8903 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.3.0-dev.1"; +export const CATE_APP_VERSION = "2.3.0-dev.2"; From 870023c232e7fdb807d55cd5373bf4bee4e2c8c4 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Wed, 24 Mar 2021 18:00:02 +0100 Subject: [PATCH 13/17] Zarr store should not cache! --- CHANGES.md | 5 ++++- Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 2 +- src/renderer/selectors.ts | 10 ++++++++-- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a78b46b..dd0c9ab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,10 +2,13 @@ * Optimisations in the DATA SOURCES panel (that have been enabled by using [xcube](https://xcube.readthedocs.io/) in the backend): - - Initalising the "ESA CCI Open Data Portal" data store + - Initalising the "CCI Open Data Portal" data store is now accellerated by a magnitude. - Local caching of remote data sources when opening datasets is now much faster and more reliable. + - Added new experimental store "CCI Zarr Store" that offers + selected CCI datasets that have been converted to Zarr format + and are read from JASMIN object storage. - Ability to add more data stores has been greatly improved. * We now obtain Cate Hub's status information from a dedicated GitHub diff --git a/Dockerfile b/Dockerfile index 06d6848..6ca358f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.3.0-dev.2" +LABEL version="2.3.0-dev.3" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index b0645a3..437b275 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.3.0-dev.2-{build} +version: 2.3.0-dev.3-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index 41fb604..d72561e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.3.0-dev.2", + "version": "2.3.0-dev.3", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", diff --git a/src/renderer/selectors.ts b/src/renderer/selectors.ts index db34967..12912f9 100644 --- a/src/renderer/selectors.ts +++ b/src/renderer/selectors.ts @@ -452,9 +452,15 @@ export const selectedDataSourceTemporalCoverageSelector = createSelector( +export const canCacheDataSourceSelector = createSelector( + selectedDataStoreIdSelector, selectedDataSourceSelector, - (selectedDataSource: DataSourceState | null): boolean => { + (selectedDataStoreId, selectedDataSource): boolean => { + if (selectedDataStoreId === null + || selectedDataStoreId === 'local' + || selectedDataStoreId === 'cci-zarr-store') { + return false; + } return selectedDataSource ? canCacheDataSource(selectedDataSource) : false; } ); diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index 9753dab..ddf9d09 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.3.0-dev.2"; +const CATE_PWA_VERSION = "2.3.0-dev.3"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index 82f8903..0fcaf09 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.3.0-dev.2"; +export const CATE_APP_VERSION = "2.3.0-dev.3"; From 736d5f93fb714172e640f215b91139ee45c1b3d1 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Wed, 24 Mar 2021 18:09:45 +0100 Subject: [PATCH 14/17] 2.3.0-dev.4 --- Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 2 +- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6ca358f..389e358 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.3.0-dev.3" +LABEL version="2.3.0-dev.4" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index 437b275..1ab930a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.3.0-dev.3-{build} +version: 2.3.0-dev.4-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index d72561e..535ebe8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.3.0-dev.3", + "version": "2.3.0-dev.4", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index ddf9d09..9a5a050 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.3.0-dev.3"; +const CATE_PWA_VERSION = "2.3.0-dev.4"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index 0fcaf09..94cd7a2 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.3.0-dev.3"; +export const CATE_APP_VERSION = "2.3.0-dev.4"; From 4c6a66750022ba37c8bc262b69228ebf0f46af9b Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 30 Apr 2021 11:22:46 +0200 Subject: [PATCH 15/17] Addresses #150 --- CHANGES.md | 8 +++++- src/renderer/actions.ts | 38 ++++++++++++++++++++-------- src/renderer/webapi/WebAPIClient.ts | 15 ++++++++--- src/renderer/webapi/WebSocketMock.ts | 10 ++++++++ 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dd0c9ab..a77ce21 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,10 @@ -### Changes 2.3.0 (in development) +### Changes 3.0 (in development) + +* Cate App 3.0 now requires the Web API service of the `cate 3.0+` + Python package. + +* In order to keep alive the connection to the Web API service, + Cate App now sends a keepalive signal every 2.5 seconds. (#150) * Optimisations in the DATA SOURCES panel (that have been enabled by using [xcube](https://xcube.readthedocs.io/) in the backend): diff --git a/src/renderer/actions.ts b/src/renderer/actions.ts index 8f74d31..3e0d5f2 100644 --- a/src/renderer/actions.ts +++ b/src/renderer/actions.ts @@ -314,6 +314,27 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction { const webAPIClient = newWebAPIClient(selectors.apiWebSocketsUrlSelector(getState())); + const formatMessage = (message: string, event: any): string => { + if (event.message) { + return `${message} (${event.message})`; + } else { + return message; + } + }; + + /** + * Called to inform backend we are still alive. + * Hopefully avoids closing WebSocket connection. + */ + const keepAlive = () => { + if (webAPIClient.isOpen) { + console.debug("calling keep_alive()"); + webAPIClient.call('keep_alive', []) + } + }; + + let keepAliveTimer = null; + webAPIClient.onOpen = () => { dispatch(setWebAPIClient(webAPIClient)); dispatch(loadBackendConfig()); @@ -321,22 +342,19 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction { dispatch(loadPreferences()); dispatch(loadDataStores()); dispatch(loadOperations()); - }; - - const formatMessage = (message: string, event: any): string => { - if (event.message) { - return `${message} (${event.message})`; - } else { - return message; - } + keepAliveTimer = setInterval(keepAlive, 2500); }; webAPIClient.onClose = (event) => { + if (keepAliveTimer !== null) { + clearInterval(keepAliveTimer); + } const webAPIStatus = getState().communication.webAPIStatus; if (webAPIStatus === 'shuttingDown' || webAPIStatus === 'loggingOut') { // When we are logging off, the webAPIClient is expected to close. return; } + // When we end up here, the connection closed unintentionally. console.error('webAPIClient.onClose:', event); dispatch(setWebAPIStatus('closed')); showToast({type: 'notification', text: formatMessage('Connection to Cate service closed', event)}); @@ -355,8 +373,8 @@ export function connectWebAPIService(webAPIServiceURL: string): ThunkAction { }; } -export function updateHubStatus(hubStatus: HubStatus): Action { - return { type: UPDATE_HUB_STATUS, payload: hubStatus}; +export function updateHubStatus(hubStatus: HubStatus): Action { + return {type: UPDATE_HUB_STATUS, payload: hubStatus}; } export function updateInitialState(initialState: Object): Action { diff --git a/src/renderer/webapi/WebAPIClient.ts b/src/renderer/webapi/WebAPIClient.ts index a6ed030..f3f8bbe 100644 --- a/src/renderer/webapi/WebAPIClient.ts +++ b/src/renderer/webapi/WebAPIClient.ts @@ -85,7 +85,12 @@ export type JobResponseTransformer = (any) => JobResponse; * This is non JSON-RCP, which only allows for either the "response" or an "error" object. */ export interface WebAPIClient { + /** + * Test if the connection is open and active. + */ + readonly isOpen: boolean; readonly url: string; + onOpen: (event) => void; onClose: (event) => void; onError: (event) => void; @@ -142,16 +147,14 @@ class WebAPIClientImpl implements WebAPIClient { onError: (event) => void; onWarning: (event) => void; - readonly socket: WebSocketMin; + private readonly socket: WebSocketMin; private currentMessageId = 0; - private activeJobs: JobImpl[]; - private isOpen: boolean; + private readonly activeJobs: JobImpl[]; constructor(url: string, firstMessageId = 0, socket?: WebSocketMin) { this.url = url; this.currentMessageId = firstMessageId; this.activeJobs = []; - this.isOpen = false; this.socket = socket ? socket : new WebSocket(url); this.socket.onopen = (event) => { if (this.onOpen) { @@ -174,6 +177,10 @@ class WebAPIClientImpl implements WebAPIClient { } } + get isOpen(): boolean { + return this.socket.readyState === WebSocket.OPEN; + } + call(method: string, params: Array | Object, onProgress?: (progress: JobProgress) => void, diff --git a/src/renderer/webapi/WebSocketMock.ts b/src/renderer/webapi/WebSocketMock.ts index 7dd1cda..af5d68e 100644 --- a/src/renderer/webapi/WebSocketMock.ts +++ b/src/renderer/webapi/WebSocketMock.ts @@ -4,6 +4,8 @@ * @author Norman Fomferra */ export interface WebSocketMin { + readonly readyState: number; + onclose: (this: this, ev: CloseEvent) => any; onerror: (this: this, ev: ErrorEvent) => any; onmessage: (this: this, ev: MessageEvent) => any; @@ -52,6 +54,7 @@ export class WebSocketMock implements WebSocketMin { onclose: (this: this, ev: any) => any; readonly messageLog: string[] = []; readonly serviceObj: any; + readyState: number; // <<<< WebSocketMin implementation //////////////////////////////////////////// @@ -68,6 +71,7 @@ export class WebSocketMock implements WebSocketMin { } this.serviceObj = serviceObj; this.asyncCalls = asyncCalls; + this.readyState = WebSocket.CONNECTING; } send(data: string) { @@ -77,6 +81,7 @@ export class WebSocketMock implements WebSocketMin { close(code?: number, reason?: string): void { this.onclose({code, reason}); + this.readyState = WebSocket.CLOSED; } emulateIncomingMessages(...messages: Object[]) { @@ -95,14 +100,19 @@ export class WebSocketMock implements WebSocketMin { emulateOpen(event) { this.onopen(event); + this.readyState = WebSocket.OPEN; } emulateError(event) { + this.readyState = WebSocket.CLOSING; this.onerror(event); + this.readyState = WebSocket.CLOSED; } emulateClose(event) { + this.readyState = WebSocket.CLOSING; this.onclose(event); + this.readyState = WebSocket.CLOSED; } private maybeUseServiceObj(messageText) { From 8b2ebbc11cad0be18cb56be6dccd5ed0229cd1dd Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 30 Apr 2021 11:24:40 +0200 Subject: [PATCH 16/17] Started 3.0.0 dev --- Dockerfile | 2 +- appveyor.yml | 2 +- package.json | 2 +- src/serviceWorker.ts | 2 +- src/version.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 389e358..1a8232c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM node:stretch-slim as build-deps LABEL maintainer="helge.dzierzon@brockmann-consult.de" LABEL name="Cate App" -LABEL version="2.3.0-dev.4" +LABEL version="3.0.0-dev.0" RUN apt-get -y update && apt-get install -y git apt-utils wget vim diff --git a/appveyor.yml b/appveyor.yml index 1ab930a..50c3f78 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -version: 2.3.0-dev.4-{build} +version: 3.0.0-dev.0-{build} image: Ubuntu stack: node 12 install: diff --git a/package.json b/package.json index 535ebe8..3761057 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cate-app", - "version": "2.3.0-dev.4", + "version": "3.0.0-dev.0", "private": true, "dependencies": { "@blueprintjs/core": "^3.30.1", diff --git a/src/serviceWorker.ts b/src/serviceWorker.ts index 9a5a050..88a54bf 100644 --- a/src/serviceWorker.ts +++ b/src/serviceWorker.ts @@ -14,7 +14,7 @@ // "./service-worker.js" file changes). Therefore we use a version number here, // so we can force updates. // -const CATE_PWA_VERSION = "2.3.0-dev.4"; +const CATE_PWA_VERSION = "3.0.0-dev.0"; console.debug(`Cate PWA version ${CATE_PWA_VERSION}`); diff --git a/src/version.ts b/src/version.ts index 94cd7a2..ac6ce31 100644 --- a/src/version.ts +++ b/src/version.ts @@ -2,4 +2,4 @@ // 1. with the version field in "../package.json". // 2. with CATE_PWA_VERSION in "./serviceWorker.ts" // -export const CATE_APP_VERSION = "2.3.0-dev.4"; +export const CATE_APP_VERSION = "3.0.0-dev.0"; From 59bf2b0973536d9a932df8c664cd370b4041bcaa Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Mon, 3 May 2021 17:07:54 +0200 Subject: [PATCH 17/17] Update src/renderer/actions.ts Co-authored-by: Tonio Fincke --- src/renderer/actions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/renderer/actions.ts b/src/renderer/actions.ts index 3e0d5f2..56b5165 100644 --- a/src/renderer/actions.ts +++ b/src/renderer/actions.ts @@ -44,7 +44,8 @@ import { ControlState, DatasetDescriptor, DataSourceState, DataStoreState, - GeographicPosition, HubStatus, + GeographicPosition, + HubStatus, ImageStatisticsState, LayerState, MessageState, @@ -2757,4 +2758,3 @@ function readDroppedFile(file: File, dispatch: Dispatch) { console.warn('Dropped file of unrecognized type: ', file.name); } } -