From d15a80776fbca2bcc29bb388eab64f1efac4d453 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Sun, 11 Mar 2018 22:49:17 +0000 Subject: [PATCH 01/25] Fixes for 'mvn clean install' Fixes for: * test results, including results when run in GMT timezone * a dependency version in maven pom --- .../hadoop/GranuleTilesInputFormatTest.java | 2 +- .../reporting/code/CodeReportTest.java | 12 ++++---- .../CalvalusDataInputsTest.java | 30 +++++++++---------- .../wps/utils/ProductMetadataTest.java | 14 ++++----- .../CalvalusExecuteOperationTest.java | 4 +-- .../CalvalusGetStatusOperationTest.java | 6 ++-- pom.xml | 2 +- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/hadoop/GranuleTilesInputFormatTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/hadoop/GranuleTilesInputFormatTest.java index 500c02773..fbc1f1e91 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/hadoop/GranuleTilesInputFormatTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/hadoop/GranuleTilesInputFormatTest.java @@ -50,6 +50,6 @@ public LocatedFileStatus next() throws IOException { }; inputFormat.createSplits(null, remoteIter, splits, new Configuration(), 0); - assertEquals(2, splits.size()); + assertEquals(4, splits.size()); } } diff --git a/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java b/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java index aec4955b7..6f3ef86dc 100644 --- a/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java +++ b/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java @@ -51,7 +51,7 @@ public void canDeserializeCorrectly() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -120,7 +120,7 @@ public void canDeserializeL2ProcessFormatting() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -189,7 +189,7 @@ public void canDeserializeL2ProcessNoProcessor() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -258,7 +258,7 @@ public void canDeserializeL3ProcessFormatting() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -327,7 +327,7 @@ public void canDeserializeL3ProcessAggregation() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -415,4 +415,4 @@ public void canRoundDecimalNumbersWhenDeserialize() { assertThat(codeReport.toJson(), containsString(" \"version\": \"1.0\"\n" + "}")); } -} \ No newline at end of file +} diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java index db4832222..6736fbbca 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java @@ -57,8 +57,8 @@ public void canGetInputMapFormatted() throws Exception { assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.0b411")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T01:00:00+01:00")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T00:00:00Z")); } @Test @@ -77,8 +77,8 @@ public void canGetInputMapFormattedWithProductionSetDateRange() throws Exception assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.0b411")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2000-01-01T01:00:00+01:00")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2010-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2000-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2010-01-01T00:00:00Z")); } @Test @@ -97,8 +97,8 @@ public void canGetInputMapFormattedWithProductionSetGeoDb() throws Exception { assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.10-SNAPSHOT")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T01:00:00+01:00")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T00:00:00Z")); assertThat(calvalusDataInputs.getInputMapFormatted().get("geoInventory"), equalTo("/calvalus/geoInventory/URBAN_FOOTPRINT_GUF_GLOBAL_75m")); } @@ -280,7 +280,7 @@ public void canListAllParameters() throws Exception { assertThat(calvalusDataInputs.toString(), equalTo("calvalus.wps.remote.ref : 1738ad7b-534e-4aca-9861-b26fb9c0f983\n" + - "minDate : 2016-01-01T01:00:00+01:00\n" + + "minDate : 2016-01-01T00:00:00Z\n" + "calvalus.output.compression : none\n" + "processorBundleLocation : hdfs://calvalus/calvalus/software/1.0/beam-buildin-1.0\n" + "calvalus.wps.remote.user : dummyRemoteUser\n" + @@ -290,16 +290,16 @@ public void canListAllParameters() throws Exception { "processorBundleVersion : null\n" + "calvalus.calvalus.bundle : calvalus-2.0b411\n" + "inputPath : /calvalus/eodata/MER_RR__1P/r03/${yyyy}/${MM}/${dd}/.*.N1\n" + - "minDateSource : 2016-01-01T01:00:00+01:00\n" + + "minDateSource : 2016-01-01T00:00:00Z\n" + "quicklooks : true\n" + "autoStaging : true\n" + "calvalus.snap.bundle : snap-3.0.0\n" + "processorName : null\n" + - "maxDate : 2017-01-01T01:00:00+01:00\n" + + "maxDate : 2017-01-01T00:00:00Z\n" + "processorBundleName : null\n" + "processorParameters : null\n" + "productionName : dummyProductionName\n" + - "maxDateSource : 2017-01-01T01:00:00+01:00\n" + + "maxDateSource : 2017-01-01T00:00:00Z\n" + "calvalus.system.snap.dataio.bigtiff.support.pushprocessing : false\n")); } @@ -324,7 +324,7 @@ public void canIgnoreDefaultBeamAndCalvalusBundleVersionWhenNotProvided() throws assertThat(calvalusDataInputs.toString(), equalTo("calvalus.wps.remote.ref : 1738ad7b-534e-4aca-9861-b26fb9c0f983\n" + - "minDate : 2016-01-01T01:00:00+01:00\n" + + "minDate : 2016-01-01T00:00:00Z\n" + "calvalus.output.compression : none\n" + "processorBundleLocation : null\n" + "calvalus.wps.remote.user : dummyRemoteUser\n" + @@ -332,14 +332,14 @@ public void canIgnoreDefaultBeamAndCalvalusBundleVersionWhenNotProvided() throws "inputDataSetName : MERIS RR r03 L1b 2002-2012\n" + "processorBundleVersion : null\n" + "inputPath : /calvalus/eodata/MER_RR__1P/r03/${yyyy}/${MM}/${dd}/.*.N1\n" + - "minDateSource : 2016-01-01T01:00:00+01:00\n" + + "minDateSource : 2016-01-01T00:00:00Z\n" + "quicklooks : true\n" + "autoStaging : true\n" + "processorName : null\n" + - "maxDate : 2017-01-01T01:00:00+01:00\n" + + "maxDate : 2017-01-01T00:00:00Z\n" + "processorBundleName : null\n" + "processorParameters : null\n" + - "maxDateSource : 2017-01-01T01:00:00+01:00\n" + + "maxDateSource : 2017-01-01T00:00:00Z\n" + "calvalus.system.snap.dataio.bigtiff.support.pushprocessing : false\n")); } @@ -492,4 +492,4 @@ private Map getMinimalRawMap() { return mockInputMapRaw; } -} \ No newline at end of file +} diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java index cb6d2fd75..5d5dab367 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java @@ -71,14 +71,14 @@ public void canCreateProductMetadata() throws Exception { .build(); Map contextMap = productMetadata.getContextMap(); - assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T00:00:00.000Z")); assertThat(contextMap.get("productOutputDir"), equalTo("user/20160317_10000000")); assertThat(contextMap.get("productionName"), equalTo("TEP_Subset_test")); assertThat(contextMap.get("processName"), equalTo("Subset")); assertThat(contextMap.get("inputDatasetName"), equalTo("Urban Footprint Global (Urban TEP)")); assertThat(contextMap.get("regionWkt"), equalTo("-10 100 0 100 0 110 -10 110 -10 100")); - assertThat(contextMap.get("startDate"), equalTo("2000-01-01T01:00:00.000+01:00")); - assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("startDate"), equalTo("2000-01-01T00:00:00.000Z")); + assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T00:00:00.000Z")); assertThat(contextMap.get("collectionUrl"), equalTo("http://www.brockmann-consult.de:80/bc-wps/staging/user/20160317_10000000")); assertThat(contextMap.get("processorVersion"), equalTo("1.0")); assertThat(contextMap.get("productionType"), equalTo("2")); @@ -121,14 +121,14 @@ public void canEncodeProductionNameWithSpecialChars() throws Exception { .build(); Map contextMap = productMetadata.getContextMap(); - assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T00:00:00.000Z")); assertThat(contextMap.get("productOutputDir"), equalTo("user/20160317_10000000")); assertThat(contextMap.get("productionName"), equalTo("http%3A%2F%2Fcatalog.terradue.com%2Furban-bc%2Fsearch%3Fformat%3Djson%26uid%3Dtep_inverted_timescan_Dakar%2Ftep_user%2F20170427185240_L3_130ec6a9dc24be")); assertThat(contextMap.get("processName"), equalTo("Subset")); assertThat(contextMap.get("inputDatasetName"), equalTo("Urban Footprint Global (Urban TEP)")); assertThat(contextMap.get("regionWkt"), equalTo("-10 100 0 100 0 110 -10 110 -10 100")); - assertThat(contextMap.get("startDate"), equalTo("2000-01-01T01:00:00.000+01:00")); - assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("startDate"), equalTo("2000-01-01T00:00:00.000Z")); + assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T00:00:00.000Z")); assertThat(contextMap.get("collectionUrl"), equalTo("http://www.brockmann-consult.de:80/bc-wps/staging/user/20160317_10000000")); assertThat(contextMap.get("processorVersion"), equalTo("1.0")); assertThat(contextMap.get("productionType"), equalTo("2")); @@ -177,4 +177,4 @@ private void configureMockServerContext() { when(mockServerContext.getPort()).thenReturn(80); when(mockServerContext.getRequestUrl()).thenReturn("http://www.brockmann-consult.de/bc-wps/wps/calvalus?Service=WPS&Request=GetStatus&JobId=20160429140605_L2Plus_16bd8f26b258fc"); } -} \ No newline at end of file +} diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java index 1ef2819a5..2f860a19c 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java @@ -137,7 +137,7 @@ public void canExecuteSync() throws Exception { assertThat(executeResponse.getStatus().getProcessSucceeded(), equalTo("The request has been processed successfully.")); assertThat(executeResponse.getProcess().getIdentifier().getValue(), equalTo("process1")); - assertThat(executeResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); + assertThat(executeResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); assertThat(executeResponse.getProcessOutputs().getOutput().size(), equalTo(2)); assertThat(executeResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), equalTo("resultUrl1")); assertThat(executeResponse.getProcessOutputs().getOutput().get(1).getReference().getHref(), equalTo("resultUrl2")); @@ -229,4 +229,4 @@ private ProductionRequest configureProcessingMock() throws Exception { return mockProductionRequest; } -} \ No newline at end of file +} diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java index bb7607f9f..903764596 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java @@ -220,7 +220,7 @@ public void canGetSuccessfulStatus() throws Exception { assertThat(getStatusResponse.getProcess().getIdentifier().getValue(), equalTo("mockBundle~1.0~mockProcessor")); assertThat(getStatusResponse.getProcess().getProcessVersion(), equalTo("1.0")); - assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); + assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getIdentifier().getValue(), equalTo("production_result")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), @@ -250,7 +250,7 @@ public void canGetSuccessfulStatusLocalProcess() throws Exception { assertThat(getStatusResponse.getProcess().getIdentifier().getValue(), equalTo("local~0.0.1~Subset")); assertThat(getStatusResponse.getProcess().getProcessVersion(), equalTo("0.0.1")); - assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); + assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getIdentifier().getValue(), equalTo("production_result")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), @@ -343,4 +343,4 @@ private LocalProductionStatus getDoneAndSuccessfulLocalProcessStatus(List1.2 7.0.0-SNAPSHOT [6.0.0-SNAPSHOT,7.0.0-SNAPSHOT] - 1.7-SNAPSHOT + 1.7 0.6.1 17.1 1.7.21 From 6414181e880e629c8360420f59ab8f6416df015c Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Sun, 11 Mar 2018 23:59:56 +0000 Subject: [PATCH 02/25] Initial quicklook support in Calvalus portal This initial test version is for Multi-band (RGB). The quicklook parameters are hard-coded. --- .../portal/client/OrderL2ProductionView.java | 4 ++ .../client/QuicklookParametersForm.java | 55 +++++++++++++++++++ .../portal/server/BackendServiceImpl.java | 17 ++++++ .../client/QuicklookParametersForm.ui.xml | 25 +++++++++ .../production/ProductionRequest.java | 4 ++ 5 files changed, 105 insertions(+) create mode 100755 calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java create mode 100755 calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java index e1bb9f9d8..9ced300fd 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java @@ -44,6 +44,7 @@ public class OrderL2ProductionView extends OrderProductionView { private ProductsFromCatalogueForm productsFromCatalogueForm; private L2ConfigForm l2ConfigForm; private OutputParametersForm outputParametersForm; + private QuicklookParametersForm quicklookParametersForm; private Widget widget; public OrderL2ProductionView(PortalContext portalContext) { @@ -77,6 +78,7 @@ public void onClearSelectionClick() { } outputParametersForm = new OutputParametersForm(portalContext); + quicklookParametersForm = new QuicklookParametersForm(portalContext); l2ConfigForm.setProductSet(productSetSelectionForm.getSelectedProductSet()); handleProcessorChanged(); @@ -89,6 +91,7 @@ public void onClearSelectionClick() { } panel.add(l2ConfigForm); panel.add(outputParametersForm); + panel.add(quicklookParametersForm); Anchor l2Help = new Anchor("Show Help"); l2Help.getElement().getStyle().setProperty("textDecoration", "none"); l2Help.addStyleName("anchor"); @@ -168,6 +171,7 @@ protected HashMap getProductionParameters() { } parameters.putAll(l2ConfigForm.getValueMap()); parameters.putAll(outputParametersForm.getValueMap()); + parameters.putAll(quicklookParametersForm.getValueMap()); return parameters; } diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java new file mode 100755 index 000000000..e8270cec5 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -0,0 +1,55 @@ +package com.bc.calvalus.portal.client; + +import com.google.gwt.core.client.GWT; +import com.google.gwt.event.logical.shared.ValueChangeEvent; +import com.google.gwt.event.logical.shared.ValueChangeHandler; +import com.google.gwt.uibinder.client.UiBinder; +import com.google.gwt.uibinder.client.UiField; +import com.google.gwt.user.client.ui.CheckBox; +import com.google.gwt.user.client.ui.Composite; +import com.google.gwt.user.client.ui.DoubleBox; +import com.google.gwt.user.client.ui.IntegerBox; +import com.google.gwt.user.client.ui.ListBox; +import com.google.gwt.user.client.ui.Panel; +import com.google.gwt.user.client.ui.RadioButton; +import com.google.gwt.user.client.ui.TextArea; +import com.google.gwt.user.client.ui.TextBox; +import com.google.gwt.user.client.ui.Widget; + +import java.util.HashMap; +import java.util.Map; + + +/** + * This form is used regarding visualisations settings. + * + * @author Declan + */ +public class QuicklookParametersForm extends Composite { + + interface TheUiBinder extends UiBinder { + + } + + private static TheUiBinder uiBinder = GWT.create(TheUiBinder.class); + + @UiField + CheckBox enableQuicklook; + + public QuicklookParametersForm(PortalContext portalContext) { + initWidget(uiBinder.createAndBindUi(this)); + } + + public Map getValueMap() { + Map parameters = new HashMap(); + parameters.put("enableQuicklook", enableQuicklook.getValue().toString()); + return parameters; + } + + public void setValues(Map parameters) { + String enableQuicklookValue = parameters.get("enableQuicklook"); + if (enableQuicklookValue != null) { + enableQuicklook.setValue(Boolean.valueOf(enableQuicklookValue)); + } + } +} \ No newline at end of file diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java index 9e5cf3d09..a70872deb 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java @@ -451,6 +451,23 @@ public DtoProductionResponse orderProduction(DtoProductionRequest dtoProductionR if (requestSizeLimit != null) { productionRequest.setParameter(JobConfigNames.CALVALUS_REQUEST_SIZE_LIMIT, (String) requestSizeLimit); } + + // uc1: multi-band (RGB) + // uc2: single-band with color table + boolean isEnableQuicklook = productionRequest.isEnableQuicklook(); + if( isEnableQuicklook ) { + // uc1: multi-band (RGB) + productionRequest.setParameter("quicklooks", "true"); + String ql_params = "\n" + + " \n" + + " \n" + + " ndbi,ndvi,ndwi\n" + + " png\n" + + " \n" + + " \n" + + ""; + productionRequest.setParameter("calvalus.ql.parameters", ql_params); + } ProductionResponse productionResponse = serviceContainer.getProductionService().orderProduction(productionRequest, hook); return convert(productionResponse); } catch (ProductionException e) { diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml new file mode 100755 index 000000000..71ea02700 --- /dev/null +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -0,0 +1,25 @@ + + + + + + + + + Quicklook + + + + + +
+ + + + + diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java b/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java index 0055f083a..72347e83c 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java @@ -428,6 +428,10 @@ private static String decodeXML(String s) { return s.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); } + public boolean isEnableQuicklook() throws ProductionException { + return getBoolean("enableQuicklook", false); + } + ///////////////////////////////////////////////////////////////////////// // XmlConvertible From e0bd555642b066a13a460378f864c8cb52c39ddb Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 16 Apr 2018 17:08:34 +0100 Subject: [PATCH 03/25] Configurable quicklook support in Calvalus portal * Add all quicklook parameters to the GUI for user configuration * Supports single-band colour table and multi-band RGBA * Moved calvalus.ql.parameters XML configuration to the client-side --- .../portal/client/OrderL2ProductionView.java | 1 + .../client/QuicklookParametersForm.java | 439 +++++++++++++++++- .../portal/server/BackendServiceImpl.java | 17 - .../com/bc/calvalus/portal/client/style.css | 20 + .../bc/calvalus/portal/CalvalusPortal.gwt.xml | 2 + .../client/QuicklookParametersForm.ui.xml | 163 ++++++- .../production/ProductionRequest.java | 4 - 7 files changed, 609 insertions(+), 37 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java index 9ced300fd..b133f01c9 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java @@ -189,6 +189,7 @@ public void setProductionParameters(Map parameters) { } l2ConfigForm.setValues(parameters); outputParametersForm.setValues(parameters); + quicklookParametersForm.setValues(parameters); } private class InputSelectionCallback implements AsyncCallback { diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index e8270cec5..78ea662b7 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + package com.bc.calvalus.portal.client; import com.google.gwt.core.client.GWT; @@ -7,19 +23,19 @@ import com.google.gwt.uibinder.client.UiField; import com.google.gwt.user.client.ui.CheckBox; import com.google.gwt.user.client.ui.Composite; -import com.google.gwt.user.client.ui.DoubleBox; import com.google.gwt.user.client.ui.IntegerBox; import com.google.gwt.user.client.ui.ListBox; -import com.google.gwt.user.client.ui.Panel; import com.google.gwt.user.client.ui.RadioButton; -import com.google.gwt.user.client.ui.TextArea; import com.google.gwt.user.client.ui.TextBox; import com.google.gwt.user.client.ui.Widget; +import com.google.gwt.xml.client.Document; +import com.google.gwt.xml.client.Node; +import com.google.gwt.xml.client.NodeList; +import com.google.gwt.xml.client.XMLParser; import java.util.HashMap; import java.util.Map; - /** * This form is used regarding visualisations settings. * @@ -28,28 +44,431 @@ public class QuicklookParametersForm extends Composite { interface TheUiBinder extends UiBinder { - } private static TheUiBinder uiBinder = GWT.create(TheUiBinder.class); @UiField - CheckBox enableQuicklook; + RadioButton quicklookNone; + @UiField + RadioButton quicklookSingleBand; + @UiField + RadioButton quicklookMultiBand; + + @UiField + TextBox bandName; + @UiField + TextBox cpdURL; + + @UiField + TextBox rgbaExpressionsRedBand; + @UiField + TextBox rgbaExpressionsGreenBand; + @UiField + TextBox rgbaExpressionsBlueBand; + @UiField + TextBox rgbaExpressionsAlphaBand; + @UiField + TextBox rgbaMinSamples; + @UiField + TextBox rgbaMaxSamples; + + @UiField + ListBox imageType; + @UiField + TextBox overlayURL; + @UiField + TextBox maskOverlays; + @UiField + IntegerBox subSamplingX; + @UiField + IntegerBox subSamplingY; + @UiField + TextBox backgroundColor; + @UiField + CheckBox legendEnabled; public QuicklookParametersForm(PortalContext portalContext) { initWidget(uiBinder.createAndBindUi(this)); + + quicklookNone.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + setQuicklookNone(); + } + }); + + quicklookSingleBand.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + setQuicklookSingleBandEnabled(); + } + }); + + quicklookMultiBand.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + setQuicklookMultiBandEnabled(); + } + }); + + setAvailableImageTypes(); + quicklookNone.setValue(true); + setQuicklookNone(); } public Map getValueMap() { + Map parameters = new HashMap(); - parameters.put("enableQuicklook", enableQuicklook.getValue().toString()); + if( quicklookNone.getValue() ) { + return parameters; + } + + String indentXML = " "; + + // imageType + String imageTypeXML = indentXML + "" + imageType.getSelectedValue() + "\n"; + + // overlayURL + String overlayURLValue = overlayURL.getValue(); + String overlayURLXML = ""; + if( overlayURLValue !=null && !overlayURLValue.isEmpty() ) { + overlayURLXML = indentXML + "" + overlayURLValue + "\n"; + } + + // maskOverlays + String maskOverlaysValue = maskOverlays.getValue(); + String maskOverlaysXML = ""; + if( maskOverlaysValue !=null && !maskOverlaysValue.isEmpty() ) { + maskOverlaysXML = indentXML + "" + maskOverlaysValue + "\n"; + } + + // subSamplingX + Integer subSamplingXValue = subSamplingX.getValue(); + String subSamplingXXML = ""; + if( subSamplingXValue !=null && subSamplingXValue >= 1 ) { + subSamplingXXML = indentXML + "" + subSamplingXValue + "\n"; + } + + // subSamplingY + Integer subSamplingYValue = subSamplingY.getValue(); + String subSamplingYXML = ""; + if( subSamplingYValue !=null && subSamplingYValue >= 1 ) { + subSamplingYXML = indentXML + "" + subSamplingYValue + "\n"; + } + + // backgroundColor + String backgroundColorValue = backgroundColor.getValue(); + String backgroundColorXML = ""; + if( backgroundColorValue !=null && !backgroundColorValue.isEmpty() ) { + backgroundColorXML = indentXML + "" + backgroundColorValue + "\n"; + } + + // legendEnabled + Boolean legendEnabledValue = legendEnabled.getValue(); + String legendEnabledXML = ""; + if( legendEnabled.getValue() ) { + legendEnabledXML = indentXML + "true\n"; + } + + String quicklookParameters = ""; + if( quicklookSingleBand.getValue() ) { + + // bandName + String bandNameXML = ""; + String bandNameValue = bandName.getValue(); + if( bandNameValue != null && !bandNameValue.isEmpty() ) { + bandNameXML = indentXML + "" + bandNameValue + "\n"; + } + + // cpdURL + String cpdURLXML = ""; + String cpdURLValue = cpdURL.getValue(); + if( cpdURLValue != null && !cpdURLValue .isEmpty() ) { + cpdURLXML = indentXML + "" + cpdURLValue + "\n"; + } + + quicklookParameters = "\n" + + " \n" + + " \n" + + bandNameXML + + cpdURLXML + + imageTypeXML + + overlayURLXML + + maskOverlaysXML + + subSamplingXXML + + subSamplingYXML + + backgroundColorXML + + legendEnabledXML + + " \n" + + " \n" + + ""; + } + else if( quicklookMultiBand.getValue() ) { + + // RGBAExpressions + String rgbaExpressionsXML = ""; + String rgbaExpressionsRedBandValue = rgbaExpressionsRedBand.getValue(); + String rgbaExpressionsGreenBandValue = rgbaExpressionsGreenBand.getValue(); + String rgbaExpressionsBlueBandValue = rgbaExpressionsBlueBand.getValue(); + String rgbaExpressionsAlphaBandValue = rgbaExpressionsAlphaBand.getValue(); + + if( rgbaExpressionsRedBandValue != null && !rgbaExpressionsRedBandValue.isEmpty() && + rgbaExpressionsGreenBandValue != null && !rgbaExpressionsGreenBandValue.isEmpty() && + rgbaExpressionsBlueBandValue != null && !rgbaExpressionsBlueBandValue.isEmpty() ) { + + rgbaExpressionsXML = indentXML + "" + + rgbaExpressionsRedBandValue + "," + + rgbaExpressionsGreenBandValue + "," + + rgbaExpressionsBlueBandValue; + + if( rgbaExpressionsAlphaBandValue != null && !rgbaExpressionsAlphaBandValue.isEmpty() ) + rgbaExpressionsXML = rgbaExpressionsXML + "," + rgbaExpressionsAlphaBandValue; + + rgbaExpressionsXML = rgbaExpressionsXML + "\n"; + } + + // rgbaMinSamples + String rgbaMinSamplesXML = ""; + String rgbaMinSamplesValue = rgbaMinSamples.getValue(); + if( rgbaMinSamplesValue !=null && !rgbaMinSamplesValue.isEmpty() ) { + rgbaMinSamplesXML = indentXML + "" + rgbaMinSamplesValue + "\n"; + } + + // rgbaMaxSamples + String rgbaMaxSamplesXML = ""; + String rgbaMaxSamplesValue = rgbaMaxSamples.getValue(); + if( rgbaMaxSamplesValue !=null && !rgbaMaxSamplesValue.isEmpty() ) { + rgbaMaxSamplesXML = indentXML + "" + rgbaMaxSamplesValue + "\n"; + } + + quicklookParameters = "\n" + + " \n" + + " \n" + + rgbaExpressionsXML + + rgbaMinSamplesXML + + rgbaMaxSamplesXML + + imageTypeXML + + overlayURLXML + + maskOverlaysXML + + subSamplingXXML + + subSamplingYXML + + backgroundColorXML + + legendEnabledXML + + " \n" + + " \n" + + ""; + } + + if( !quicklookParameters.isEmpty() ) { + parameters.put("quicklooks", "true"); + parameters.put("calvalus.ql.parameters", quicklookParameters); + } + return parameters; } public void setValues(Map parameters) { - String enableQuicklookValue = parameters.get("enableQuicklook"); - if (enableQuicklookValue != null) { - enableQuicklook.setValue(Boolean.valueOf(enableQuicklookValue)); + + String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); + if( quicklookParametersValue == null || quicklookParametersValue.isEmpty() ) { + quicklookNone.setValue(true, true); + setQuicklookNone(); + + bandName.setValue(null); + cpdURL.setValue(null); + + rgbaExpressionsRedBand.setValue( null ); + rgbaExpressionsGreenBand.setValue( null ); + rgbaExpressionsBlueBand.setValue( null ); + rgbaExpressionsAlphaBand.setValue( null ); + rgbaMinSamples.setValue(null); + rgbaMaxSamples.setValue(null); + + imageType.setSelectedIndex(0); + overlayURL.setValue(null); + maskOverlays.setValue(null); + subSamplingX.setValue(null); + maskOverlays.setValue(null); + legendEnabled.setValue(false); + } + else { + Document dom = XMLParser.parse(quicklookParametersValue); + + // bandName + String bandNameValue = getTagValue(dom, "bandName"); + if( bandNameValue != null ) + quicklookSingleBand.setValue(true, true); + bandName.setValue(bandNameValue); + + // cpdURL + String cpdURLValue = getTagValue(dom, "cpdURL"); + cpdURL.setValue(cpdURLValue); + + // RGBAExpressions + String RGBAExpressionsValue = getTagValue(dom, "RGBAExpressions"); + if( RGBAExpressionsValue != null ) { + String[] tokens = RGBAExpressionsValue.split(","); + int numTokens = tokens.length; + rgbaExpressionsRedBand.setValue( numTokens>0 ? tokens[0] : null ); + rgbaExpressionsGreenBand.setValue( numTokens>1 ? tokens[1] : null ); + rgbaExpressionsBlueBand.setValue( numTokens>2 ? tokens[2] : null ); + rgbaExpressionsAlphaBand.setValue( numTokens>3 ? tokens[3] : null ); + quicklookMultiBand.setValue(true, true); + } + else{ + rgbaExpressionsRedBand.setValue( null ); + rgbaExpressionsGreenBand.setValue( null ); + rgbaExpressionsBlueBand.setValue( null ); + rgbaExpressionsAlphaBand.setValue( null ); + } + + // rgbaMinSamples + String rgbaMinSamplesValue = getTagValue(dom, "rgbaMinSamples"); + rgbaMinSamples.setValue(rgbaMinSamplesValue); + + // rgbaMaxSamples + String rgbaMaxSamplesValue = getTagValue(dom, "rgbaMaxSamples"); + rgbaMaxSamples.setValue(rgbaMaxSamplesValue); + + // imageType + String imageTypeValue = getTagValue(dom, "imageType"); + if( imageTypeValue == null ) + imageType.setSelectedIndex(0); + else if( imageTypeValue.equalsIgnoreCase("jpeg") ) + imageType.setSelectedIndex(0); + else if( imageTypeValue.equalsIgnoreCase("tiff") ) + imageType.setSelectedIndex(1); + else if( imageTypeValue.equalsIgnoreCase("png") ) + imageType.setSelectedIndex(2); + + // overlayURL + String overlayURLValue = getTagValue(dom, "overlayURL"); + overlayURL.setValue(overlayURLValue); + + // maskOverlays + String maskOverlaysValue = getTagValue(dom, "maskOverlays"); + maskOverlays.setValue(maskOverlaysValue); + + // subSamplingX + String subSamplingXValue = getTagValue(dom, "subSamplingX"); + try { + Integer subSamplingXIntValue = Integer.valueOf(subSamplingXValue); + subSamplingX.setValue(subSamplingXIntValue); + } + catch (NumberFormatException e) { + subSamplingX.setValue(null); + } + + // subSamplingY + String subSamplingYValue = getTagValue(dom, "subSamplingY"); + try { + Integer subSamplingYIntValue = Integer.valueOf(subSamplingYValue); + subSamplingY.setValue(subSamplingYIntValue); + } + catch (NumberFormatException e) { + subSamplingY.setValue(null); + } + + // backgroundColor + String backgroundColorValue = getTagValue(dom, "backgroundColor"); + backgroundColor.setValue(backgroundColorValue); + + // legendEnabled + String legendEnabledValue = getTagValue(dom, "legendEnabled"); + if( legendEnabledValue != null && legendEnabledValue.equalsIgnoreCase("true") ) + legendEnabled.setValue(true); + else + legendEnabled.setValue(false); + legendEnabled.setEnabled(true); + } + } + + private String getTagValue(Document dom, String tagName) { + NodeList nodeList = dom.getElementsByTagName(tagName); + if (nodeList != null && nodeList.getLength() > 0) { + Node node = nodeList.item(0); + if (node != null && node.hasChildNodes()) { + Node firstChild = node.getFirstChild(); + if (firstChild != null) { + return(firstChild.getNodeValue()); + } + } + } + return null; + } + + private void setQuicklookNone() { + bandName.setEnabled(false); + cpdURL.setEnabled(false); + + rgbaExpressionsRedBand.setEnabled(false); + rgbaExpressionsGreenBand.setEnabled(false); + rgbaExpressionsBlueBand.setEnabled(false); + rgbaExpressionsAlphaBand.setEnabled(false); + rgbaMinSamples.setEnabled(false); + rgbaMaxSamples.setEnabled(false); + + imageType.setEnabled(false); + overlayURL.setEnabled(false); + maskOverlays.setEnabled(false); + subSamplingX.setEnabled(false); + subSamplingY.setEnabled(false); + backgroundColor.setEnabled(false); + legendEnabled.setEnabled(false); + } + + private void setQuicklookSingleBandEnabled() { + bandName.setEnabled(true); + cpdURL.setEnabled(true); + + rgbaExpressionsRedBand.setEnabled(false); + rgbaExpressionsGreenBand.setEnabled(false); + rgbaExpressionsBlueBand.setEnabled(false); + rgbaExpressionsAlphaBand.setEnabled(false); + rgbaMinSamples.setEnabled(false); + rgbaMaxSamples.setEnabled(false); + + imageType.setEnabled(true); + overlayURL.setEnabled(true); + maskOverlays.setEnabled(true); + subSamplingX.setEnabled(true); + subSamplingY.setEnabled(true); + backgroundColor.setEnabled(true); + legendEnabled.setEnabled(true); + } + + private void setQuicklookMultiBandEnabled() { + bandName.setEnabled(false); + cpdURL.setEnabled(false); + + rgbaExpressionsRedBand.setEnabled(true); + rgbaExpressionsGreenBand.setEnabled(true); + rgbaExpressionsBlueBand.setEnabled(true); + rgbaExpressionsAlphaBand.setEnabled(true); + rgbaMinSamples.setEnabled(true); + rgbaMaxSamples.setEnabled(true); + + imageType.setEnabled(true); + overlayURL.setEnabled(true); + maskOverlays.setEnabled(true); + subSamplingX.setEnabled(true); + subSamplingY.setEnabled(true); + backgroundColor.setEnabled(true); + legendEnabled.setEnabled(true); + } + + public void setAvailableImageTypes() { + String [] imageNames = {"jpeg", "tiff", "png"}; + int selectedIndex = imageType.getSelectedIndex(); + imageType.clear(); + for( String imageName : imageNames) { + imageType.addItem(imageName); + } + if (selectedIndex >= 0 && selectedIndex < imageNames.length) { + imageType.setSelectedIndex(selectedIndex); + } else { + imageType.setSelectedIndex(0); } } } \ No newline at end of file diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java index a70872deb..9e5cf3d09 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java @@ -451,23 +451,6 @@ public DtoProductionResponse orderProduction(DtoProductionRequest dtoProductionR if (requestSizeLimit != null) { productionRequest.setParameter(JobConfigNames.CALVALUS_REQUEST_SIZE_LIMIT, (String) requestSizeLimit); } - - // uc1: multi-band (RGB) - // uc2: single-band with color table - boolean isEnableQuicklook = productionRequest.isEnableQuicklook(); - if( isEnableQuicklook ) { - // uc1: multi-band (RGB) - productionRequest.setParameter("quicklooks", "true"); - String ql_params = "\n" + - " \n" + - " \n" + - " ndbi,ndvi,ndwi\n" + - " png\n" + - " \n" + - " \n" + - ""; - productionRequest.setParameter("calvalus.ql.parameters", ql_params); - } ProductionResponse productionResponse = serviceContainer.getProductionService().orderProduction(productionRequest, hook); return convert(productionResponse); } catch (ProductionException e) { diff --git a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css index c65a29fe3..a70020512 100644 --- a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css +++ b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css @@ -118,6 +118,26 @@ font-size: 14px; } +/************************** +* Quicklook Panel styling * +***************************/ +.quicklookPanel { + margin: 50px 15px; + width: 62em; +} + +.quicklookParametersPanel { + display: flex; + flex-direction: column; +} + +.quicklookInputLabel { + flex: 0 0 130px; + margin: auto 0; + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; +} + /********************** * Input files styling * ***********************/ diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml index e7a6bf344..3ad07324a 100644 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/CalvalusPortal.gwt.xml @@ -17,6 +17,8 @@ + + diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index 71ea02700..bf83b07bc 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -4,17 +4,168 @@ - + Quicklook - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
   Band name: +   +
   Colour table: +   +
+ +
   Red band expression: +   +
   Green band expression: +   +
   Blue band expression: +   +
   Alpha band expression: +   +
   Minimum samples: + +
   Maximum samples: + +
   
+ More quicklook options: +
   Image type: + +
   Overlay URL: +   +
   Mask overlays: +   +
   Sub sampling (X): +   +
   Sub sampling (Y): +   +
   Background Color: +   +
    + +    
+ +
diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java b/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java index 72347e83c..0055f083a 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/ProductionRequest.java @@ -428,10 +428,6 @@ private static String decodeXML(String s) { return s.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); } - public boolean isEnableQuicklook() throws ProductionException { - return getBoolean("enableQuicklook", false); - } - ///////////////////////////////////////////////////////////////////////// // XmlConvertible From 31f3c91da42d64ad5d2dc78b807f396d208eb6a7 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 30 Apr 2018 18:01:13 +0100 Subject: [PATCH 04/25] Remove redundant code Change handlers for the quicklook radio buttons manages the form state --- .../com/bc/calvalus/portal/client/QuicklookParametersForm.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 78ea662b7..c2665b7ce 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -273,7 +273,6 @@ public void setValues(Map parameters) { String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); if( quicklookParametersValue == null || quicklookParametersValue.isEmpty() ) { quicklookNone.setValue(true, true); - setQuicklookNone(); bandName.setValue(null); cpdURL.setValue(null); @@ -380,7 +379,6 @@ else if( imageTypeValue.equalsIgnoreCase("png") ) legendEnabled.setValue(true); else legendEnabled.setValue(false); - legendEnabled.setEnabled(true); } } From 550bf892edb8dad13b08a490a57c89750f759599 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 30 Apr 2018 18:39:25 +0100 Subject: [PATCH 05/25] Reformat and rearrange code --- .../client/QuicklookParametersForm.java | 245 +++++++++--------- 1 file changed, 119 insertions(+), 126 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index c2665b7ce..a824b5a79 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -117,10 +117,83 @@ public void onValueChange(ValueChangeEvent event) { setQuicklookNone(); } - public Map getValueMap() { + private void setQuicklookNone() { + bandName.setEnabled(false); + cpdURL.setEnabled(false); + + rgbaExpressionsRedBand.setEnabled(false); + rgbaExpressionsGreenBand.setEnabled(false); + rgbaExpressionsBlueBand.setEnabled(false); + rgbaExpressionsAlphaBand.setEnabled(false); + rgbaMinSamples.setEnabled(false); + rgbaMaxSamples.setEnabled(false); + + imageType.setEnabled(false); + overlayURL.setEnabled(false); + maskOverlays.setEnabled(false); + subSamplingX.setEnabled(false); + subSamplingY.setEnabled(false); + backgroundColor.setEnabled(false); + legendEnabled.setEnabled(false); + } + + private void setQuicklookSingleBandEnabled() { + bandName.setEnabled(true); + cpdURL.setEnabled(true); + + rgbaExpressionsRedBand.setEnabled(false); + rgbaExpressionsGreenBand.setEnabled(false); + rgbaExpressionsBlueBand.setEnabled(false); + rgbaExpressionsAlphaBand.setEnabled(false); + rgbaMinSamples.setEnabled(false); + rgbaMaxSamples.setEnabled(false); + + imageType.setEnabled(true); + overlayURL.setEnabled(true); + maskOverlays.setEnabled(true); + subSamplingX.setEnabled(true); + subSamplingY.setEnabled(true); + backgroundColor.setEnabled(true); + legendEnabled.setEnabled(true); + } + + private void setQuicklookMultiBandEnabled() { + bandName.setEnabled(false); + cpdURL.setEnabled(false); + + rgbaExpressionsRedBand.setEnabled(true); + rgbaExpressionsGreenBand.setEnabled(true); + rgbaExpressionsBlueBand.setEnabled(true); + rgbaExpressionsAlphaBand.setEnabled(true); + rgbaMinSamples.setEnabled(true); + rgbaMaxSamples.setEnabled(true); + + imageType.setEnabled(true); + overlayURL.setEnabled(true); + maskOverlays.setEnabled(true); + subSamplingX.setEnabled(true); + subSamplingY.setEnabled(true); + backgroundColor.setEnabled(true); + legendEnabled.setEnabled(true); + } + public void setAvailableImageTypes() { + String[] imageNames = {"jpeg", "tiff", "png"}; + int selectedIndex = imageType.getSelectedIndex(); + imageType.clear(); + for (String imageName : imageNames) { + imageType.addItem(imageName); + } + if (selectedIndex >= 0 && selectedIndex < imageNames.length) { + imageType.setSelectedIndex(selectedIndex); + } else { + imageType.setSelectedIndex(0); + } + } + + public Map getValueMap() { Map parameters = new HashMap(); - if( quicklookNone.getValue() ) { + if (quicklookNone.getValue()) { return parameters; } @@ -132,59 +205,59 @@ public Map getValueMap() { // overlayURL String overlayURLValue = overlayURL.getValue(); String overlayURLXML = ""; - if( overlayURLValue !=null && !overlayURLValue.isEmpty() ) { + if (overlayURLValue != null && !overlayURLValue.isEmpty()) { overlayURLXML = indentXML + "" + overlayURLValue + "\n"; } // maskOverlays String maskOverlaysValue = maskOverlays.getValue(); String maskOverlaysXML = ""; - if( maskOverlaysValue !=null && !maskOverlaysValue.isEmpty() ) { + if (maskOverlaysValue != null && !maskOverlaysValue.isEmpty()) { maskOverlaysXML = indentXML + "" + maskOverlaysValue + "\n"; } // subSamplingX Integer subSamplingXValue = subSamplingX.getValue(); String subSamplingXXML = ""; - if( subSamplingXValue !=null && subSamplingXValue >= 1 ) { + if (subSamplingXValue != null && subSamplingXValue >= 1) { subSamplingXXML = indentXML + "" + subSamplingXValue + "\n"; } // subSamplingY Integer subSamplingYValue = subSamplingY.getValue(); String subSamplingYXML = ""; - if( subSamplingYValue !=null && subSamplingYValue >= 1 ) { + if (subSamplingYValue != null && subSamplingYValue >= 1) { subSamplingYXML = indentXML + "" + subSamplingYValue + "\n"; } // backgroundColor String backgroundColorValue = backgroundColor.getValue(); String backgroundColorXML = ""; - if( backgroundColorValue !=null && !backgroundColorValue.isEmpty() ) { + if (backgroundColorValue != null && !backgroundColorValue.isEmpty()) { backgroundColorXML = indentXML + "" + backgroundColorValue + "\n"; } // legendEnabled Boolean legendEnabledValue = legendEnabled.getValue(); String legendEnabledXML = ""; - if( legendEnabled.getValue() ) { + if (legendEnabled.getValue()) { legendEnabledXML = indentXML + "true\n"; } String quicklookParameters = ""; - if( quicklookSingleBand.getValue() ) { + if (quicklookSingleBand.getValue()) { // bandName String bandNameXML = ""; String bandNameValue = bandName.getValue(); - if( bandNameValue != null && !bandNameValue.isEmpty() ) { - bandNameXML = indentXML + "" + bandNameValue + "\n"; + if (bandNameValue != null && !bandNameValue.isEmpty()) { + bandNameXML = indentXML + "" + bandNameValue + "\n"; } // cpdURL String cpdURLXML = ""; String cpdURLValue = cpdURL.getValue(); - if( cpdURLValue != null && !cpdURLValue .isEmpty() ) { + if (cpdURLValue != null && !cpdURLValue.isEmpty()) { cpdURLXML = indentXML + "" + cpdURLValue + "\n"; } @@ -203,8 +276,7 @@ public Map getValueMap() { " \n" + " \n" + ""; - } - else if( quicklookMultiBand.getValue() ) { + } else if (quicklookMultiBand.getValue()) { // RGBAExpressions String rgbaExpressionsXML = ""; @@ -213,32 +285,32 @@ else if( quicklookMultiBand.getValue() ) { String rgbaExpressionsBlueBandValue = rgbaExpressionsBlueBand.getValue(); String rgbaExpressionsAlphaBandValue = rgbaExpressionsAlphaBand.getValue(); - if( rgbaExpressionsRedBandValue != null && !rgbaExpressionsRedBandValue.isEmpty() && - rgbaExpressionsGreenBandValue != null && !rgbaExpressionsGreenBandValue.isEmpty() && - rgbaExpressionsBlueBandValue != null && !rgbaExpressionsBlueBandValue.isEmpty() ) { + if (rgbaExpressionsRedBandValue != null && !rgbaExpressionsRedBandValue.isEmpty() && + rgbaExpressionsGreenBandValue != null && !rgbaExpressionsGreenBandValue.isEmpty() && + rgbaExpressionsBlueBandValue != null && !rgbaExpressionsBlueBandValue.isEmpty()) { rgbaExpressionsXML = indentXML + "" + rgbaExpressionsRedBandValue + "," + rgbaExpressionsGreenBandValue + "," + rgbaExpressionsBlueBandValue; - if( rgbaExpressionsAlphaBandValue != null && !rgbaExpressionsAlphaBandValue.isEmpty() ) + if (rgbaExpressionsAlphaBandValue != null && !rgbaExpressionsAlphaBandValue.isEmpty()) rgbaExpressionsXML = rgbaExpressionsXML + "," + rgbaExpressionsAlphaBandValue; - rgbaExpressionsXML = rgbaExpressionsXML + "\n"; + rgbaExpressionsXML = rgbaExpressionsXML + "\n"; } // rgbaMinSamples String rgbaMinSamplesXML = ""; String rgbaMinSamplesValue = rgbaMinSamples.getValue(); - if( rgbaMinSamplesValue !=null && !rgbaMinSamplesValue.isEmpty() ) { + if (rgbaMinSamplesValue != null && !rgbaMinSamplesValue.isEmpty()) { rgbaMinSamplesXML = indentXML + "" + rgbaMinSamplesValue + "\n"; } // rgbaMaxSamples String rgbaMaxSamplesXML = ""; String rgbaMaxSamplesValue = rgbaMaxSamples.getValue(); - if( rgbaMaxSamplesValue !=null && !rgbaMaxSamplesValue.isEmpty() ) { + if (rgbaMaxSamplesValue != null && !rgbaMaxSamplesValue.isEmpty()) { rgbaMaxSamplesXML = indentXML + "" + rgbaMaxSamplesValue + "\n"; } @@ -260,7 +332,7 @@ else if( quicklookMultiBand.getValue() ) { ""; } - if( !quicklookParameters.isEmpty() ) { + if (!quicklookParameters.isEmpty()) { parameters.put("quicklooks", "true"); parameters.put("calvalus.ql.parameters", quicklookParameters); } @@ -269,18 +341,17 @@ else if( quicklookMultiBand.getValue() ) { } public void setValues(Map parameters) { - String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); - if( quicklookParametersValue == null || quicklookParametersValue.isEmpty() ) { + if (quicklookParametersValue == null || quicklookParametersValue.isEmpty()) { quicklookNone.setValue(true, true); bandName.setValue(null); cpdURL.setValue(null); - rgbaExpressionsRedBand.setValue( null ); - rgbaExpressionsGreenBand.setValue( null ); - rgbaExpressionsBlueBand.setValue( null ); - rgbaExpressionsAlphaBand.setValue( null ); + rgbaExpressionsRedBand.setValue(null); + rgbaExpressionsGreenBand.setValue(null); + rgbaExpressionsBlueBand.setValue(null); + rgbaExpressionsAlphaBand.setValue(null); rgbaMinSamples.setValue(null); rgbaMaxSamples.setValue(null); @@ -290,13 +361,12 @@ public void setValues(Map parameters) { subSamplingX.setValue(null); maskOverlays.setValue(null); legendEnabled.setValue(false); - } - else { + } else { Document dom = XMLParser.parse(quicklookParametersValue); // bandName String bandNameValue = getTagValue(dom, "bandName"); - if( bandNameValue != null ) + if (bandNameValue != null) quicklookSingleBand.setValue(true, true); bandName.setValue(bandNameValue); @@ -306,20 +376,19 @@ public void setValues(Map parameters) { // RGBAExpressions String RGBAExpressionsValue = getTagValue(dom, "RGBAExpressions"); - if( RGBAExpressionsValue != null ) { + if (RGBAExpressionsValue != null) { String[] tokens = RGBAExpressionsValue.split(","); int numTokens = tokens.length; - rgbaExpressionsRedBand.setValue( numTokens>0 ? tokens[0] : null ); - rgbaExpressionsGreenBand.setValue( numTokens>1 ? tokens[1] : null ); - rgbaExpressionsBlueBand.setValue( numTokens>2 ? tokens[2] : null ); - rgbaExpressionsAlphaBand.setValue( numTokens>3 ? tokens[3] : null ); + rgbaExpressionsRedBand.setValue(numTokens > 0 ? tokens[0] : null); + rgbaExpressionsGreenBand.setValue(numTokens > 1 ? tokens[1] : null); + rgbaExpressionsBlueBand.setValue(numTokens > 2 ? tokens[2] : null); + rgbaExpressionsAlphaBand.setValue(numTokens > 3 ? tokens[3] : null); quicklookMultiBand.setValue(true, true); - } - else{ - rgbaExpressionsRedBand.setValue( null ); - rgbaExpressionsGreenBand.setValue( null ); - rgbaExpressionsBlueBand.setValue( null ); - rgbaExpressionsAlphaBand.setValue( null ); + } else { + rgbaExpressionsRedBand.setValue(null); + rgbaExpressionsGreenBand.setValue(null); + rgbaExpressionsBlueBand.setValue(null); + rgbaExpressionsAlphaBand.setValue(null); } // rgbaMinSamples @@ -332,13 +401,13 @@ public void setValues(Map parameters) { // imageType String imageTypeValue = getTagValue(dom, "imageType"); - if( imageTypeValue == null ) + if (imageTypeValue == null) imageType.setSelectedIndex(0); - else if( imageTypeValue.equalsIgnoreCase("jpeg") ) + else if (imageTypeValue.equalsIgnoreCase("jpeg")) imageType.setSelectedIndex(0); - else if( imageTypeValue.equalsIgnoreCase("tiff") ) + else if (imageTypeValue.equalsIgnoreCase("tiff")) imageType.setSelectedIndex(1); - else if( imageTypeValue.equalsIgnoreCase("png") ) + else if (imageTypeValue.equalsIgnoreCase("png")) imageType.setSelectedIndex(2); // overlayURL @@ -354,8 +423,7 @@ else if( imageTypeValue.equalsIgnoreCase("png") ) try { Integer subSamplingXIntValue = Integer.valueOf(subSamplingXValue); subSamplingX.setValue(subSamplingXIntValue); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { subSamplingX.setValue(null); } @@ -364,8 +432,7 @@ else if( imageTypeValue.equalsIgnoreCase("png") ) try { Integer subSamplingYIntValue = Integer.valueOf(subSamplingYValue); subSamplingY.setValue(subSamplingYIntValue); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { subSamplingY.setValue(null); } @@ -375,7 +442,7 @@ else if( imageTypeValue.equalsIgnoreCase("png") ) // legendEnabled String legendEnabledValue = getTagValue(dom, "legendEnabled"); - if( legendEnabledValue != null && legendEnabledValue.equalsIgnoreCase("true") ) + if (legendEnabledValue != null && legendEnabledValue.equalsIgnoreCase("true")) legendEnabled.setValue(true); else legendEnabled.setValue(false); @@ -389,84 +456,10 @@ private String getTagValue(Document dom, String tagName) { if (node != null && node.hasChildNodes()) { Node firstChild = node.getFirstChild(); if (firstChild != null) { - return(firstChild.getNodeValue()); + return (firstChild.getNodeValue()); } } } return null; } - - private void setQuicklookNone() { - bandName.setEnabled(false); - cpdURL.setEnabled(false); - - rgbaExpressionsRedBand.setEnabled(false); - rgbaExpressionsGreenBand.setEnabled(false); - rgbaExpressionsBlueBand.setEnabled(false); - rgbaExpressionsAlphaBand.setEnabled(false); - rgbaMinSamples.setEnabled(false); - rgbaMaxSamples.setEnabled(false); - - imageType.setEnabled(false); - overlayURL.setEnabled(false); - maskOverlays.setEnabled(false); - subSamplingX.setEnabled(false); - subSamplingY.setEnabled(false); - backgroundColor.setEnabled(false); - legendEnabled.setEnabled(false); - } - - private void setQuicklookSingleBandEnabled() { - bandName.setEnabled(true); - cpdURL.setEnabled(true); - - rgbaExpressionsRedBand.setEnabled(false); - rgbaExpressionsGreenBand.setEnabled(false); - rgbaExpressionsBlueBand.setEnabled(false); - rgbaExpressionsAlphaBand.setEnabled(false); - rgbaMinSamples.setEnabled(false); - rgbaMaxSamples.setEnabled(false); - - imageType.setEnabled(true); - overlayURL.setEnabled(true); - maskOverlays.setEnabled(true); - subSamplingX.setEnabled(true); - subSamplingY.setEnabled(true); - backgroundColor.setEnabled(true); - legendEnabled.setEnabled(true); - } - - private void setQuicklookMultiBandEnabled() { - bandName.setEnabled(false); - cpdURL.setEnabled(false); - - rgbaExpressionsRedBand.setEnabled(true); - rgbaExpressionsGreenBand.setEnabled(true); - rgbaExpressionsBlueBand.setEnabled(true); - rgbaExpressionsAlphaBand.setEnabled(true); - rgbaMinSamples.setEnabled(true); - rgbaMaxSamples.setEnabled(true); - - imageType.setEnabled(true); - overlayURL.setEnabled(true); - maskOverlays.setEnabled(true); - subSamplingX.setEnabled(true); - subSamplingY.setEnabled(true); - backgroundColor.setEnabled(true); - legendEnabled.setEnabled(true); - } - - public void setAvailableImageTypes() { - String [] imageNames = {"jpeg", "tiff", "png"}; - int selectedIndex = imageType.getSelectedIndex(); - imageType.clear(); - for( String imageName : imageNames) { - imageType.addItem(imageName); - } - if (selectedIndex >= 0 && selectedIndex < imageNames.length) { - imageType.setSelectedIndex(selectedIndex); - } else { - imageType.setSelectedIndex(0); - } - } } \ No newline at end of file From 4ea67acb435f8bc613fc3e39c56eb94038eee0aa Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Mon, 30 Apr 2018 19:29:17 +0100 Subject: [PATCH 06/25] Band name - add processor variables selection list User can select band name from the list, or enter a custom band name --- .../portal/client/OrderL2ProductionView.java | 10 ++++- .../client/QuicklookParametersForm.java | 45 +++++++++++++++++-- .../client/QuicklookParametersForm.ui.xml | 7 +++ 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java index b133f01c9..de201dc1a 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java @@ -18,6 +18,7 @@ import com.bc.calvalus.portal.shared.DtoInputSelection; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; +import com.bc.calvalus.portal.shared.DtoProcessorVariable; import com.google.gwt.core.client.Scheduler; import com.google.gwt.user.client.rpc.AsyncCallback; import com.google.gwt.user.client.ui.Anchor; @@ -221,6 +222,13 @@ private void handleProcessorChanged() { add("BigGeoTiff", outputFormats); } outputParametersForm.setAvailableOutputFormats(outputFormats.toArray(new String[0])); + + List processorVariables = new ArrayList<>(); + DtoProcessorVariable[] dtoProcessorVariables = processorDescriptor.getProcessorVariables(); + for (DtoProcessorVariable dtoProcessorVariable : dtoProcessorVariables) { + processorVariables.add(dtoProcessorVariable.getName()); + } + quicklookParametersForm.setProcessorVariables(processorVariables.toArray(new String[0])); } } @@ -229,4 +237,4 @@ private static void add(String format, List outputFormats) { outputFormats.add(format); } } -} \ No newline at end of file +} diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index a824b5a79..1441e9174 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -17,6 +17,8 @@ package com.bc.calvalus.portal.client; import com.google.gwt.core.client.GWT; +import com.google.gwt.event.dom.client.ChangeEvent; +import com.google.gwt.event.dom.client.ChangeHandler; import com.google.gwt.event.logical.shared.ValueChangeEvent; import com.google.gwt.event.logical.shared.ValueChangeHandler; import com.google.gwt.uibinder.client.UiBinder; @@ -55,6 +57,8 @@ interface TheUiBinder extends UiBinder { @UiField RadioButton quicklookMultiBand; + @UiField + ListBox bandNameListBox; @UiField TextBox bandName; @UiField @@ -112,6 +116,19 @@ public void onValueChange(ValueChangeEvent event) { } }); + bandNameListBox.addChangeHandler(new ChangeHandler() { + @Override + public void onChange(ChangeEvent event) { + int selectedIndex = bandNameListBox.getSelectedIndex(); + if (selectedIndex == 0) { + bandName.setEnabled(true); + } else if (selectedIndex > 0) { + bandName.setValue(bandNameListBox.getValue(selectedIndex)); + bandName.setEnabled(false); + } + } + }); + setAvailableImageTypes(); quicklookNone.setValue(true); setQuicklookNone(); @@ -191,6 +208,18 @@ public void setAvailableImageTypes() { } } + public void setProcessorVariables(String... processorVariables) { + bandNameListBox.clear(); + bandNameListBox.addItem(""); + bandNameListBox.setSelectedIndex(0); + if (processorVariables == null) + return; + + for (String processorVariable : processorVariables) { + bandNameListBox.addItem(processorVariable); + } + } + public Map getValueMap() { Map parameters = new HashMap(); if (quicklookNone.getValue()) { @@ -366,9 +395,19 @@ public void setValues(Map parameters) { // bandName String bandNameValue = getTagValue(dom, "bandName"); - if (bandNameValue != null) + if (bandNameValue != null) { quicklookSingleBand.setValue(true, true); - bandName.setValue(bandNameValue); + bandName.setValue(bandNameValue); + int itemCount = bandNameListBox.getItemCount(); + for (int i = 0; i < itemCount; i++) { + String listBand = bandNameListBox.getValue(i); + if (listBand != null && listBand.equals(bandNameValue)) { + bandNameListBox.setSelectedIndex(i); + bandName.setEnabled(false); + break; + } + } + } // cpdURL String cpdURLValue = getTagValue(dom, "cpdURL"); @@ -462,4 +501,4 @@ private String getTagValue(Document dom, String tagName) { } return null; } -} \ No newline at end of file +} diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index bf83b07bc..22fe91141 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -35,6 +35,13 @@     Band name: + + + + + +     +       From 0110169b81ff15cf32b2c71d5dad07c2148553f5 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Thu, 17 May 2018 18:57:13 +0100 Subject: [PATCH 07/25] Add color palette drop-down list The list is populated from the backend through a configuration file. The default configuration file on the Calvalus filesytem is: /calvalus/auxiliary/color-palette-sets.csv An example entry in color-palette-sets.csv is: #name, path wew_water_chl;hdfs://myserver:9000/calvalus/auxiliary/wew_water_chl.cpd --- .../inventory/ColorPaletteService.java | 40 ++++++ .../calvalus/inventory/ColorPaletteSet.java | 45 +++++++ .../inventory/ColorPaletteSetPersistable.java | 67 ++++++++++ .../inventory/DefaultColorPaletteService.java | 115 ++++++++++++++++++ .../DefaultColorPaletteServiceTest.java | 35 ++++++ .../portal/client/CalvalusPortal.java | 24 ++++ .../calvalus/portal/client/PortalContext.java | 3 + .../client/QuicklookParametersForm.java | 50 ++++++-- .../portal/server/BackendServiceImpl.java | 25 ++++ .../portal/shared/BackendService.java | 11 ++ .../portal/shared/BackendServiceAsync.java | 2 + .../portal/shared/DtoColorPaletteSet.java | 75 ++++++++++++ .../client/QuicklookParametersForm.ui.xml | 2 +- .../calvalus/production/ServiceContainer.java | 10 +- .../hadoop/HadoopServiceContainerFactory.java | 6 +- .../local/LocalServiceContainerFactory.java | 5 +- .../production/TestColorPaletteService.java | 40 ++++++ .../calvalusfacade/CalvalusProduction.java | 7 ++ 18 files changed, 547 insertions(+), 15 deletions(-) create mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java create mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java create mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java create mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java create mode 100644 calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java create mode 100644 calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java new file mode 100644 index 000000000..d474ed4ca --- /dev/null +++ b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.inventory; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * The interface to the Calvalus color palette inventory. + * + * @author Declan + */ +public interface ColorPaletteService { + + /** + * Gets color palette sets. + * + * @param filter A filter expression (unused) + * + * @return The array color palette sets, which may be empty. + * + * @throws java.io.IOException If an I/O error occurs + */ + ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException; +} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java new file mode 100644 index 000000000..bd9803db5 --- /dev/null +++ b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.inventory; + +/** + * Color palette set. + * + * @author Declan + */ +public class ColorPaletteSet { + + private final String name; + private final String path; + + public ColorPaletteSet(String name, String path) { + if (name == null) { + throw new NullPointerException("name"); + } + if (path == null) { + throw new NullPointerException("path"); + } + this.name = name; + this.path = path; + } + + public String getName() { + return name; + } + + public String getPath() { return path; } +} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java new file mode 100644 index 000000000..5b3d1e7e4 --- /dev/null +++ b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.inventory; + +/** + * this class is responsible for converting a {@link ProductSet} into a {@link String} and back. + * + * @author Declan + */ +public class ColorPaletteSetPersistable { + + public static final String FILENAME = "color-palette-sets.csv"; + + private ColorPaletteSetPersistable() { + } + + public static String convertToCSV(ColorPaletteSet colorPaletteSet) { + StringBuilder sb = new StringBuilder(); + sb.append(colorPaletteSet.getName()); + sb.append(';'); + sb.append(colorPaletteSet.getPath()); + sb.append(';'); + return sb.toString(); + } + + public static ColorPaletteSet convertFromCSV(String text) { + String trimmedText = text.trim(); + if (trimmedText.startsWith("#") || trimmedText.isEmpty()) { + return null; //comments are ignored + } + String[] splits = trimmedText.split(";"); + + String name; + String path; + + if (splits.length >= 2) { + name = nullAware(splits[0]); + path = nullAware(splits[1]); + } else { + // less than 2 fields currently not supported + return null; + } + return new ColorPaletteSet(name, path); + } + + static String nullAware(String text) { + if (text == null || "null".equals(text)) { + return null; + } else { + return text; + } + } +} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java new file mode 100644 index 000000000..ea82a6e82 --- /dev/null +++ b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.inventory; + +import com.bc.calvalus.commons.CalvalusLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + + +/** + * Default implementation of the {@link ColorPaletteService}. + * + * @author Declan + */ +public class DefaultColorPaletteService implements ColorPaletteService { + + private static final Logger LOG = CalvalusLogger.getLogger(); + private static final String USER_FILTER = "user="; + private final AbstractFileSystemService fileSystemService; + private final boolean withExternalAccessControl; + private String colorPaletteRootDir = "aux"; + + public DefaultColorPaletteService(AbstractFileSystemService fileSystemService, String colorPaletteRootDir) { + this.fileSystemService = fileSystemService; + this.colorPaletteRootDir = colorPaletteRootDir; + withExternalAccessControl = Boolean.getBoolean("calvalus.accesscontrol.external"); + } + + @Override + public ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException { + UserGroupInformation remoteUser = UserGroupInformation.createRemoteUser(username); + try { + return remoteUser.doAs((PrivilegedExceptionAction) () -> { + FileSystem fileSystem = fileSystemService.getFileSystem(username); + LOG.fine("DefaultColorPaletteService user " + username + " fs " + fileSystem + " jcm " + fileSystemService.getJobClientsMap()); + return loadPredefined(fileSystem); + }); + } catch (InterruptedException e) { + throw new IOException("failed to retrieve color palette sets for user " + username, e); + } + } + + private ColorPaletteSet[] loadPredefined(FileSystem fileSystem) throws IOException { + Path databasePath = fileSystemService.makeQualified(fileSystem, colorPaletteRootDir + "/" + ColorPaletteSetPersistable.FILENAME ); + if (fileSystem.exists(databasePath)) { + final ColorPaletteSet[] colorPaletteSets = readColorPaletteSets(fileSystem, new Path[]{databasePath}); + return colorPaletteSets; + } + return new ColorPaletteSet[0]; + } + + private ColorPaletteSet[] readColorPaletteSets(FileSystem fileSystem, Path[] paths) throws IOException { + if (paths == null || paths.length == 0) { + return new ColorPaletteSet[0]; + } else { + List colorPaletteSetList = new ArrayList(); + for (Path path : paths) { + try { + colorPaletteSetList.addAll(readColorPaletteSetFile(fileSystem, path)); + } catch (AccessControlException ignore) {} + } + return colorPaletteSetList.toArray(new ColorPaletteSet[0]); + } + } + + private List readColorPaletteSetFile(FileSystem fileSystem, Path path) throws IOException { + return readColorPaletteSetFromCsv(fileSystem.open(path)); + } + + static List readColorPaletteSetFromCsv(InputStream is) throws IOException { + InputStreamReader reader = new InputStreamReader(is); + try (BufferedReader bufferedReader = new BufferedReader(reader)) { + List colorPaletteSets = new ArrayList(); + + String line = bufferedReader.readLine(); + while (line != null) { + ColorPaletteSet colorPaletteSet= ColorPaletteSetPersistable.convertFromCSV(line); + if (colorPaletteSet != null) { + colorPaletteSets.add(colorPaletteSet); + } + line = bufferedReader.readLine(); + } + return colorPaletteSets; + } + } +} diff --git a/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java b/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java new file mode 100644 index 000000000..bbf0b614f --- /dev/null +++ b/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.inventory; + +import org.junit.Test; + +/** + * Test implementation of DefaultColorPaletteService. + * + * @author Declan + */ +public class DefaultColorPaletteServiceTest { + + @Test + public void testCreatePaletteSet() throws Exception { + ColorPaletteSet[] test = new ColorPaletteSet[]{ + new ColorPaletteSet("name0", "path0"), + new ColorPaletteSet("name1", "path1"), + }; + } +} diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java index 695911351..f4c0ae114 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java @@ -11,6 +11,7 @@ import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; +import com.bc.calvalus.portal.shared.DtoColorPaletteSet; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -74,6 +75,7 @@ public class CalvalusPortal implements EntryPoint, PortalContext { // Data provided by various external services private ListDataProvider regions; private DtoProductSet[] productSets; + private DtoColorPaletteSet[] colorPaletteSets; private DtoProcessorDescriptor[] systemProcessors; private DtoProcessorDescriptor[] userProcessors; private DtoProcessorDescriptor[] allUserProcessors; @@ -128,6 +130,7 @@ public void onModuleLoad() { public void run() { backendService.loadRegions(NO_FILTER, new InitRegionsCallback()); backendService.getProductSets(NO_FILTER, new InitProductSetsCallback()); + backendService.getColorPaletteSets(NO_FILTER, new InitColorPaletteSetsCallback()); final BundleFilter systemFilter = new BundleFilter(); systemFilter.withProvider(BundleFilter.PROVIDER_SYSTEM); @@ -181,6 +184,11 @@ public DtoProductSet[] getProductSets() { return productSets; } + @Override + public DtoColorPaletteSet[] getColorPaletteSets() { + return colorPaletteSets; + } + @Override public DtoProcessorDescriptor[] getProcessors(String filter) { if (filter.equals(BundleFilter.PROVIDER_SYSTEM)) { @@ -476,6 +484,22 @@ public void onFailure(Throwable caught) { } } + private class InitColorPaletteSetsCallback implements AsyncCallback { + + @Override + public void onSuccess(DtoColorPaletteSet[] dtoColorPaletteSets) { + CalvalusPortal.this.colorPaletteSets = dtoColorPaletteSets; + maybeInitFrontend(); + } + + @Override + public void onFailure(Throwable caught) { + caught.printStackTrace(System.err); + Dialog.error("Server-side Error", caught.getMessage()); + CalvalusPortal.this.colorPaletteSets = new DtoColorPaletteSet[0]; + } + } + private class InitProcessorsCallback implements AsyncCallback { private final String filter; diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java index afd2b87ea..ccb931c91 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java @@ -5,6 +5,7 @@ import com.bc.calvalus.portal.shared.BackendServiceAsync; import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; +import com.bc.calvalus.portal.shared.DtoColorPaletteSet; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -27,6 +28,8 @@ public interface PortalContext { // make this return ListDataProvider DtoProductSet[] getProductSets(); + DtoColorPaletteSet[] getColorPaletteSets(); + // make this return ListDataProvider DtoProcessorDescriptor[] getProcessors(String filter); DtoAggregatorDescriptor[] getAggregators(String filter); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 1441e9174..125e42636 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -16,6 +16,7 @@ package com.bc.calvalus.portal.client; +import com.bc.calvalus.portal.shared.DtoColorPaletteSet; import com.google.gwt.core.client.GWT; import com.google.gwt.event.dom.client.ChangeEvent; import com.google.gwt.event.dom.client.ChangeHandler; @@ -50,6 +51,8 @@ interface TheUiBinder extends UiBinder { private static TheUiBinder uiBinder = GWT.create(TheUiBinder.class); + private final PortalContext portal; + @UiField RadioButton quicklookNone; @UiField @@ -62,7 +65,7 @@ interface TheUiBinder extends UiBinder { @UiField TextBox bandName; @UiField - TextBox cpdURL; + ListBox colorPalette; @UiField TextBox rgbaExpressionsRedBand; @@ -92,7 +95,11 @@ interface TheUiBinder extends UiBinder { @UiField CheckBox legendEnabled; + private DtoColorPaletteSet[] colorPaletteSet; + public QuicklookParametersForm(PortalContext portalContext) { + this.portal = portalContext; + initWidget(uiBinder.createAndBindUi(this)); quicklookNone.addValueChangeHandler(new ValueChangeHandler() { @@ -130,13 +137,15 @@ public void onChange(ChangeEvent event) { }); setAvailableImageTypes(); + setColorPaletteSets(); + quicklookNone.setValue(true); setQuicklookNone(); } private void setQuicklookNone() { bandName.setEnabled(false); - cpdURL.setEnabled(false); + colorPalette.setEnabled(false); rgbaExpressionsRedBand.setEnabled(false); rgbaExpressionsGreenBand.setEnabled(false); @@ -156,7 +165,7 @@ private void setQuicklookNone() { private void setQuicklookSingleBandEnabled() { bandName.setEnabled(true); - cpdURL.setEnabled(true); + colorPalette.setEnabled(true); rgbaExpressionsRedBand.setEnabled(false); rgbaExpressionsGreenBand.setEnabled(false); @@ -176,7 +185,7 @@ private void setQuicklookSingleBandEnabled() { private void setQuicklookMultiBandEnabled() { bandName.setEnabled(false); - cpdURL.setEnabled(false); + colorPalette.setEnabled(false); rgbaExpressionsRedBand.setEnabled(true); rgbaExpressionsGreenBand.setEnabled(true); @@ -208,6 +217,16 @@ public void setAvailableImageTypes() { } } + private void setColorPaletteSets() { + this.colorPaletteSet = portal.getColorPaletteSets(); + colorPalette.clear(); + colorPalette.addItem(""); + for (DtoColorPaletteSet dtoColorPalette : this.colorPaletteSet ) { + colorPalette.addItem(dtoColorPalette.getPath()); + } + colorPalette.setSelectedIndex(0); + } + public void setProcessorVariables(String... processorVariables) { bandNameListBox.clear(); bandNameListBox.addItem(""); @@ -283,11 +302,11 @@ public Map getValueMap() { bandNameXML = indentXML + "" + bandNameValue + "\n"; } - // cpdURL + // color palette String cpdURLXML = ""; - String cpdURLValue = cpdURL.getValue(); - if (cpdURLValue != null && !cpdURLValue.isEmpty()) { - cpdURLXML = indentXML + "" + cpdURLValue + "\n"; + String colorPaletteValue = colorPalette.getSelectedValue(); + if (colorPaletteValue != null && !colorPaletteValue.isEmpty()) { + cpdURLXML = indentXML + "" + colorPaletteValue + "\n"; } quicklookParameters = "\n" + @@ -375,7 +394,7 @@ public void setValues(Map parameters) { quicklookNone.setValue(true, true); bandName.setValue(null); - cpdURL.setValue(null); + colorPalette.setSelectedIndex(0); rgbaExpressionsRedBand.setValue(null); rgbaExpressionsGreenBand.setValue(null); @@ -409,9 +428,18 @@ public void setValues(Map parameters) { } } - // cpdURL + // color palette String cpdURLValue = getTagValue(dom, "cpdURL"); - cpdURL.setValue(cpdURLValue); + if (cpdURLValue == null || this.colorPaletteSet == null) { + colorPalette.setSelectedIndex(0); + } else { + for (int i=0; i < this.colorPaletteSet.length; i++ ) { + if( this.colorPaletteSet[i].getPath().equals(cpdURLValue) ) { + colorPalette.setSelectedIndex(i+1); + break; + } + } + } // RGBAExpressions String RGBAExpressionsValue = getTagValue(dom, "RGBAExpressions"); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java index 9e5cf3d09..8680df698 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java @@ -23,12 +23,14 @@ import com.bc.calvalus.commons.WorkflowItem; import com.bc.calvalus.commons.shared.BundleFilter; import com.bc.calvalus.inventory.AbstractFileSystemService; +import com.bc.calvalus.inventory.ColorPaletteSet; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.ProductSet; import com.bc.calvalus.portal.shared.BackendService; import com.bc.calvalus.portal.shared.BackendServiceException; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; +import com.bc.calvalus.portal.shared.DtoColorPaletteSet; import com.bc.calvalus.portal.shared.DtoMaskDescriptor; import com.bc.calvalus.portal.shared.DtoParameterDescriptor; import com.bc.calvalus.portal.shared.DtoProcessState; @@ -265,6 +267,24 @@ public DtoProductSet[] getProductSets(String filter) throws BackendServiceExcept } } + @Override + public DtoColorPaletteSet[] getColorPaletteSets(String filter) throws BackendServiceException { + if (filter.contains("dummy")) { + filter = filter.replace("dummy", getUserName()); + } + try { + ColorPaletteSet[] colorPaletteSets = serviceContainer.getColorPaletteService().getColorPaletteSets(getUserName(), filter); + DtoColorPaletteSet[] dtoColorPaletteSets = new DtoColorPaletteSet[colorPaletteSets.length]; + for (int i = 0; i < dtoColorPaletteSets.length; i++) { + dtoColorPaletteSets[i] = convert(colorPaletteSets[i]); + } + LOG.fine("dtoColorPaletteSets returns " + dtoColorPaletteSets.length); + return dtoColorPaletteSets; + } catch (IOException e) { + throw convert(e); + } + } + @Override public DtoProcessorDescriptor[] getProcessors(String filterString) throws BackendServiceException { try { @@ -710,6 +730,11 @@ private DtoProductSet convert(ProductSet productSet) { productSet.getGeoInventory()); } + private DtoColorPaletteSet convert(ColorPaletteSet colorPaletteSet) { + return new DtoColorPaletteSet(colorPaletteSet.getName(), + colorPaletteSet.getPath()); + } + private DtoProcessorDescriptor convert(String bundleName, String bundleVersion, String bundlePath, String owner, ProcessorDescriptor processorDescriptor) { return new DtoProcessorDescriptor(processorDescriptor.getExecutableName(), diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java index d2024b5d0..8c30a5eb7 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java @@ -43,6 +43,17 @@ public interface BackendService extends RemoteService { */ DtoProductSet[] getProductSets(String filter) throws BackendServiceException; + /** + * Gets all known color palette sets. + * + * @param filter A filter expression (not yet used). + * + * @return The product sets. + * + * @throws BackendServiceException If a server error occurred. + */ + DtoColorPaletteSet[] getColorPaletteSets(String filter) throws BackendServiceException; + /** * Gets all known processors. * diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java index 165fb7198..a7fd283c5 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java @@ -23,6 +23,8 @@ public interface BackendServiceAsync { void getProductSets(String filter, AsyncCallback callback); + void getColorPaletteSets(String filter, AsyncCallback callback); + void getProcessors(String filter, AsyncCallback callback); void getAggregators(String filter, AsyncCallback callback); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java new file mode 100644 index 000000000..3399293d5 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.portal.shared; + +import com.google.gwt.user.client.rpc.IsSerializable; + +/** + * GWT-serializable version of the {@link com.bc.calvalus.inventory.ColorPaletteSet} class. + * + * @author Declan + */ +public class DtoColorPaletteSet implements IsSerializable { + + private String name; + private String path; + + /** + * No-arg constructor as required by {@link IsSerializable}. Don't use directly. + */ + public DtoColorPaletteSet() { + } + + public DtoColorPaletteSet(String name, String path) { + if (name == null) { + throw new NullPointerException("name"); + } + if (path == null) { + throw new NullPointerException("path"); + } + this.name = name; + this.path = path; + } + + public String getName() { + return name; + } + + public String getPath() { + return path; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DtoColorPaletteSet that = (DtoColorPaletteSet) o; + + if (!name.equals(that.name)) { + return false; + } + if (!path.equals(that.path)) { + return false; + } + return true; + } +} diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index 22fe91141..07ac1e224 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -50,7 +50,7 @@     Colour table: -   +   diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java b/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java index 10d9ae3d1..9c2cbc6f2 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java @@ -16,6 +16,7 @@ package com.bc.calvalus.production; +import com.bc.calvalus.inventory.ColorPaletteService; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.InventoryService; import org.apache.hadoop.conf.Configuration; @@ -27,15 +28,18 @@ public class ServiceContainer { private final ProductionService productionService; private final FileSystemService fileSystemService; private final InventoryService inventoryService; + private final ColorPaletteService colorPaletteService; private final Configuration hadoopConfiguration; public ServiceContainer(ProductionService productionService, FileSystemService fileSystemService, - InventoryService inventoryService, + InventoryService inventoryService, + ColorPaletteService colorPaletteService, Configuration hadoopConfiguration) { this.productionService = productionService; this.fileSystemService = fileSystemService; this.inventoryService = inventoryService; + this.colorPaletteService = colorPaletteService; this.hadoopConfiguration = hadoopConfiguration; } @@ -51,6 +55,10 @@ public InventoryService getInventoryService() { return inventoryService; } + public ColorPaletteService getColorPaletteService() { + return colorPaletteService; + } + public Configuration getHadoopConfiguration() { return hadoopConfiguration; } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java index c66d77504..40abb814c 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java @@ -1,6 +1,8 @@ package com.bc.calvalus.production.hadoop; import com.bc.calvalus.JobClientsMap; +import com.bc.calvalus.inventory.ColorPaletteService; +import com.bc.calvalus.inventory.DefaultColorPaletteService; import com.bc.calvalus.inventory.DefaultInventoryService; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.InventoryService; @@ -43,6 +45,7 @@ public ServiceContainer create(Map serviceConfiguration, // Prevent Windows from using ';' as path separator System.setProperty("path.separator", ":"); String archiveRootDir = serviceConfiguration.getOrDefault("calvalus.portal.archiveRootDir", "eodata"); + String colorPaletteRootDir = serviceConfiguration.getOrDefault("calvalus.portal.colorPaletteRootDir", "auxiliary"); String softwareDir = serviceConfiguration.getOrDefault("calvalus.portal.softwareDir", HadoopProcessingService.CALVALUS_SOFTWARE_PATH); // disable cache, otherwise org.apache.hadoop.fs.FileSystem.Cache grows forever @@ -59,6 +62,7 @@ public ServiceContainer create(Map serviceConfiguration, JobClientsMap jobClientsMap = new JobClientsMap(jobConf); final HdfsFileSystemService hdfsFileSystemService = new HdfsFileSystemService(jobClientsMap); final InventoryService inventoryService = new DefaultInventoryService(hdfsFileSystemService, archiveRootDir); + final ColorPaletteService colorPaletteService = new DefaultColorPaletteService(hdfsFileSystemService, colorPaletteRootDir); final HadoopProcessingService processingService = new HadoopProcessingService(jobClientsMap, softwareDir); final ProductionStore productionStore; if ("memory".equals(serviceConfiguration.get("production.db.type"))) { @@ -81,7 +85,7 @@ public ServiceContainer create(Map serviceConfiguration, productionStore, productionTypes); stagingService.setProductionService((Observable) productionService); - return new ServiceContainer(productionService, hdfsFileSystemService, inventoryService, hadoopConfiguration); + return new ServiceContainer(productionService, hdfsFileSystemService, inventoryService, colorPaletteService, hadoopConfiguration); } catch (IOException e) { throw new ProductionException("Failed to create Hadoop JobClient." + e.getMessage(), e); } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java b/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java index 11b97942f..f23777a1f 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java @@ -17,6 +17,8 @@ package com.bc.calvalus.production.local; import com.bc.calvalus.inventory.AbstractFileSystemService; +import com.bc.calvalus.inventory.ColorPaletteService; +import com.bc.calvalus.inventory.DefaultColorPaletteService; import com.bc.calvalus.inventory.DefaultInventoryService; import com.bc.calvalus.inventory.InventoryService; import com.bc.calvalus.processing.AggregatorDescriptor; @@ -48,6 +50,7 @@ public ServiceContainer create(Map serviceConfiguration, AbstractFileSystemService fileSystemService = new LocalFileSystemService(); InventoryService inventoryService = new DefaultInventoryService(fileSystemService, "eodata"); + ColorPaletteService colorPaletteService = new DefaultColorPaletteService(fileSystemService, "auxiliary"); ProcessorDescriptor case2r = new ProcessorDescriptor("pc1", "MERIS IOP Case2R", "1.5", "a=2\nb=5", new ProcessorDescriptor.Variable("chl_conc", "AVG", "0.5"), @@ -155,7 +158,7 @@ public ServiceContainer create(Map serviceConfiguration, "autoStaging", "false")); } - return new ServiceContainer(productionService, fileSystemService, inventoryService, new Configuration()); + return new ServiceContainer(productionService, fileSystemService, inventoryService, colorPaletteService, new Configuration()); } diff --git a/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java b/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java new file mode 100644 index 000000000..f73502186 --- /dev/null +++ b/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.production; + +import com.bc.calvalus.inventory.ColorPaletteService; +import com.bc.calvalus.inventory.ColorPaletteSet; +import org.junit.Ignore; + +import java.io.IOException; + +/** + * Test implementation of ColorPaletteService. + * + * @author Declan + */ +@Ignore +public class TestColorPaletteService implements ColorPaletteService { + + @Override + public ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException { + return new ColorPaletteSet[]{ + new ColorPaletteSet("name0", "path0"), + new ColorPaletteSet("name1", "path1"), + }; + } +} diff --git a/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java b/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java index 89574574d..4bdb81f33 100644 --- a/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java +++ b/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java @@ -4,6 +4,7 @@ import com.bc.calvalus.commons.ProcessState; import com.bc.calvalus.commons.ProcessStatus; import com.bc.calvalus.commons.WorkflowItem; +import com.bc.calvalus.inventory.ColorPaletteSet; import com.bc.calvalus.inventory.ProductSet; import com.bc.calvalus.production.Production; import com.bc.calvalus.production.ProductionException; @@ -149,6 +150,12 @@ private ProductSet[] getProductSets(String userName, ServiceContainer serviceCon return productSets.toArray(new ProductSet[productSets.size()]); } + private ColorPaletteSet[] getColorPaletteSets(String userName, ServiceContainer serviceContainer) throws ProductionException, IOException { + List colorPaletteSets = new ArrayList<>(); + colorPaletteSets.addAll(Arrays.asList(serviceContainer.getColorPaletteService().getColorPaletteSets(userName, ""))); + colorPaletteSets.addAll(Arrays.asList(serviceContainer.getColorPaletteService().getColorPaletteSets(userName, "user=" + userName))); + return colorPaletteSets.toArray(new ColorPaletteSet[colorPaletteSets.size()]); + } private void observeProduction(ProductionService productionService, Production production) throws InterruptedException { final Thread shutDownHook = createShutdownHook(production.getWorkflow()); From f293913196aa2aa19d1184789d7153d4525589f8 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Wed, 20 Jun 2018 13:24:59 +0200 Subject: [PATCH 08/25] Fixes for 'mvn clean install' Fixes for: * revert test results, which include an expected time, back to Central Europe timezone as now developing on a server in that timezone. * Update dependency versions in maven pom to fix build errors. --- .../reporting/code/CodeReportTest.java | 12 ++++---- .../CalvalusDataInputsTest.java | 30 +++++++++---------- .../wps/utils/ProductMetadataTest.java | 14 ++++----- .../CalvalusExecuteOperationTest.java | 4 +-- .../CalvalusGetStatusOperationTest.java | 6 ++-- pom.xml | 6 ++-- 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java b/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java index 6f3ef86dc..aec4955b7 100644 --- a/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java +++ b/calvalus-reporting/src/test/java/com/bc/calvalus/reporting/code/CodeReportTest.java @@ -51,7 +51,7 @@ public void canDeserializeCorrectly() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -120,7 +120,7 @@ public void canDeserializeL2ProcessFormatting() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -189,7 +189,7 @@ public void canDeserializeL2ProcessNoProcessor() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -258,7 +258,7 @@ public void canDeserializeL3ProcessFormatting() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -327,7 +327,7 @@ public void canDeserializeL3ProcessAggregation() { assertThat(codeReport.toJson(), containsString("{\n" + " \"requestId\": \"job-01\",\n" + " \"jobName\": \"Job 01\",\n" + - " \"jobSubmissionTime\": \"2018-01-01T00:00:00.000Z\",\n" + + " \"jobSubmissionTime\": \"2018-01-01T01:00:00.000Z\",\n" + " \"userName\": \"user\",\n" + " \"queueName\": \"queue\",\n" + " \"inProducts\": \"/path/to/output\",\n" + @@ -415,4 +415,4 @@ public void canRoundDecimalNumbersWhenDeserialize() { assertThat(codeReport.toJson(), containsString(" \"version\": \"1.0\"\n" + "}")); } -} +} \ No newline at end of file diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java index 6736fbbca..db4832222 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/calvalusfacade/CalvalusDataInputsTest.java @@ -57,8 +57,8 @@ public void canGetInputMapFormatted() throws Exception { assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.0b411")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T00:00:00Z")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T01:00:00+01:00")); } @Test @@ -77,8 +77,8 @@ public void canGetInputMapFormattedWithProductionSetDateRange() throws Exception assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.0b411")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2000-01-01T00:00:00Z")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2010-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2000-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2010-01-01T01:00:00+01:00")); } @Test @@ -97,8 +97,8 @@ public void canGetInputMapFormattedWithProductionSetGeoDb() throws Exception { assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.calvalus.bundle"), equalTo("calvalus-2.10-SNAPSHOT")); assertThat(calvalusDataInputs.getInputMapFormatted().get("calvalus.snap.bundle"), equalTo("snap-3.0.0")); assertThat(calvalusDataInputs.getInputMapFormatted().get("productionName"), equalTo("dummyProductionName")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T00:00:00Z")); - assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T00:00:00Z")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("minDateSource"), equalTo("2016-01-01T01:00:00+01:00")); + assertThat(calvalusDataInputs.getInputMapFormatted().get("maxDateSource"), equalTo("2017-01-01T01:00:00+01:00")); assertThat(calvalusDataInputs.getInputMapFormatted().get("geoInventory"), equalTo("/calvalus/geoInventory/URBAN_FOOTPRINT_GUF_GLOBAL_75m")); } @@ -280,7 +280,7 @@ public void canListAllParameters() throws Exception { assertThat(calvalusDataInputs.toString(), equalTo("calvalus.wps.remote.ref : 1738ad7b-534e-4aca-9861-b26fb9c0f983\n" + - "minDate : 2016-01-01T00:00:00Z\n" + + "minDate : 2016-01-01T01:00:00+01:00\n" + "calvalus.output.compression : none\n" + "processorBundleLocation : hdfs://calvalus/calvalus/software/1.0/beam-buildin-1.0\n" + "calvalus.wps.remote.user : dummyRemoteUser\n" + @@ -290,16 +290,16 @@ public void canListAllParameters() throws Exception { "processorBundleVersion : null\n" + "calvalus.calvalus.bundle : calvalus-2.0b411\n" + "inputPath : /calvalus/eodata/MER_RR__1P/r03/${yyyy}/${MM}/${dd}/.*.N1\n" + - "minDateSource : 2016-01-01T00:00:00Z\n" + + "minDateSource : 2016-01-01T01:00:00+01:00\n" + "quicklooks : true\n" + "autoStaging : true\n" + "calvalus.snap.bundle : snap-3.0.0\n" + "processorName : null\n" + - "maxDate : 2017-01-01T00:00:00Z\n" + + "maxDate : 2017-01-01T01:00:00+01:00\n" + "processorBundleName : null\n" + "processorParameters : null\n" + "productionName : dummyProductionName\n" + - "maxDateSource : 2017-01-01T00:00:00Z\n" + + "maxDateSource : 2017-01-01T01:00:00+01:00\n" + "calvalus.system.snap.dataio.bigtiff.support.pushprocessing : false\n")); } @@ -324,7 +324,7 @@ public void canIgnoreDefaultBeamAndCalvalusBundleVersionWhenNotProvided() throws assertThat(calvalusDataInputs.toString(), equalTo("calvalus.wps.remote.ref : 1738ad7b-534e-4aca-9861-b26fb9c0f983\n" + - "minDate : 2016-01-01T00:00:00Z\n" + + "minDate : 2016-01-01T01:00:00+01:00\n" + "calvalus.output.compression : none\n" + "processorBundleLocation : null\n" + "calvalus.wps.remote.user : dummyRemoteUser\n" + @@ -332,14 +332,14 @@ public void canIgnoreDefaultBeamAndCalvalusBundleVersionWhenNotProvided() throws "inputDataSetName : MERIS RR r03 L1b 2002-2012\n" + "processorBundleVersion : null\n" + "inputPath : /calvalus/eodata/MER_RR__1P/r03/${yyyy}/${MM}/${dd}/.*.N1\n" + - "minDateSource : 2016-01-01T00:00:00Z\n" + + "minDateSource : 2016-01-01T01:00:00+01:00\n" + "quicklooks : true\n" + "autoStaging : true\n" + "processorName : null\n" + - "maxDate : 2017-01-01T00:00:00Z\n" + + "maxDate : 2017-01-01T01:00:00+01:00\n" + "processorBundleName : null\n" + "processorParameters : null\n" + - "maxDateSource : 2017-01-01T00:00:00Z\n" + + "maxDateSource : 2017-01-01T01:00:00+01:00\n" + "calvalus.system.snap.dataio.bigtiff.support.pushprocessing : false\n")); } @@ -492,4 +492,4 @@ private Map getMinimalRawMap() { return mockInputMapRaw; } -} +} \ No newline at end of file diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java index 5d5dab367..cb6d2fd75 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/utils/ProductMetadataTest.java @@ -71,14 +71,14 @@ public void canCreateProductMetadata() throws Exception { .build(); Map contextMap = productMetadata.getContextMap(); - assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T00:00:00.000Z")); + assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T01:00:00.000+01:00")); assertThat(contextMap.get("productOutputDir"), equalTo("user/20160317_10000000")); assertThat(contextMap.get("productionName"), equalTo("TEP_Subset_test")); assertThat(contextMap.get("processName"), equalTo("Subset")); assertThat(contextMap.get("inputDatasetName"), equalTo("Urban Footprint Global (Urban TEP)")); assertThat(contextMap.get("regionWkt"), equalTo("-10 100 0 100 0 110 -10 110 -10 100")); - assertThat(contextMap.get("startDate"), equalTo("2000-01-01T00:00:00.000Z")); - assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T00:00:00.000Z")); + assertThat(contextMap.get("startDate"), equalTo("2000-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T01:00:00.000+01:00")); assertThat(contextMap.get("collectionUrl"), equalTo("http://www.brockmann-consult.de:80/bc-wps/staging/user/20160317_10000000")); assertThat(contextMap.get("processorVersion"), equalTo("1.0")); assertThat(contextMap.get("productionType"), equalTo("2")); @@ -121,14 +121,14 @@ public void canEncodeProductionNameWithSpecialChars() throws Exception { .build(); Map contextMap = productMetadata.getContextMap(); - assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T00:00:00.000Z")); + assertThat(contextMap.get("jobFinishTime"), equalTo("2016-01-01T01:00:00.000+01:00")); assertThat(contextMap.get("productOutputDir"), equalTo("user/20160317_10000000")); assertThat(contextMap.get("productionName"), equalTo("http%3A%2F%2Fcatalog.terradue.com%2Furban-bc%2Fsearch%3Fformat%3Djson%26uid%3Dtep_inverted_timescan_Dakar%2Ftep_user%2F20170427185240_L3_130ec6a9dc24be")); assertThat(contextMap.get("processName"), equalTo("Subset")); assertThat(contextMap.get("inputDatasetName"), equalTo("Urban Footprint Global (Urban TEP)")); assertThat(contextMap.get("regionWkt"), equalTo("-10 100 0 100 0 110 -10 110 -10 100")); - assertThat(contextMap.get("startDate"), equalTo("2000-01-01T00:00:00.000Z")); - assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T00:00:00.000Z")); + assertThat(contextMap.get("startDate"), equalTo("2000-01-01T01:00:00.000+01:00")); + assertThat(contextMap.get("stopDate"), equalTo("2020-01-01T01:00:00.000+01:00")); assertThat(contextMap.get("collectionUrl"), equalTo("http://www.brockmann-consult.de:80/bc-wps/staging/user/20160317_10000000")); assertThat(contextMap.get("processorVersion"), equalTo("1.0")); assertThat(contextMap.get("productionType"), equalTo("2")); @@ -177,4 +177,4 @@ private void configureMockServerContext() { when(mockServerContext.getPort()).thenReturn(80); when(mockServerContext.getRequestUrl()).thenReturn("http://www.brockmann-consult.de/bc-wps/wps/calvalus?Service=WPS&Request=GetStatus&JobId=20160429140605_L2Plus_16bd8f26b258fc"); } -} +} \ No newline at end of file diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java index 2f860a19c..1ef2819a5 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusExecuteOperationTest.java @@ -137,7 +137,7 @@ public void canExecuteSync() throws Exception { assertThat(executeResponse.getStatus().getProcessSucceeded(), equalTo("The request has been processed successfully.")); assertThat(executeResponse.getProcess().getIdentifier().getValue(), equalTo("process1")); - assertThat(executeResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); + assertThat(executeResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); assertThat(executeResponse.getProcessOutputs().getOutput().size(), equalTo(2)); assertThat(executeResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), equalTo("resultUrl1")); assertThat(executeResponse.getProcessOutputs().getOutput().get(1).getReference().getHref(), equalTo("resultUrl2")); @@ -229,4 +229,4 @@ private ProductionRequest configureProcessingMock() throws Exception { return mockProductionRequest; } -} +} \ No newline at end of file diff --git a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java index 903764596..bb7607f9f 100644 --- a/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java +++ b/calvalus-wps-impl/src/test/java/com/bc/calvalus/wps/wpsoperations/CalvalusGetStatusOperationTest.java @@ -220,7 +220,7 @@ public void canGetSuccessfulStatus() throws Exception { assertThat(getStatusResponse.getProcess().getIdentifier().getValue(), equalTo("mockBundle~1.0~mockProcessor")); assertThat(getStatusResponse.getProcess().getProcessVersion(), equalTo("1.0")); - assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); + assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getIdentifier().getValue(), equalTo("production_result")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), @@ -250,7 +250,7 @@ public void canGetSuccessfulStatusLocalProcess() throws Exception { assertThat(getStatusResponse.getProcess().getIdentifier().getValue(), equalTo("local~0.0.1~Subset")); assertThat(getStatusResponse.getProcess().getProcessVersion(), equalTo("0.0.1")); - assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T00:00:00.000Z")); + assertThat(getStatusResponse.getStatus().getCreationTime().toString(), equalTo("2016-01-01T01:00:00.000+01:00")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getIdentifier().getValue(), equalTo("production_result")); assertThat(getStatusResponse.getProcessOutputs().getOutput().get(0).getReference().getHref(), @@ -343,4 +343,4 @@ private LocalProductionStatus getDoneAndSuccessfulLocalProcessStatus(List2.15-SNAPSHOT 1.2 7.0.0-SNAPSHOT - [6.0.0-SNAPSHOT,7.0.0-SNAPSHOT] - 1.7 + 7.0.0-SNAPSHOT + 1.7.3-SNAPSHOT 0.6.1 17.1 1.7.21 2.7.1 - [6.0.0,7.0.0-SNAPSHOT] + 7.0.0-SNAPSHOT UTF-8 From b44f1ad223e3c13c16cf87fd9690651349494248 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 26 Jun 2018 15:50:26 +0200 Subject: [PATCH 09/25] Update seadas-reader version in maven POM --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c30f71f12..7fad2ec80 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ 1.2 7.0.0-SNAPSHOT 7.0.0-SNAPSHOT - 1.7.3-SNAPSHOT + 1.8-SNAPSHOT 0.6.1 17.1 1.7.21 From f3450b0947dce6a261073b8c73b5fc23bb1ec5ea Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 17 Jul 2018 00:27:09 +0200 Subject: [PATCH 10/25] Redesign backend for configuring color palettes New design is based on the backend code for managing spatial regions. The previous code design for color palettes based on product-sets code is removed. The default configuration file in the deployed Calvalus portal web application is: calvalus-portal/WEB-INF/classes/com/bc/calvalus/portal/server/color-palettes.properties User configuration files are also supported on the server where the Calvalus portal is deployed: [USER]/.calvalus/[USER]-color-palettes.properties Example entries in color-palettes.properties: #name = path wew_water_chl1 = hdfs://my_server:9000/calvalus/auxiliary/wew_water_chl.cpd wew_water_chl2 = http://my_server:8080/calvalus-portal/auxiliary/wew_water_chl.cpd --- .../inventory/ColorPaletteService.java | 40 ------ .../calvalus/inventory/ColorPaletteSet.java | 45 ------- .../inventory/ColorPaletteSetPersistable.java | 67 ---------- .../inventory/DefaultColorPaletteService.java | 115 ------------------ .../DefaultColorPaletteServiceTest.java | 35 ------ .../portal/client/CalvalusPortal.java | 18 +-- .../calvalus/portal/client/PortalContext.java | 4 +- .../client/QuicklookParametersForm.java | 20 +-- .../portal/server/BackendServiceImpl.java | 49 ++++---- .../server/ColorPalettePersistence.java | 111 +++++++++++++++++ .../portal/shared/BackendService.java | 19 ++- .../portal/shared/BackendServiceAsync.java | 6 +- ...orPaletteSet.java => DtoColorPalette.java} | 40 ++++-- .../calvalus/production/ServiceContainer.java | 8 -- .../hadoop/HadoopServiceContainerFactory.java | 6 +- .../local/LocalServiceContainerFactory.java | 5 +- .../production/TestColorPaletteService.java | 40 ------ .../calvalusfacade/CalvalusProduction.java | 7 -- 18 files changed, 208 insertions(+), 427 deletions(-) delete mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java delete mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java delete mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java delete mode 100644 calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java delete mode 100644 calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java rename calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/{DtoColorPaletteSet.java => DtoColorPalette.java} (64%) delete mode 100644 calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java deleted file mode 100644 index d474ed4ca..000000000 --- a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.inventory; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; - -/** - * The interface to the Calvalus color palette inventory. - * - * @author Declan - */ -public interface ColorPaletteService { - - /** - * Gets color palette sets. - * - * @param filter A filter expression (unused) - * - * @return The array color palette sets, which may be empty. - * - * @throws java.io.IOException If an I/O error occurs - */ - ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException; -} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java deleted file mode 100644 index bd9803db5..000000000 --- a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSet.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.inventory; - -/** - * Color palette set. - * - * @author Declan - */ -public class ColorPaletteSet { - - private final String name; - private final String path; - - public ColorPaletteSet(String name, String path) { - if (name == null) { - throw new NullPointerException("name"); - } - if (path == null) { - throw new NullPointerException("path"); - } - this.name = name; - this.path = path; - } - - public String getName() { - return name; - } - - public String getPath() { return path; } -} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java deleted file mode 100644 index 5b3d1e7e4..000000000 --- a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/ColorPaletteSetPersistable.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.inventory; - -/** - * this class is responsible for converting a {@link ProductSet} into a {@link String} and back. - * - * @author Declan - */ -public class ColorPaletteSetPersistable { - - public static final String FILENAME = "color-palette-sets.csv"; - - private ColorPaletteSetPersistable() { - } - - public static String convertToCSV(ColorPaletteSet colorPaletteSet) { - StringBuilder sb = new StringBuilder(); - sb.append(colorPaletteSet.getName()); - sb.append(';'); - sb.append(colorPaletteSet.getPath()); - sb.append(';'); - return sb.toString(); - } - - public static ColorPaletteSet convertFromCSV(String text) { - String trimmedText = text.trim(); - if (trimmedText.startsWith("#") || trimmedText.isEmpty()) { - return null; //comments are ignored - } - String[] splits = trimmedText.split(";"); - - String name; - String path; - - if (splits.length >= 2) { - name = nullAware(splits[0]); - path = nullAware(splits[1]); - } else { - // less than 2 fields currently not supported - return null; - } - return new ColorPaletteSet(name, path); - } - - static String nullAware(String text) { - if (text == null || "null".equals(text)) { - return null; - } else { - return text; - } - } -} diff --git a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java b/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java deleted file mode 100644 index ea82a6e82..000000000 --- a/calvalus-inventory/src/main/java/com/bc/calvalus/inventory/DefaultColorPaletteService.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.inventory; - -import com.bc.calvalus.commons.CalvalusLogger; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - - -/** - * Default implementation of the {@link ColorPaletteService}. - * - * @author Declan - */ -public class DefaultColorPaletteService implements ColorPaletteService { - - private static final Logger LOG = CalvalusLogger.getLogger(); - private static final String USER_FILTER = "user="; - private final AbstractFileSystemService fileSystemService; - private final boolean withExternalAccessControl; - private String colorPaletteRootDir = "aux"; - - public DefaultColorPaletteService(AbstractFileSystemService fileSystemService, String colorPaletteRootDir) { - this.fileSystemService = fileSystemService; - this.colorPaletteRootDir = colorPaletteRootDir; - withExternalAccessControl = Boolean.getBoolean("calvalus.accesscontrol.external"); - } - - @Override - public ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException { - UserGroupInformation remoteUser = UserGroupInformation.createRemoteUser(username); - try { - return remoteUser.doAs((PrivilegedExceptionAction) () -> { - FileSystem fileSystem = fileSystemService.getFileSystem(username); - LOG.fine("DefaultColorPaletteService user " + username + " fs " + fileSystem + " jcm " + fileSystemService.getJobClientsMap()); - return loadPredefined(fileSystem); - }); - } catch (InterruptedException e) { - throw new IOException("failed to retrieve color palette sets for user " + username, e); - } - } - - private ColorPaletteSet[] loadPredefined(FileSystem fileSystem) throws IOException { - Path databasePath = fileSystemService.makeQualified(fileSystem, colorPaletteRootDir + "/" + ColorPaletteSetPersistable.FILENAME ); - if (fileSystem.exists(databasePath)) { - final ColorPaletteSet[] colorPaletteSets = readColorPaletteSets(fileSystem, new Path[]{databasePath}); - return colorPaletteSets; - } - return new ColorPaletteSet[0]; - } - - private ColorPaletteSet[] readColorPaletteSets(FileSystem fileSystem, Path[] paths) throws IOException { - if (paths == null || paths.length == 0) { - return new ColorPaletteSet[0]; - } else { - List colorPaletteSetList = new ArrayList(); - for (Path path : paths) { - try { - colorPaletteSetList.addAll(readColorPaletteSetFile(fileSystem, path)); - } catch (AccessControlException ignore) {} - } - return colorPaletteSetList.toArray(new ColorPaletteSet[0]); - } - } - - private List readColorPaletteSetFile(FileSystem fileSystem, Path path) throws IOException { - return readColorPaletteSetFromCsv(fileSystem.open(path)); - } - - static List readColorPaletteSetFromCsv(InputStream is) throws IOException { - InputStreamReader reader = new InputStreamReader(is); - try (BufferedReader bufferedReader = new BufferedReader(reader)) { - List colorPaletteSets = new ArrayList(); - - String line = bufferedReader.readLine(); - while (line != null) { - ColorPaletteSet colorPaletteSet= ColorPaletteSetPersistable.convertFromCSV(line); - if (colorPaletteSet != null) { - colorPaletteSets.add(colorPaletteSet); - } - line = bufferedReader.readLine(); - } - return colorPaletteSets; - } - } -} diff --git a/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java b/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java deleted file mode 100644 index bbf0b614f..000000000 --- a/calvalus-inventory/src/test/java/com/bc/calvalus/inventory/DefaultColorPaletteServiceTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.inventory; - -import org.junit.Test; - -/** - * Test implementation of DefaultColorPaletteService. - * - * @author Declan - */ -public class DefaultColorPaletteServiceTest { - - @Test - public void testCreatePaletteSet() throws Exception { - ColorPaletteSet[] test = new ColorPaletteSet[]{ - new ColorPaletteSet("name0", "path0"), - new ColorPaletteSet("name1", "path1"), - }; - } -} diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java index f4c0ae114..626d57e93 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java @@ -11,7 +11,7 @@ import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; -import com.bc.calvalus.portal.shared.DtoColorPaletteSet; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -75,7 +75,7 @@ public class CalvalusPortal implements EntryPoint, PortalContext { // Data provided by various external services private ListDataProvider regions; private DtoProductSet[] productSets; - private DtoColorPaletteSet[] colorPaletteSets; + private DtoColorPalette[] colorPalettes; private DtoProcessorDescriptor[] systemProcessors; private DtoProcessorDescriptor[] userProcessors; private DtoProcessorDescriptor[] allUserProcessors; @@ -129,8 +129,8 @@ public void onModuleLoad() { Runnable runnable = new Runnable() { public void run() { backendService.loadRegions(NO_FILTER, new InitRegionsCallback()); + backendService.loadColorPalettes(NO_FILTER, new InitColorPaletteSetsCallback()); backendService.getProductSets(NO_FILTER, new InitProductSetsCallback()); - backendService.getColorPaletteSets(NO_FILTER, new InitColorPaletteSetsCallback()); final BundleFilter systemFilter = new BundleFilter(); systemFilter.withProvider(BundleFilter.PROVIDER_SYSTEM); @@ -185,8 +185,8 @@ public DtoProductSet[] getProductSets() { } @Override - public DtoColorPaletteSet[] getColorPaletteSets() { - return colorPaletteSets; + public DtoColorPalette[] getColorPalettes() { + return colorPalettes; } @Override @@ -484,11 +484,11 @@ public void onFailure(Throwable caught) { } } - private class InitColorPaletteSetsCallback implements AsyncCallback { + private class InitColorPaletteSetsCallback implements AsyncCallback { @Override - public void onSuccess(DtoColorPaletteSet[] dtoColorPaletteSets) { - CalvalusPortal.this.colorPaletteSets = dtoColorPaletteSets; + public void onSuccess(DtoColorPalette[] dtoColorPalettes) { + CalvalusPortal.this.colorPalettes = dtoColorPalettes; maybeInitFrontend(); } @@ -496,7 +496,7 @@ public void onSuccess(DtoColorPaletteSet[] dtoColorPaletteSets) { public void onFailure(Throwable caught) { caught.printStackTrace(System.err); Dialog.error("Server-side Error", caught.getMessage()); - CalvalusPortal.this.colorPaletteSets = new DtoColorPaletteSet[0]; + CalvalusPortal.this.colorPalettes = new DtoColorPalette[0]; } } diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java index ccb931c91..17548259f 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/PortalContext.java @@ -5,7 +5,7 @@ import com.bc.calvalus.portal.shared.BackendServiceAsync; import com.bc.calvalus.portal.shared.ContextRetrievalServiceAsync; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; -import com.bc.calvalus.portal.shared.DtoColorPaletteSet; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; import com.bc.calvalus.portal.shared.DtoProductSet; import com.bc.calvalus.portal.shared.DtoProduction; @@ -28,7 +28,7 @@ public interface PortalContext { // make this return ListDataProvider DtoProductSet[] getProductSets(); - DtoColorPaletteSet[] getColorPaletteSets(); + DtoColorPalette[] getColorPalettes(); // make this return ListDataProvider DtoProcessorDescriptor[] getProcessors(String filter); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 125e42636..7476546bc 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -16,7 +16,7 @@ package com.bc.calvalus.portal.client; -import com.bc.calvalus.portal.shared.DtoColorPaletteSet; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.google.gwt.core.client.GWT; import com.google.gwt.event.dom.client.ChangeEvent; import com.google.gwt.event.dom.client.ChangeHandler; @@ -95,7 +95,7 @@ interface TheUiBinder extends UiBinder { @UiField CheckBox legendEnabled; - private DtoColorPaletteSet[] colorPaletteSet; + private DtoColorPalette[] availableColorPalettes; public QuicklookParametersForm(PortalContext portalContext) { this.portal = portalContext; @@ -137,7 +137,7 @@ public void onChange(ChangeEvent event) { }); setAvailableImageTypes(); - setColorPaletteSets(); + setColorPalettes(); quicklookNone.setValue(true); setQuicklookNone(); @@ -217,12 +217,12 @@ public void setAvailableImageTypes() { } } - private void setColorPaletteSets() { - this.colorPaletteSet = portal.getColorPaletteSets(); + private void setColorPalettes() { + this.availableColorPalettes = portal.getColorPalettes(); colorPalette.clear(); colorPalette.addItem(""); - for (DtoColorPaletteSet dtoColorPalette : this.colorPaletteSet ) { - colorPalette.addItem(dtoColorPalette.getPath()); + for (DtoColorPalette dtoColorPalette : this.availableColorPalettes ) { + colorPalette.addItem(dtoColorPalette.getCpdURL()); } colorPalette.setSelectedIndex(0); } @@ -430,11 +430,11 @@ public void setValues(Map parameters) { // color palette String cpdURLValue = getTagValue(dom, "cpdURL"); - if (cpdURLValue == null || this.colorPaletteSet == null) { + if (cpdURLValue == null || this.availableColorPalettes == null) { colorPalette.setSelectedIndex(0); } else { - for (int i=0; i < this.colorPaletteSet.length; i++ ) { - if( this.colorPaletteSet[i].getPath().equals(cpdURLValue) ) { + for (int i=0; i < this.availableColorPalettes.length; i++ ) { + if( this.availableColorPalettes[i].getPath().equals(cpdURLValue) ) { colorPalette.setSelectedIndex(i+1); break; } diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java index 8680df698..dd17487de 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/BackendServiceImpl.java @@ -23,14 +23,13 @@ import com.bc.calvalus.commons.WorkflowItem; import com.bc.calvalus.commons.shared.BundleFilter; import com.bc.calvalus.inventory.AbstractFileSystemService; -import com.bc.calvalus.inventory.ColorPaletteSet; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.ProductSet; import com.bc.calvalus.portal.shared.BackendService; import com.bc.calvalus.portal.shared.BackendServiceException; import com.bc.calvalus.portal.shared.DtoAggregatorDescriptor; import com.bc.calvalus.portal.shared.DtoCalvalusConfig; -import com.bc.calvalus.portal.shared.DtoColorPaletteSet; +import com.bc.calvalus.portal.shared.DtoColorPalette; import com.bc.calvalus.portal.shared.DtoMaskDescriptor; import com.bc.calvalus.portal.shared.DtoParameterDescriptor; import com.bc.calvalus.portal.shared.DtoProcessState; @@ -250,36 +249,41 @@ public void storeRegions(DtoRegion[] regions) throws BackendServiceException { } @Override - public DtoProductSet[] getProductSets(String filter) throws BackendServiceException { - if (filter.contains("dummy")) { - filter = filter.replace("dummy", getUserName()); + public DtoColorPalette[] loadColorPalettes(String filter) throws BackendServiceException { + ColorPalettePersistence colorPalettePersistence = new ColorPalettePersistence(getUserName(), ProductionServiceConfig.getUserAppDataDir()); + try { + DtoColorPalette[] dtoColorPalettes = colorPalettePersistence.loadColorPalettes(); + LOG.fine("loadColorPalettes returns " + dtoColorPalettes.length); + return dtoColorPalettes; + } catch (IOException e) { + log(e.getMessage(), e); + throw new BackendServiceException("Failed to load color palettes: " + e.getMessage(), e); } + } + + @Override + public void storeColorPalettes(DtoColorPalette[] colorPalettes) throws BackendServiceException { + ColorPalettePersistence colorPalettePersistence = new ColorPalettePersistence(getUserName(), ProductionServiceConfig.getUserAppDataDir()); try { - ProductSet[] productSets = serviceContainer.getInventoryService().getProductSets(getUserName(), filter); - DtoProductSet[] dtoProductSets = new DtoProductSet[productSets.length]; - for (int i = 0; i < productSets.length; i++) { - dtoProductSets[i] = convert(productSets[i]); - } - LOG.fine("getProductSets returns " + dtoProductSets.length); - return dtoProductSets; + colorPalettePersistence.storeColorPalettes(colorPalettes); } catch (IOException e) { - throw convert(e); + throw new BackendServiceException("Failed to store color palettes: " + e.getMessage(), e); } } @Override - public DtoColorPaletteSet[] getColorPaletteSets(String filter) throws BackendServiceException { + public DtoProductSet[] getProductSets(String filter) throws BackendServiceException { if (filter.contains("dummy")) { filter = filter.replace("dummy", getUserName()); } try { - ColorPaletteSet[] colorPaletteSets = serviceContainer.getColorPaletteService().getColorPaletteSets(getUserName(), filter); - DtoColorPaletteSet[] dtoColorPaletteSets = new DtoColorPaletteSet[colorPaletteSets.length]; - for (int i = 0; i < dtoColorPaletteSets.length; i++) { - dtoColorPaletteSets[i] = convert(colorPaletteSets[i]); + ProductSet[] productSets = serviceContainer.getInventoryService().getProductSets(getUserName(), filter); + DtoProductSet[] dtoProductSets = new DtoProductSet[productSets.length]; + for (int i = 0; i < productSets.length; i++) { + dtoProductSets[i] = convert(productSets[i]); } - LOG.fine("dtoColorPaletteSets returns " + dtoColorPaletteSets.length); - return dtoColorPaletteSets; + LOG.fine("getProductSets returns " + dtoProductSets.length); + return dtoProductSets; } catch (IOException e) { throw convert(e); } @@ -730,11 +734,6 @@ private DtoProductSet convert(ProductSet productSet) { productSet.getGeoInventory()); } - private DtoColorPaletteSet convert(ColorPaletteSet colorPaletteSet) { - return new DtoColorPaletteSet(colorPaletteSet.getName(), - colorPaletteSet.getPath()); - } - private DtoProcessorDescriptor convert(String bundleName, String bundleVersion, String bundlePath, String owner, ProcessorDescriptor processorDescriptor) { return new DtoProcessorDescriptor(processorDescriptor.getExecutableName(), diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java new file mode 100644 index 000000000..20ec4c795 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/server/ColorPalettePersistence.java @@ -0,0 +1,111 @@ +package com.bc.calvalus.portal.server; + +import com.bc.calvalus.portal.shared.DtoColorPalette; + +import java.io.*; +import java.util.*; + +/** + * A location that stores color palettes. + * + * @author Declan + */ +public class ColorPalettePersistence { + + private final String userName; + private final File userAppDataDir; + + public ColorPalettePersistence(String userName, File userAppDataDir) { + this.userName = userName; + this.userAppDataDir = userAppDataDir; + } + + public DtoColorPalette[] loadColorPalettes() throws IOException { + Properties colorPaletteProperties = loadDefaultColorPalettes(); + Properties userColorPaletteProperties = loadUserColorPalettes(); + colorPaletteProperties.putAll(userColorPaletteProperties); + + ArrayList colorPalettes = new ArrayList(); + Set colorPaletteNames = colorPaletteProperties.stringPropertyNames(); + for (String colorPaletteFullName : colorPaletteNames) { + String[] split = colorPaletteFullName.split("\\."); + String colorPaletteName = split[split.length - 1]; + String[] colorPalettePath = Arrays.copyOf(split, split.length - 1); + String cpdURL = colorPaletteProperties.getProperty(colorPaletteFullName); + DtoColorPalette colorPalette = new DtoColorPalette(colorPaletteName, colorPalettePath, cpdURL); + colorPalettes.add(colorPalette); + } + + Collections.sort(colorPalettes, new Comparator() { + @Override + public int compare(DtoColorPalette o1, DtoColorPalette o2) { + return o1.getQualifiedName().compareToIgnoreCase(o2.getQualifiedName()); + } + }); + return colorPalettes.toArray(new DtoColorPalette[colorPalettes.size()]); + } + + public void storeColorPalettes(DtoColorPalette[] colorPalettes) throws IOException { + Properties userColorPalettes = getUserColorPalettes(colorPalettes); + File file = getColorPaletteFile(getUserName()); + file.getParentFile().mkdirs(); + FileWriter fileWriter = new FileWriter(file); + try { + userColorPalettes.store(fileWriter, "Calvalus color palettes for user " + getUserName()); + } finally { + fileWriter.close(); + } + } + + private File getColorPaletteFile(String user) { + return new File(userAppDataDir, user + "-color-palettes.properties"); + } + + private String getUserName() { + return userName; + } + + private Properties getUserColorPalettes(DtoColorPalette[] colorPalettes) { + Properties userColorPalettes = new Properties(); + for (DtoColorPalette colorPalette : colorPalettes) { + if (colorPalette.isUserColorPalette()) { + String fullName = colorPalette.getQualifiedName(); + System.out.println("storing color palette " + fullName + " = " + colorPalette.getCpdURL()); + userColorPalettes.put(fullName, colorPalette.getCpdURL()); + } + } + return userColorPalettes; + } + + private Properties loadDefaultColorPalettes() throws IOException { + InputStream stream = getClass().getResourceAsStream("color-palettes.properties"); + Properties systemColorPalettes = loadColorPalettes(new BufferedReader(new InputStreamReader(stream))); + + File additionalSystemColorPaletteFile = getColorPaletteFile("SYSTEM"); + if (additionalSystemColorPaletteFile.exists()) { + Properties additionalSystemColorPalettes = loadColorPalettes(new FileReader(additionalSystemColorPaletteFile)); + systemColorPalettes.putAll(additionalSystemColorPalettes); + } + return systemColorPalettes; + } + + private Properties loadUserColorPalettes() throws IOException { + File userColorPaletteFile = getColorPaletteFile(getUserName()); + if (userColorPaletteFile.exists()) { + return loadColorPalettes(new FileReader(userColorPaletteFile)); + } else { + return new Properties(); + } + } + + private Properties loadColorPalettes(Reader stream) throws IOException { + Properties colorPalettes = new Properties(); + try { + colorPalettes.load(stream); + } finally { + stream.close(); + } + return colorPalettes; + } + +} diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java index 8c30a5eb7..f1525f35e 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendService.java @@ -33,18 +33,27 @@ public interface BackendService extends RemoteService { void storeRegions(DtoRegion[] regions) throws BackendServiceException; /** - * Gets all known product sets. + * Gets all known color palettes. * * @param filter A filter expression (not yet used). * - * @return The product sets. + * @return The array of color palettes. * * @throws BackendServiceException If a server error occurred. */ - DtoProductSet[] getProductSets(String filter) throws BackendServiceException; + DtoColorPalette[] loadColorPalettes(String filter) throws BackendServiceException; /** - * Gets all known color palette sets. + * Persists the provided color palettes. + * + * @param colorPalettes The color palettes to persist. + * + * @throws BackendServiceException If a server error occurred. + */ + void storeColorPalettes(DtoColorPalette[] colorPalettes) throws BackendServiceException; + + /** + * Gets all known product sets. * * @param filter A filter expression (not yet used). * @@ -52,7 +61,7 @@ public interface BackendService extends RemoteService { * * @throws BackendServiceException If a server error occurred. */ - DtoColorPaletteSet[] getColorPaletteSets(String filter) throws BackendServiceException; + DtoProductSet[] getProductSets(String filter) throws BackendServiceException; /** * Gets all known processors. diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java index a7fd283c5..1bad84fcd 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/BackendServiceAsync.java @@ -13,6 +13,10 @@ public interface BackendServiceAsync { void storeRegions(DtoRegion[] regions, AsyncCallback callback); + void loadColorPalettes(String filter, AsyncCallback callback); + + void storeColorPalettes(DtoColorPalette[] colorPalettes, AsyncCallback callback); + void listUserFiles(String dir, AsyncCallback callback); void removeUserFile(String path, AsyncCallback callback); @@ -23,8 +27,6 @@ public interface BackendServiceAsync { void getProductSets(String filter, AsyncCallback callback); - void getColorPaletteSets(String filter, AsyncCallback callback); - void getProcessors(String filter, AsyncCallback callback); void getAggregators(String filter, AsyncCallback callback); diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java similarity index 64% rename from calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java rename to calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java index 3399293d5..864abfcb4 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPaletteSet.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/shared/DtoColorPalette.java @@ -19,40 +19,61 @@ import com.google.gwt.user.client.rpc.IsSerializable; /** - * GWT-serializable version of the {@link com.bc.calvalus.inventory.ColorPaletteSet} class. - * * @author Declan */ -public class DtoColorPaletteSet implements IsSerializable { +public class DtoColorPalette implements IsSerializable { private String name; - private String path; + private String[] path; + private String cpdURL; /** * No-arg constructor as required by {@link IsSerializable}. Don't use directly. */ - public DtoColorPaletteSet() { + public DtoColorPalette() { } - public DtoColorPaletteSet(String name, String path) { + public DtoColorPalette(String name, String[] path, String cpdURL) { if (name == null) { throw new NullPointerException("name"); } if (path == null) { throw new NullPointerException("path"); } + if (path == null) { + throw new NullPointerException("cpdURL"); + } this.name = name; this.path = path; + this.cpdURL = cpdURL; } public String getName() { return name; } - public String getPath() { + public String[] getPath() { return path; } + public String getCpdURL() { + return cpdURL; + } + + public boolean isUserColorPalette() { + return path != null && path.length > 0 && "user".equals(path[0]); + } + + public String getQualifiedName() { + StringBuilder sb = new StringBuilder(); + for (String pathElement : path) { + sb.append(pathElement); + sb.append("."); + } + sb.append(name); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -62,7 +83,7 @@ public boolean equals(Object o) { return false; } - DtoColorPaletteSet that = (DtoColorPaletteSet) o; + DtoColorPalette that = (DtoColorPalette) o; if (!name.equals(that.name)) { return false; @@ -70,6 +91,9 @@ public boolean equals(Object o) { if (!path.equals(that.path)) { return false; } + if (!path.equals(that.cpdURL)) { + return false; + } return true; } } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java b/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java index 9c2cbc6f2..92fe8f216 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/ServiceContainer.java @@ -16,7 +16,6 @@ package com.bc.calvalus.production; -import com.bc.calvalus.inventory.ColorPaletteService; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.InventoryService; import org.apache.hadoop.conf.Configuration; @@ -28,18 +27,15 @@ public class ServiceContainer { private final ProductionService productionService; private final FileSystemService fileSystemService; private final InventoryService inventoryService; - private final ColorPaletteService colorPaletteService; private final Configuration hadoopConfiguration; public ServiceContainer(ProductionService productionService, FileSystemService fileSystemService, InventoryService inventoryService, - ColorPaletteService colorPaletteService, Configuration hadoopConfiguration) { this.productionService = productionService; this.fileSystemService = fileSystemService; this.inventoryService = inventoryService; - this.colorPaletteService = colorPaletteService; this.hadoopConfiguration = hadoopConfiguration; } @@ -55,10 +51,6 @@ public InventoryService getInventoryService() { return inventoryService; } - public ColorPaletteService getColorPaletteService() { - return colorPaletteService; - } - public Configuration getHadoopConfiguration() { return hadoopConfiguration; } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java index 40abb814c..c66d77504 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/HadoopServiceContainerFactory.java @@ -1,8 +1,6 @@ package com.bc.calvalus.production.hadoop; import com.bc.calvalus.JobClientsMap; -import com.bc.calvalus.inventory.ColorPaletteService; -import com.bc.calvalus.inventory.DefaultColorPaletteService; import com.bc.calvalus.inventory.DefaultInventoryService; import com.bc.calvalus.inventory.FileSystemService; import com.bc.calvalus.inventory.InventoryService; @@ -45,7 +43,6 @@ public ServiceContainer create(Map serviceConfiguration, // Prevent Windows from using ';' as path separator System.setProperty("path.separator", ":"); String archiveRootDir = serviceConfiguration.getOrDefault("calvalus.portal.archiveRootDir", "eodata"); - String colorPaletteRootDir = serviceConfiguration.getOrDefault("calvalus.portal.colorPaletteRootDir", "auxiliary"); String softwareDir = serviceConfiguration.getOrDefault("calvalus.portal.softwareDir", HadoopProcessingService.CALVALUS_SOFTWARE_PATH); // disable cache, otherwise org.apache.hadoop.fs.FileSystem.Cache grows forever @@ -62,7 +59,6 @@ public ServiceContainer create(Map serviceConfiguration, JobClientsMap jobClientsMap = new JobClientsMap(jobConf); final HdfsFileSystemService hdfsFileSystemService = new HdfsFileSystemService(jobClientsMap); final InventoryService inventoryService = new DefaultInventoryService(hdfsFileSystemService, archiveRootDir); - final ColorPaletteService colorPaletteService = new DefaultColorPaletteService(hdfsFileSystemService, colorPaletteRootDir); final HadoopProcessingService processingService = new HadoopProcessingService(jobClientsMap, softwareDir); final ProductionStore productionStore; if ("memory".equals(serviceConfiguration.get("production.db.type"))) { @@ -85,7 +81,7 @@ public ServiceContainer create(Map serviceConfiguration, productionStore, productionTypes); stagingService.setProductionService((Observable) productionService); - return new ServiceContainer(productionService, hdfsFileSystemService, inventoryService, colorPaletteService, hadoopConfiguration); + return new ServiceContainer(productionService, hdfsFileSystemService, inventoryService, hadoopConfiguration); } catch (IOException e) { throw new ProductionException("Failed to create Hadoop JobClient." + e.getMessage(), e); } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java b/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java index f23777a1f..11b97942f 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/local/LocalServiceContainerFactory.java @@ -17,8 +17,6 @@ package com.bc.calvalus.production.local; import com.bc.calvalus.inventory.AbstractFileSystemService; -import com.bc.calvalus.inventory.ColorPaletteService; -import com.bc.calvalus.inventory.DefaultColorPaletteService; import com.bc.calvalus.inventory.DefaultInventoryService; import com.bc.calvalus.inventory.InventoryService; import com.bc.calvalus.processing.AggregatorDescriptor; @@ -50,7 +48,6 @@ public ServiceContainer create(Map serviceConfiguration, AbstractFileSystemService fileSystemService = new LocalFileSystemService(); InventoryService inventoryService = new DefaultInventoryService(fileSystemService, "eodata"); - ColorPaletteService colorPaletteService = new DefaultColorPaletteService(fileSystemService, "auxiliary"); ProcessorDescriptor case2r = new ProcessorDescriptor("pc1", "MERIS IOP Case2R", "1.5", "a=2\nb=5", new ProcessorDescriptor.Variable("chl_conc", "AVG", "0.5"), @@ -158,7 +155,7 @@ public ServiceContainer create(Map serviceConfiguration, "autoStaging", "false")); } - return new ServiceContainer(productionService, fileSystemService, inventoryService, colorPaletteService, new Configuration()); + return new ServiceContainer(productionService, fileSystemService, inventoryService, new Configuration()); } diff --git a/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java b/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java deleted file mode 100644 index f73502186..000000000 --- a/calvalus-production/src/test/java/com/bc/calvalus/production/TestColorPaletteService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2018 Brockmann Consult GmbH (info@brockmann-consult.de) - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 3 of the License, or (at your option) - * any later version. - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, see http://www.gnu.org/licenses/ - */ - -package com.bc.calvalus.production; - -import com.bc.calvalus.inventory.ColorPaletteService; -import com.bc.calvalus.inventory.ColorPaletteSet; -import org.junit.Ignore; - -import java.io.IOException; - -/** - * Test implementation of ColorPaletteService. - * - * @author Declan - */ -@Ignore -public class TestColorPaletteService implements ColorPaletteService { - - @Override - public ColorPaletteSet[] getColorPaletteSets(String username, String filter) throws IOException { - return new ColorPaletteSet[]{ - new ColorPaletteSet("name0", "path0"), - new ColorPaletteSet("name1", "path1"), - }; - } -} diff --git a/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java b/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java index 4bdb81f33..89574574d 100644 --- a/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java +++ b/calvalus-wps-impl/src/main/java/com/bc/calvalus/wps/calvalusfacade/CalvalusProduction.java @@ -4,7 +4,6 @@ import com.bc.calvalus.commons.ProcessState; import com.bc.calvalus.commons.ProcessStatus; import com.bc.calvalus.commons.WorkflowItem; -import com.bc.calvalus.inventory.ColorPaletteSet; import com.bc.calvalus.inventory.ProductSet; import com.bc.calvalus.production.Production; import com.bc.calvalus.production.ProductionException; @@ -150,12 +149,6 @@ private ProductSet[] getProductSets(String userName, ServiceContainer serviceCon return productSets.toArray(new ProductSet[productSets.size()]); } - private ColorPaletteSet[] getColorPaletteSets(String userName, ServiceContainer serviceContainer) throws ProductionException, IOException { - List colorPaletteSets = new ArrayList<>(); - colorPaletteSets.addAll(Arrays.asList(serviceContainer.getColorPaletteService().getColorPaletteSets(userName, ""))); - colorPaletteSets.addAll(Arrays.asList(serviceContainer.getColorPaletteService().getColorPaletteSets(userName, "user=" + userName))); - return colorPaletteSets.toArray(new ColorPaletteSet[colorPaletteSets.size()]); - } private void observeProduction(ProductionService productionService, Production production) throws InterruptedException { final Thread shutDownHook = createShutdownHook(production.getWorkflow()); From ad0981c508c717240fdc80e68802ec20e4fb865a Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 17 Jul 2018 15:06:34 +0200 Subject: [PATCH 11/25] Bug fix for resetting colour table drop-down list --- .../com/bc/calvalus/portal/client/QuicklookParametersForm.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 7476546bc..265dd340e 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -434,7 +434,7 @@ public void setValues(Map parameters) { colorPalette.setSelectedIndex(0); } else { for (int i=0; i < this.availableColorPalettes.length; i++ ) { - if( this.availableColorPalettes[i].getPath().equals(cpdURLValue) ) { + if( this.availableColorPalettes[i].getCpdURL().equals(cpdURLValue) ) { colorPalette.setSelectedIndex(i+1); break; } From f3b3d6c5d7f8420ada4330f28c905e43797d04d6 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 17 Jul 2018 15:48:04 +0200 Subject: [PATCH 12/25] New logic for enabling/disabling band name field --- .../client/QuicklookParametersForm.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 265dd340e..4ba413d89 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -126,13 +126,7 @@ public void onValueChange(ValueChangeEvent event) { bandNameListBox.addChangeHandler(new ChangeHandler() { @Override public void onChange(ChangeEvent event) { - int selectedIndex = bandNameListBox.getSelectedIndex(); - if (selectedIndex == 0) { - bandName.setEnabled(true); - } else if (selectedIndex > 0) { - bandName.setValue(bandNameListBox.getValue(selectedIndex)); - bandName.setEnabled(false); - } + bandNameListBoxChangeHandler(); } }); @@ -143,7 +137,18 @@ public void onChange(ChangeEvent event) { setQuicklookNone(); } + private void bandNameListBoxChangeHandler() { + int selectedIndex = bandNameListBox.getSelectedIndex(); + if (selectedIndex == 0) { + bandName.setEnabled(true); + } else if (selectedIndex > 0) { + bandName.setValue(bandNameListBox.getValue(selectedIndex)); + bandName.setEnabled(false); + } + } + private void setQuicklookNone() { + bandNameListBox.setEnabled(false); bandName.setEnabled(false); colorPalette.setEnabled(false); @@ -164,7 +169,8 @@ private void setQuicklookNone() { } private void setQuicklookSingleBandEnabled() { - bandName.setEnabled(true); + bandNameListBox.setEnabled(true); + bandNameListBoxChangeHandler(); colorPalette.setEnabled(true); rgbaExpressionsRedBand.setEnabled(false); @@ -184,6 +190,7 @@ private void setQuicklookSingleBandEnabled() { } private void setQuicklookMultiBandEnabled() { + bandNameListBox.setEnabled(false); bandName.setEnabled(false); colorPalette.setEnabled(false); From f7a8e918dae236eec7d4513b1ab866c8c9f866bc Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 17 Jul 2018 17:16:17 +0200 Subject: [PATCH 13/25] Use text names for colour palette drop-down list instead of displaying full URLs --- .../com/bc/calvalus/portal/client/QuicklookParametersForm.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 4ba413d89..3cbe61b09 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -229,7 +229,7 @@ private void setColorPalettes() { colorPalette.clear(); colorPalette.addItem(""); for (DtoColorPalette dtoColorPalette : this.availableColorPalettes ) { - colorPalette.addItem(dtoColorPalette.getCpdURL()); + colorPalette.addItem(dtoColorPalette.getQualifiedName(), dtoColorPalette.getCpdURL()); } colorPalette.setSelectedIndex(0); } From 8f7381e9db7ff60305eb64c0b75bd58e05119e4c Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Tue, 17 Jul 2018 17:57:23 +0200 Subject: [PATCH 14/25] Change the spelling of color to colour to keep it consistent in the user interface --- .../bc/calvalus/portal/client/QuicklookParametersForm.ui.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index 07ac1e224..bf7c6fa66 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -153,7 +153,7 @@     - Background Color: + Background Colour:   From d772358132c9d95b783e175b5213b7da0c3c1e0b Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Thu, 23 Aug 2018 15:40:38 +0200 Subject: [PATCH 15/25] Add production type for creating quicklook images For portal access control and visual configuration, please set: calvalus.portal.qlView in webapps/calvalus-portal/config/calvalus.properties --- .../portal/client/CalvalusPortal.java | 3 + .../portal/client/OrderL2ProductionView.java | 2 +- .../portal/client/OrderQLProductionView.java | 200 ++++++++++++++++++ .../client/QuicklookParametersForm.java | 8 +- .../production/hadoop/QLProductionType.java | 12 +- 5 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java index 626d57e93..5509730da 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/CalvalusPortal.java @@ -61,6 +61,7 @@ public class CalvalusPortal implements EntryPoint, PortalContext { "vicariousCalibrationView", "matchupComparisonView", "l2ToL3ComparisonView", + "qlView", "regionsView", "requestsView", "bundlesView", @@ -287,6 +288,8 @@ private PortalView createViewOf(String name) { return new OrderMACProductionView(this); case "l2ToL3ComparisonView": return new OrderL2toL3ProductionView(this); + case "qlView": + return new OrderQLProductionView(this); case "regionsView": return new ManageRegionsView(this); case "bundlesView": diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java index de201dc1a..970009328 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL2ProductionView.java @@ -228,7 +228,7 @@ private void handleProcessorChanged() { for (DtoProcessorVariable dtoProcessorVariable : dtoProcessorVariables) { processorVariables.add(dtoProcessorVariable.getName()); } - quicklookParametersForm.setProcessorVariables(processorVariables.toArray(new String[0])); + quicklookParametersForm.setBandNames(processorVariables.toArray(new String[0])); } } diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java new file mode 100644 index 000000000..af0f49b52 --- /dev/null +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderQLProductionView.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2013 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.portal.client; + +import com.bc.calvalus.portal.shared.DtoInputSelection; +import com.bc.calvalus.portal.shared.DtoProcessorDescriptor; +import com.bc.calvalus.portal.shared.DtoProcessorVariable; +import com.google.gwt.core.client.Scheduler; +import com.google.gwt.user.client.rpc.AsyncCallback; +import com.google.gwt.user.client.ui.Anchor; +import com.google.gwt.user.client.ui.VerticalPanel; +import com.google.gwt.user.client.ui.Widget; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Demo view that lets users submit a new L2 production. + * + * @author Norman + */ +public class OrderQLProductionView extends OrderProductionView { + + public static final String ID = OrderQLProductionView.class.getName(); + + private ProductSetSelectionForm productSetSelectionForm; + private ProductSetFilterForm productSetFilterForm; + private ProductsFromCatalogueForm productsFromCatalogueForm; + private OutputParametersForm outputParametersForm; + private QuicklookParametersForm quicklookParametersForm; + private Widget widget; + + public OrderQLProductionView(PortalContext portalContext) { + super(portalContext); + + productSetSelectionForm = new ProductSetSelectionForm(getPortal()); + productSetFilterForm = new ProductSetFilterForm(portalContext); + productSetFilterForm.setProductSet(productSetSelectionForm.getSelectedProductSet()); + + if (getPortal().withPortalFeature(INPUT_FILES_PANEL)) { + productsFromCatalogueForm = new ProductsFromCatalogueForm(getPortal()); + productsFromCatalogueForm.addInputSelectionHandler(new ProductsFromCatalogueForm.InputSelectionHandler() { + @Override + public AsyncCallback getInputSelectionChangedCallback() { + return new InputSelectionCallback(); + } + + @Override + public void onClearSelectionClick() { + productsFromCatalogueForm.removeSelections(); + } + }); + } + + outputParametersForm = new OutputParametersForm(portalContext); + outputParametersForm.showFormatSelectionPanel(false); + outputParametersForm.setAvailableOutputFormats("Image"); + + quicklookParametersForm = new QuicklookParametersForm(portalContext); + quicklookParametersForm.setBandNames(); + + VerticalPanel panel = new VerticalPanel(); + panel.setWidth("100%"); + panel.add(productSetSelectionForm); + panel.add(productSetFilterForm); + if (getPortal().withPortalFeature(INPUT_FILES_PANEL)){ + panel.add(productsFromCatalogueForm); + } + panel.add(outputParametersForm); + panel.add(quicklookParametersForm); + Anchor l2Help = new Anchor("Show Help"); + l2Help.getElement().getStyle().setProperty("textDecoration", "none"); + l2Help.addStyleName("anchor"); + panel.add(l2Help); + HelpSystem.addClickHandler(l2Help, "l2Processing"); + //panel.add(new HTML("
")); + panel.add(createOrderPanel()); + + this.widget = panel; + } + + @Override + public Widget asWidget() { + return widget; + } + + @Override + public String getViewId() { + return ID; + } + + @Override + public String getTitle() { + return "Quicklook generation"; + } + + @Override + protected String getProductionType() { + return "QL"; + } + + @Override + public void onShowing() { + // make sure #triggerResize is called after the new view is shown + Scheduler.get().scheduleFinally(() -> { + // See http://code.google.com/p/gwt-google-apis/issues/detail?id=127 + productSetFilterForm.getRegionMap().getMapWidget().triggerResize(); + }); + } + + @Override + protected boolean validateForm() { + try { + productSetSelectionForm.validateForm(); + productSetFilterForm.validateForm(); + if (productsFromCatalogueForm != null) { + productsFromCatalogueForm.validateForm(productSetSelectionForm.getSelectedProductSet().getName()); + } + outputParametersForm.validateForm(); + + + if (!getPortal().withPortalFeature("unlimitedJobSize")) { + try { + final int numDaysValue = Integer.parseInt(productSetFilterForm.numDays.getValue()); + if (numDaysValue > 365 + 366) { + throw new ValidationException(productSetFilterForm.numDays, "time range larger than allowed"); + } + } catch (NumberFormatException e) { + // ignore + } + } + return true; + } catch (ValidationException e) { + e.handle(); + return false; + } + } + + @Override + protected HashMap getProductionParameters() { + HashMap parameters = new HashMap<>(); + parameters.putAll(productSetSelectionForm.getValueMap()); + parameters.putAll(productSetFilterForm.getValueMap()); + if (productsFromCatalogueForm != null) { + parameters.putAll(productsFromCatalogueForm.getValueMap()); + } + parameters.putAll(outputParametersForm.getValueMap()); + parameters.putAll(quicklookParametersForm.getValueMap()); + return parameters; + } + + @Override + public boolean isRestoringRequestPossible() { + return true; + } + + @Override + public void setProductionParameters(Map parameters) { + productSetSelectionForm.setValues(parameters); + productSetFilterForm.setValues(parameters); + if (productsFromCatalogueForm != null) { + productsFromCatalogueForm.setValues(parameters); + } + outputParametersForm.setValues(parameters); + quicklookParametersForm.setValues(parameters); + } + + private class InputSelectionCallback implements AsyncCallback { + + @Override + public void onSuccess(DtoInputSelection inputSelection) { + Map inputSelectionMap = UIUtils.parseParametersFromContext(inputSelection); + productsFromCatalogueForm.setValues(inputSelectionMap); + productSetSelectionForm.setValues(inputSelectionMap); + productSetFilterForm.setValues(inputSelectionMap); + } + + @Override + public void onFailure(Throwable caught) { + Dialog.error("Error in retrieving input selection", caught.getMessage()); + } + } +} diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 3cbe61b09..9a9de137c 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -234,15 +234,15 @@ private void setColorPalettes() { colorPalette.setSelectedIndex(0); } - public void setProcessorVariables(String... processorVariables) { + public void setBandNames(String... bandNames) { bandNameListBox.clear(); bandNameListBox.addItem(""); bandNameListBox.setSelectedIndex(0); - if (processorVariables == null) + if (bandNames == null) return; - for (String processorVariable : processorVariables) { - bandNameListBox.addItem(processorVariable); + for (String bandName : bandNames) { + bandNameListBox.addItem(bandName); } } diff --git a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java index 1c1d84582..c04a52843 100644 --- a/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java +++ b/calvalus-production/src/main/java/com/bc/calvalus/production/hadoop/QLProductionType.java @@ -64,23 +64,17 @@ public Production createProduction(ProductionRequest productionRequest) throws P // todo - if autoStaging=true, create sequential workflow and add staging job String stagingDir = productionRequest.getStagingDirectory(productionId); - //boolean autoStaging = productionRequest.isAutoStaging(); //TODO - boolean autoStaging = false; + boolean autoStaging = productionRequest.isAutoStaging(); + String quicklookOutputDir = getOutputPath(productionRequest, productionId, ""); return new Production(productionId, productionName, - "", + quicklookOutputDir, stagingDir, autoStaging, productionRequest, workflowItem); } - // TODO, at the moment no staging implemented - @Override - protected Staging createUnsubmittedStaging(Production production) { - throw new UnsupportedOperationException("Staging currently not implemented for quick look generation."); - } - WorkflowItem createWorkflowItem(String productionId, String productionName, ProductionRequest productionRequest) throws ProductionException { From f3a77335459450901ab1c6a81e73626e19600d18 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Thu, 30 Aug 2018 18:33:22 +0200 Subject: [PATCH 16/25] Add default color-palettes.properties file These color palettes are SNAP standard cpd files, which are assumed available on Calvalus HDFS (folder /calvalus/auxiliary/cpd) --- .../portal/server/color-palettes.properties | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties new file mode 100644 index 000000000..69dde0080 --- /dev/null +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/server/color-palettes.properties @@ -0,0 +1,26 @@ +5_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/5_colors.cpd +7_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/7_colors.cpd +cc_chl = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_chl.cpd +cc_general = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_general.cpd +cc_iop_quality = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_iop_quality.cpd +cc_tsm = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_tsm.cpd +cc_yellowsubstance = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_yellowsubstance.cpd +cc_z90 = hdfs://master00:9000/calvalus/auxiliary/cpd/cc_z90.cpd +CHL_SeaWiFS = hdfs://master00:9000/calvalus/auxiliary/cpd/CHL_SeaWiFS.cpd +cubehelix_cycle = hdfs://master00:9000/calvalus/auxiliary/cpd/cubehelix_cycle.cpd +gradient_8_colors = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_8_colors.cpd +gradient_blue = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_blue.cpd +gradient_green = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_green.cpd +gradient_red = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_red.cpd +gradient_red_white_blue = hdfs://master00:9000/calvalus/auxiliary/cpd/gradient_red_white_blue.cpd +great_circle = hdfs://master00:9000/calvalus/auxiliary/cpd/great_circle.cpd +JET = hdfs://master00:9000/calvalus/auxiliary/cpd/JET.cpd +meris_algal = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_algal.cpd +meris_case2 = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_case2.cpd +meris_cloud = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_cloud.cpd +meris_pressure = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_pressure.cpd +meris_veg_index = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_veg_index.cpd +meris_wind_direction = hdfs://master00:9000/calvalus/auxiliary/cpd/meris_wind_direction.cpd +spectrum = hdfs://master00:9000/calvalus/auxiliary/cpd/spectrum.cpd +spectrum_large = hdfs://master00:9000/calvalus/auxiliary/cpd/spectrum_large.cpd +terrain = hdfs://master00:9000/calvalus/auxiliary/cpd/terrain.cpd From 493383f2568d74532fd2915ee2a1c7221153fec0 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Wed, 26 Sep 2018 16:23:03 +0200 Subject: [PATCH 17/25] Improve quicklook UI form's layout and handling --- .../client/QuicklookParametersForm.java | 231 ++++++++------ .../com/bc/calvalus/portal/client/style.css | 26 +- .../client/QuicklookParametersForm.ui.xml | 297 +++++++++++++----- 3 files changed, 376 insertions(+), 178 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 9a9de137c..f90674bf7 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -18,15 +18,20 @@ import com.bc.calvalus.portal.shared.DtoColorPalette; import com.google.gwt.core.client.GWT; +import com.google.gwt.dom.client.Element; +import com.google.gwt.dom.client.Style; import com.google.gwt.event.dom.client.ChangeEvent; import com.google.gwt.event.dom.client.ChangeHandler; import com.google.gwt.event.logical.shared.ValueChangeEvent; import com.google.gwt.event.logical.shared.ValueChangeHandler; import com.google.gwt.uibinder.client.UiBinder; import com.google.gwt.uibinder.client.UiField; +import com.google.gwt.user.client.DOM; import com.google.gwt.user.client.ui.CheckBox; import com.google.gwt.user.client.ui.Composite; +import com.google.gwt.user.client.ui.HTMLPanel; import com.google.gwt.user.client.ui.IntegerBox; +import com.google.gwt.user.client.ui.Label; import com.google.gwt.user.client.ui.ListBox; import com.google.gwt.user.client.ui.RadioButton; import com.google.gwt.user.client.ui.TextBox; @@ -51,8 +56,22 @@ interface TheUiBinder extends UiBinder { private static TheUiBinder uiBinder = GWT.create(TheUiBinder.class); + private static int instanceCounter = 0; + private int radioGroupId = 0; + + private static final String CUSTOM_BAND = ""; + + private static final String BANDNAME_LISTBOX_ROW = "bandNameListBoxRow"; + private final PortalContext portal; + @UiField + HTMLPanel singleBandPanel; + @UiField + HTMLPanel multiBandPanel; + @UiField + HTMLPanel moreOptionsPanel; + @UiField RadioButton quicklookNone; @UiField @@ -63,6 +82,8 @@ interface TheUiBinder extends UiBinder { @UiField ListBox bandNameListBox; @UiField + Label bandNameRowLabel; + @UiField TextBox bandName; @UiField ListBox colorPalette; @@ -95,31 +116,51 @@ interface TheUiBinder extends UiBinder { @UiField CheckBox legendEnabled; - private DtoColorPalette[] availableColorPalettes; + private DtoColorPalette[] availableColorPalettes = null; + private Boolean pageLoaded = false; + private String[] availableBandNames = null; public QuicklookParametersForm(PortalContext portalContext) { this.portal = portalContext; initWidget(uiBinder.createAndBindUi(this)); + instanceCounter++; + radioGroupId = instanceCounter; + + //give radio group a unique name for each instance + quicklookNone.setName("quicklookSelection" + radioGroupId); + quicklookSingleBand.setName("quicklookSelection" + radioGroupId); + quicklookMultiBand.setName("quicklookSelection" + radioGroupId); + quicklookNone.addValueChangeHandler(new ValueChangeHandler() { @Override public void onValueChange(ValueChangeEvent event) { - setQuicklookNone(); + if (event.getValue()) + quicklookNoneChangeHandler(); } }); quicklookSingleBand.addValueChangeHandler(new ValueChangeHandler() { @Override public void onValueChange(ValueChangeEvent event) { - setQuicklookSingleBandEnabled(); + if (event.getValue()) + quicklookSingleBandChangeHandler(); } }); quicklookMultiBand.addValueChangeHandler(new ValueChangeHandler() { @Override public void onValueChange(ValueChangeEvent event) { - setQuicklookMultiBandEnabled(); + if (event.getValue()) + quicklookMultiBandChangeHandler(); + } + }); + + bandName.addValueChangeHandler(new ValueChangeHandler() { + @Override + public void onValueChange(ValueChangeEvent event) { + bandNameChangeHandler(); } }); @@ -132,82 +173,83 @@ public void onChange(ChangeEvent event) { setAvailableImageTypes(); setColorPalettes(); + } + + @Override + protected void onLoad() { + //give HTML element a unique id for each instance + Element element = DOM.getElementById(BANDNAME_LISTBOX_ROW); + element.setId(BANDNAME_LISTBOX_ROW + radioGroupId); - quicklookNone.setValue(true); - setQuicklookNone(); + this.pageLoaded = true; + quicklookNone.setValue(true, true); + buildBandNameListBox(); + } + + private void bandNameChangeHandler() { + // band name field changed, reset list box index and call it's handler + int itemCount = bandNameListBox.getItemCount(); + String bandNameValue = bandName.getValue(); + bandNameListBox.setSelectedIndex(0); + for (int i = 0; i < itemCount; i++) { + String listBand = bandNameListBox.getValue(i); + if (listBand != null && listBand.equals(bandNameValue)) { + bandNameListBox.setSelectedIndex(i); + break; + } + } + bandNameListBoxChangeHandler(); } private void bandNameListBoxChangeHandler() { + if (!pageLoaded) + return; + int selectedIndex = bandNameListBox.getSelectedIndex(); - if (selectedIndex == 0) { - bandName.setEnabled(true); - } else if (selectedIndex > 0) { - bandName.setValue(bandNameListBox.getValue(selectedIndex)); - bandName.setEnabled(false); + Element element = DOM.getElementById(BANDNAME_LISTBOX_ROW + radioGroupId); + if (!quicklookSingleBand.getValue() || selectedIndex < 0) { + // single band not selected OR no values in listbox + // enable bandName textbox, make drop down list invisible + bandNameRowLabel.setText("Band name:"); + bandName.setEnabled(true); + if (element != null) + element.getStyle().setDisplay(Style.Display.NONE); + } else { + // single band is selected AND values in listbox + // make drop down list visible + if (element != null) + element.getStyle().setDisplay(Style.Display.TABLE_ROW); + bandNameRowLabel.setText(""); + if (selectedIndex == 0) { + // custom band name option selected + // enable bandName textbox + bandName.setEnabled(true); + } else if (selectedIndex > 0) { + // band name (sourced from processor) selected + // disable bandName textbox + bandName.setValue(bandNameListBox.getValue(selectedIndex)); + bandName.setEnabled(false); + } } } - private void setQuicklookNone() { - bandNameListBox.setEnabled(false); - bandName.setEnabled(false); - colorPalette.setEnabled(false); - - rgbaExpressionsRedBand.setEnabled(false); - rgbaExpressionsGreenBand.setEnabled(false); - rgbaExpressionsBlueBand.setEnabled(false); - rgbaExpressionsAlphaBand.setEnabled(false); - rgbaMinSamples.setEnabled(false); - rgbaMaxSamples.setEnabled(false); - - imageType.setEnabled(false); - overlayURL.setEnabled(false); - maskOverlays.setEnabled(false); - subSamplingX.setEnabled(false); - subSamplingY.setEnabled(false); - backgroundColor.setEnabled(false); - legendEnabled.setEnabled(false); + private void quicklookNoneChangeHandler() { + singleBandPanel.setVisible(false); + multiBandPanel.setVisible(false); + moreOptionsPanel.setVisible(false); } - private void setQuicklookSingleBandEnabled() { - bandNameListBox.setEnabled(true); - bandNameListBoxChangeHandler(); - colorPalette.setEnabled(true); - - rgbaExpressionsRedBand.setEnabled(false); - rgbaExpressionsGreenBand.setEnabled(false); - rgbaExpressionsBlueBand.setEnabled(false); - rgbaExpressionsAlphaBand.setEnabled(false); - rgbaMinSamples.setEnabled(false); - rgbaMaxSamples.setEnabled(false); - - imageType.setEnabled(true); - overlayURL.setEnabled(true); - maskOverlays.setEnabled(true); - subSamplingX.setEnabled(true); - subSamplingY.setEnabled(true); - backgroundColor.setEnabled(true); - legendEnabled.setEnabled(true); + private void quicklookSingleBandChangeHandler() { + singleBandPanel.setVisible(true); + multiBandPanel.setVisible(false); + moreOptionsPanel.setVisible(true); + bandNameChangeHandler(); } - private void setQuicklookMultiBandEnabled() { - bandNameListBox.setEnabled(false); - bandName.setEnabled(false); - colorPalette.setEnabled(false); - - rgbaExpressionsRedBand.setEnabled(true); - rgbaExpressionsGreenBand.setEnabled(true); - rgbaExpressionsBlueBand.setEnabled(true); - rgbaExpressionsAlphaBand.setEnabled(true); - rgbaMinSamples.setEnabled(true); - rgbaMaxSamples.setEnabled(true); - - imageType.setEnabled(true); - overlayURL.setEnabled(true); - maskOverlays.setEnabled(true); - subSamplingX.setEnabled(true); - subSamplingY.setEnabled(true); - backgroundColor.setEnabled(true); - legendEnabled.setEnabled(true); + private void quicklookMultiBandChangeHandler() { + singleBandPanel.setVisible(false); + multiBandPanel.setVisible(true); + moreOptionsPanel.setVisible(true); } public void setAvailableImageTypes() { @@ -228,22 +270,30 @@ private void setColorPalettes() { this.availableColorPalettes = portal.getColorPalettes(); colorPalette.clear(); colorPalette.addItem(""); - for (DtoColorPalette dtoColorPalette : this.availableColorPalettes ) { + for (DtoColorPalette dtoColorPalette : this.availableColorPalettes) { colorPalette.addItem(dtoColorPalette.getQualifiedName(), dtoColorPalette.getCpdURL()); } colorPalette.setSelectedIndex(0); } public void setBandNames(String... bandNames) { + this.availableBandNames = bandNames; + buildBandNameListBox(); + } + + private void buildBandNameListBox() { bandNameListBox.clear(); - bandNameListBox.addItem(""); - bandNameListBox.setSelectedIndex(0); - if (bandNames == null) + if (!pageLoaded) return; - for (String bandName : bandNames) { - bandNameListBox.addItem(bandName); + if (availableBandNames != null && availableBandNames.length > 0) { + bandNameListBox.addItem(CUSTOM_BAND); + bandNameListBox.setSelectedIndex(0); + for (String bandName : this.availableBandNames) { + bandNameListBox.addItem(bandName); + } } + bandNameChangeHandler(); } public Map getValueMap() { @@ -295,7 +345,7 @@ public Map getValueMap() { // legendEnabled Boolean legendEnabledValue = legendEnabled.getValue(); String legendEnabledXML = ""; - if (legendEnabled.getValue()) { + if (legendEnabledValue) { legendEnabledXML = indentXML + "true\n"; } @@ -398,7 +448,7 @@ public Map getValueMap() { public void setValues(Map parameters) { String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); if (quicklookParametersValue == null || quicklookParametersValue.isEmpty()) { - quicklookNone.setValue(true, true); + bandName.setValue(null); colorPalette.setSelectedIndex(0); @@ -416,23 +466,20 @@ public void setValues(Map parameters) { subSamplingX.setValue(null); maskOverlays.setValue(null); legendEnabled.setValue(false); + + // set no quicklook radio button in GUI + quicklookNone.setValue(true, true); } else { Document dom = XMLParser.parse(quicklookParametersValue); - // bandName + // bandName String bandNameValue = getTagValue(dom, "bandName"); if (bandNameValue != null) { - quicklookSingleBand.setValue(true, true); bandName.setValue(bandNameValue); - int itemCount = bandNameListBox.getItemCount(); - for (int i = 0; i < itemCount; i++) { - String listBand = bandNameListBox.getValue(i); - if (listBand != null && listBand.equals(bandNameValue)) { - bandNameListBox.setSelectedIndex(i); - bandName.setEnabled(false); - break; - } - } + + // As bandName has a value, assume this is a single band + // Therefore, set single band radio button in GUI + quicklookSingleBand.setValue(true, true); } // color palette @@ -440,9 +487,9 @@ public void setValues(Map parameters) { if (cpdURLValue == null || this.availableColorPalettes == null) { colorPalette.setSelectedIndex(0); } else { - for (int i=0; i < this.availableColorPalettes.length; i++ ) { - if( this.availableColorPalettes[i].getCpdURL().equals(cpdURLValue) ) { - colorPalette.setSelectedIndex(i+1); + for (int i = 0; i < this.availableColorPalettes.length; i++) { + if (this.availableColorPalettes[i].getCpdURL().equals(cpdURLValue)) { + colorPalette.setSelectedIndex(i + 1); break; } } @@ -457,6 +504,9 @@ public void setValues(Map parameters) { rgbaExpressionsGreenBand.setValue(numTokens > 1 ? tokens[1] : null); rgbaExpressionsBlueBand.setValue(numTokens > 2 ? tokens[2] : null); rgbaExpressionsAlphaBand.setValue(numTokens > 3 ? tokens[3] : null); + + // As RGBAExpressions has a value, assume this is a multi band, + // Therefore, set multi band radio button in GUI quicklookMultiBand.setValue(true, true); } else { rgbaExpressionsRedBand.setValue(null); @@ -537,3 +587,4 @@ private String getTagValue(Document dom, String tagName) { return null; } } + diff --git a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css index a70020512..e34199ea1 100644 --- a/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css +++ b/calvalus-portal/src/main/resources-default/WEB-INF/classes/com/bc/calvalus/portal/client/style.css @@ -122,18 +122,32 @@ * Quicklook Panel styling * ***************************/ .quicklookPanel { - margin: 50px 15px; + margin: 0px 15px 50px 15px; width: 62em; } +.quicklookRadioPanel { + padding: 5px 0px 5px 0px; +} + +.quicklookRadioButton { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + color: black; + padding-right: 10px; +} + .quicklookParametersPanel { display: flex; - flex-direction: column; } -.quicklookInputLabel { - flex: 0 0 130px; - margin: auto 0; +.quicklookInputLabelCell { + font-family: Calibri, Verdana, Arial, sans-serif; + font-size: 14px; + width: 185px; +} + +.quicklookInputCell { font-family: Calibri, Verdana, Arial, sans-serif; font-size: 14px; } @@ -246,4 +260,4 @@ font-size: 14px; font-weight: 700; color: #777799; -} \ No newline at end of file +} diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index bf7c6fa66..46a09bc12 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -10,166 +10,299 @@ Quicklook - - + + + + + + +
+ - + + + + - - + - - + - - - + + + + - - +
- +    
+ Band name: + +
- + + + +
   Band name: - + + + The name of a band which will be used for the image. +
       -   +    
+ Colour table: + +
   Colour table: -   + + + The colour table which will be applied to the chosen band. +
+ + + + - + + + + - - - - + - - - + - - - + - - + - - - + + + + + + + + - - +
- +    
+ Red band expression: + +
   Red band expression: -   + + Green band expression: + +
   Green band expression: -   + + Blue band expression: + +
   Blue band expression: -   + + Alpha band expression: + +
   Alpha band expression: -   + + + Add a list of 3 or 4 band names from which an RGB or RGBA colourised image will be + generated (A=alpha channel). +
   Minimum samples: - +    
+ Minimum samples: + + +
+ Maximum samples: + +
   Maximum samples: - + + + Add a list of 3 or 4 values that set minimal and maximal values for the displayed samples. +
+ + + + - + - - + - - + - - - + + + + - - + - - - + + + + - - + + + + + + + + - - + + + + + + + + + + + + - + + + + + + + + + + + -
      
- More quicklook options: + + Image type: + +
   Image type: - + + + The type of the generated image. +
   Overlay URL: -   +    
+ Background Colour: + +
   Mask overlays: -   + + + The background colour of the generated image. +
   Sub sampling (X): -   +    
+ Overlay URL: + +
   Sub sampling (Y): -   + + + A URL to a file containing a image that will be rendered above the image from the product. + +
   
+ Mask overlays: + +
   Background Colour: -   + + + Defines which product mask should be drawn on top of + the actual band or RGB image. The product must contain + the specified masks. + +
   
+ Sub sampling (X): + + +
+ Sub sampling (Y): + +
    + + + Sub sampling can be used to reduce the size of the resulting image. + The default size of the image is identical to the size of the product. + +
   
+ Legend +
    + + When set, an image legend will be drawn into the bottom-right corner of the image. + +
From 08f7928e94984b1bd2e3edda47839b44a98c968f Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Fri, 2 Nov 2018 00:09:44 +0100 Subject: [PATCH 18/25] Fixes for 'mvn clean install' --- calvalus-processing/pom.xml | 9 ++++++-- calvalus-snap/pom.xml | 41 ------------------------------------- pom.xml | 2 +- 3 files changed, 8 insertions(+), 44 deletions(-) diff --git a/calvalus-processing/pom.xml b/calvalus-processing/pom.xml index c3677beec..6abc78f8b 100644 --- a/calvalus-processing/pom.xml +++ b/calvalus-processing/pom.xml @@ -19,6 +19,12 @@ + + edu.ucar + netcdf + 4.3.23 + + org.slf4j slf4j-api @@ -56,12 +62,11 @@ org.esa.snap snap-gpf - org.xeustechnologies jtar diff --git a/calvalus-snap/pom.xml b/calvalus-snap/pom.xml index 277a8646d..e9471e97c 100644 --- a/calvalus-snap/pom.xml +++ b/calvalus-snap/pom.xml @@ -82,12 +82,6 @@ org.esa.snapsnap-watermask org.esa.snapsnap-sta - - ncsa.hdfjhdfobj - ncsa.hdfjhdf4obj - ncsa.hdfjhdf5obj - ncsa.hdflib-hdf - org.esa.s3tbxs3tbx-alos-reader org.esa.s3tbxs3tbx-atsr-reader org.esa.s3tbxs3tbx-avhrr-reader @@ -139,26 +133,6 @@ jetspeed-unpack-maven-plugin 2.2.2 - - libs - - - ncsa.hdf:lib-hdf:jar - - - lib/amd64 - libjhdf*.so - - - target/snap-bundle - - true - - package - - unpack - - libs2 @@ -181,26 +155,11 @@ - - ncsa.hdf - lib-hdf - ${ncsa.version} - org.esa.s2tbx lib-openjpeg ${snap.version} - - ncsa.hdf - jhdfobj - 2.7 - - - ncsa.hdf - jhdf5obj - 2.7 - diff --git a/pom.xml b/pom.xml index 6b48bcf46..c8a39cf59 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ 1.2 7.0.0-SNAPSHOT [6.0.0-SNAPSHOT,7.0.0-SNAPSHOT] - 1.7-SNAPSHOT + 1.8-SNAPSHOT 2.1.0-SNAPSHOT 0.6.1 17.1 From a0462d4575530be911b7d6eea77b7cc7e48ab930 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Wed, 28 Nov 2018 17:52:00 +0100 Subject: [PATCH 19/25] Add GeoTiff support to quicklook generation --- .../client/QuicklookParametersForm.java | 8 ++- .../processing/analysis/QLMapper.java | 72 +++++++++++++++++-- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index f90674bf7..4917d1f3a 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -253,7 +253,7 @@ private void quicklookMultiBandChangeHandler() { } public void setAvailableImageTypes() { - String[] imageNames = {"jpeg", "tiff", "png"}; + String[] imageNames = {"jpeg", "png", "tiff", "geotiff"}; int selectedIndex = imageType.getSelectedIndex(); imageType.clear(); for (String imageName : imageNames) { @@ -529,10 +529,12 @@ public void setValues(Map parameters) { imageType.setSelectedIndex(0); else if (imageTypeValue.equalsIgnoreCase("jpeg")) imageType.setSelectedIndex(0); - else if (imageTypeValue.equalsIgnoreCase("tiff")) - imageType.setSelectedIndex(1); else if (imageTypeValue.equalsIgnoreCase("png")) + imageType.setSelectedIndex(1); + else if (imageTypeValue.equalsIgnoreCase("tiff")) imageType.setSelectedIndex(2); + else if (imageTypeValue.equalsIgnoreCase("geotiff")) + imageType.setSelectedIndex(3); // overlayURL String overlayURLValue = getTagValue(dom, "overlayURL"); diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index 887db1d39..35d0b257e 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -29,8 +29,22 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.esa.snap.core.datamodel.GeoCoding; +import org.esa.snap.core.datamodel.GeoPos; +import org.esa.snap.core.datamodel.PixelPos; import org.esa.snap.core.datamodel.Product; import org.esa.snap.core.util.io.FileUtils; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.grid.GridCoverageFactory; +import org.geotools.coverage.grid.io.AbstractGridFormat; +//import org.geotools.coverage.grid.io.imageio.GeoToolsWriteParams; +import org.geotools.gce.geotiff.GeoTiffFormat; +import org.geotools.gce.geotiff.GeoTiffWriteParams; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.opengis.coverage.grid.GridCoverageWriter; +import org.opengis.parameter.GeneralParameterValue; +import org.opengis.parameter.ParameterValueGroup; import javax.imageio.ImageIO; import java.awt.image.RenderedImage; @@ -86,12 +100,54 @@ public static void createQuicklook(Product product, String imageFileName, Mapper // try { RenderedImage quicklookImage = new QuicklookGenerator(context, product, config).createImage(); if (quicklookImage != null) { - OutputStream outputStream = createOutputStream(context, imageFileName + "." + config.getImageType()); - OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); - try { - ImageIO.write(quicklookImage, config.getImageType(), pmOutputStream); - } finally { - outputStream.close(); + if( isGeoTiff(config)) { + OutputStream outputStream = createOutputStream(context, imageFileName + ".tiff"); + LOGGER.info("outputStream: " + outputStream.toString()); + OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); + + final int width = product.getSceneRasterWidth(); + final int height = product.getSceneRasterWidth(); + final GeoCoding geoCoding = product.getSceneGeoCoding(); + final PixelPos posA = new org.esa.snap.core.datamodel.PixelPos(0, 0); + final GeoPos geoPosA = geoCoding.getGeoPos(posA, null); + final PixelPos posB = new org.esa.snap.core.datamodel.PixelPos(width, height); + final GeoPos geoPosB = geoCoding.getGeoPos(posB, null); + + final GeoTiffFormat format = new GeoTiffFormat(); + final GeoTiffWriteParams wp = new GeoTiffWriteParams(); + //wp.setCompressionMode(GeoTiffWriteParams.MODE_EXPLICIT); + //wp.setCompressionType("LZW"); + //wp.setCompressionQuality(1.0F); + //wp.setTilingMode(GeoToolsWriteParams.MODE_EXPLICIT); + //wp.setTiling(256, 256); + final ParameterValueGroup params = format.getWriteParameters(); + params.parameter(AbstractGridFormat.GEOTOOLS_WRITE_PARAMS.getName().toString()).setValue(wp); + + try { + ReferencedEnvelope envelope = + new ReferencedEnvelope( + geoPosA.getLon(), geoPosB.getLon(), + geoPosA.getLat(), geoPosB.getLat(), + DefaultGeographicCRS.WGS84 + ); + GridCoverageFactory factory = new GridCoverageFactory(); + GridCoverage2D gridCoverage2D = factory.create(imageFileName, quicklookImage, envelope); + GridCoverageWriter writer = format.getWriter(new BufferedOutputStream(pmOutputStream)); + writer.write(gridCoverage2D, params.values().toArray(new GeneralParameterValue[1])); + writer.dispose(); + } finally { + outputStream.close(); + } + } + else { + OutputStream outputStream = createOutputStream(context, imageFileName + "." + config.getImageType()); + LOGGER.info("outputStream: " + outputStream.toString()); + OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); + try { + ImageIO.write(quicklookImage, config.getImageType(), pmOutputStream); + } finally { + outputStream.close(); + } } } // } catch (Exception e) { @@ -100,6 +156,10 @@ public static void createQuicklook(Product product, String imageFileName, Mapper // } } + private static boolean isGeoTiff(Quicklooks.QLConfig qlConfig) { + return "geotiff".equalsIgnoreCase(qlConfig.getImageType()); + } + private static OutputStream createOutputStream(Mapper.Context context, String fileName) throws IOException, InterruptedException { Path path = new Path(FileOutputFormat.getWorkOutputPath(context), fileName); final FSDataOutputStream fsDataOutputStream = path.getFileSystem(context.getConfiguration()).create(path); From 12483ea4745873e68cac75e771a67c13193d6201 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Thu, 29 Nov 2018 18:40:57 +0100 Subject: [PATCH 20/25] Fix bug in GeoTiff quicklook generation Image was not generated unless scene height was the same as scene width --- .../main/java/com/bc/calvalus/processing/analysis/QLMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index 35d0b257e..ade25a85e 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -106,7 +106,7 @@ public static void createQuicklook(Product product, String imageFileName, Mapper OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); final int width = product.getSceneRasterWidth(); - final int height = product.getSceneRasterWidth(); + final int height = product.getSceneRasterHeight(); final GeoCoding geoCoding = product.getSceneGeoCoding(); final PixelPos posA = new org.esa.snap.core.datamodel.PixelPos(0, 0); final GeoPos geoPosA = geoCoding.getGeoPos(posA, null); From 89aba1e934d89d979a3ce72cba511e9f8fecb606 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Fri, 25 Jan 2019 18:32:42 +0100 Subject: [PATCH 21/25] Support upload of quicklook images to GeoServer --- .../client/QuicklookParametersForm.java | 121 ++++++++- .../client/QuicklookParametersForm.ui.xml | 146 +++++++++++ .../processing/analysis/GeoServer.java | 230 ++++++++++++++++++ .../processing/analysis/QLMapper.java | 50 +++- .../processing/analysis/Quicklooks.java | 44 ++++ .../processing/l2/L2FormattingMapper.java | 26 +- 6 files changed, 602 insertions(+), 15 deletions(-) create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java index 4917d1f3a..4b639136b 100755 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/QuicklookParametersForm.java @@ -71,6 +71,8 @@ interface TheUiBinder extends UiBinder { HTMLPanel multiBandPanel; @UiField HTMLPanel moreOptionsPanel; + @UiField + HTMLPanel geoServerPanel; @UiField RadioButton quicklookNone; @@ -116,6 +118,21 @@ interface TheUiBinder extends UiBinder { @UiField CheckBox legendEnabled; + @UiField + TextBox geoServerRestUrl; + @UiField + TextBox geoServerUsername; + @UiField + TextBox geoServerPassword; + @UiField + TextBox geoServerWorkspace; + @UiField + TextBox geoServerStore; + @UiField + TextBox geoServerLayer; + @UiField + TextBox geoServerStyle; + private DtoColorPalette[] availableColorPalettes = null; private Boolean pageLoaded = false; private String[] availableBandNames = null; @@ -237,12 +254,14 @@ private void quicklookNoneChangeHandler() { singleBandPanel.setVisible(false); multiBandPanel.setVisible(false); moreOptionsPanel.setVisible(false); + geoServerPanel.setVisible(false); } private void quicklookSingleBandChangeHandler() { singleBandPanel.setVisible(true); multiBandPanel.setVisible(false); moreOptionsPanel.setVisible(true); + geoServerPanel.setVisible(true); bandNameChangeHandler(); } @@ -250,6 +269,7 @@ private void quicklookMultiBandChangeHandler() { singleBandPanel.setVisible(false); multiBandPanel.setVisible(true); moreOptionsPanel.setVisible(true); + geoServerPanel.setVisible(true); } public void setAvailableImageTypes() { @@ -349,6 +369,56 @@ public Map getValueMap() { legendEnabledXML = indentXML + "true\n"; } + // GeoServer section... + // geoServerRestUrl + String geoServerRestUrlValue = geoServerRestUrl.getValue(); + String geoServerRestUrlXML = ""; + if (geoServerRestUrlValue != null && !geoServerRestUrlValue.isEmpty()) { + geoServerRestUrlXML = indentXML + "" + geoServerRestUrlValue + "\n"; + } + + // geoServerUsername + String geoServerUsernameValue = geoServerUsername.getValue(); + String geoServerUsernameXML = ""; + if (geoServerUsernameValue != null && !geoServerUsernameValue.isEmpty()) { + geoServerUsernameXML = indentXML + "" + geoServerUsernameValue + "\n"; + } + + // geoServerPassword + String geoServerPasswordValue = geoServerPassword.getValue(); + String geoServerPasswordXML = ""; + if (geoServerPasswordValue != null && !geoServerPasswordValue.isEmpty()) { + geoServerPasswordXML = indentXML + "" + geoServerPasswordValue + "\n"; + } + + // geoServerWorkspace + String geoServerWorkspaceValue = geoServerWorkspace.getValue(); + String geoServerWorkspaceXML = ""; + if (geoServerWorkspaceValue != null && !geoServerWorkspaceValue.isEmpty()) { + geoServerWorkspaceXML = indentXML + "" + geoServerWorkspaceValue + "\n"; + } + + // geoServerStore + String geoServerStoreValue = geoServerStore.getValue(); + String geoServerStoreXML = ""; + if (geoServerStoreValue != null && !geoServerStoreValue.isEmpty()) { + geoServerStoreXML = indentXML + "" + geoServerStoreValue + "\n"; + } + + // geoServerLayer + String geoServerLayerValue = geoServerLayer.getValue(); + String geoServerLayerXML = ""; + if (geoServerLayerValue != null && !geoServerLayerValue.isEmpty()) { + geoServerLayerXML = indentXML + "" + geoServerLayerValue + "\n"; + } + + // geoServerStyle + String geoServerStyleValue = geoServerStyle.getValue(); + String geoServerStyleXML = ""; + if (geoServerStyleValue != null && !geoServerStyleValue.isEmpty()) { + geoServerStyleXML = indentXML + "" + geoServerStyleValue + "\n"; + } + String quicklookParameters = ""; if (quicklookSingleBand.getValue()) { @@ -378,6 +448,13 @@ public Map getValueMap() { subSamplingYXML + backgroundColorXML + legendEnabledXML + + geoServerRestUrlXML + + geoServerUsernameXML + + geoServerPasswordXML + + geoServerWorkspaceXML + + geoServerStoreXML + + geoServerLayerXML + + geoServerStyleXML + " \n" + " \n" + ""; @@ -432,6 +509,13 @@ public Map getValueMap() { subSamplingYXML + backgroundColorXML + legendEnabledXML + + geoServerRestUrlXML + + geoServerUsernameXML + + geoServerPasswordXML + + geoServerWorkspaceXML + + geoServerStoreXML + + geoServerLayerXML + + geoServerStyleXML + " \n" + " \n" + ""; @@ -449,7 +533,6 @@ public void setValues(Map parameters) { String quicklookParametersValue = parameters.get("calvalus.ql.parameters"); if (quicklookParametersValue == null || quicklookParametersValue.isEmpty()) { - bandName.setValue(null); colorPalette.setSelectedIndex(0); @@ -467,6 +550,14 @@ public void setValues(Map parameters) { maskOverlays.setValue(null); legendEnabled.setValue(false); + geoServerRestUrl.setValue(null); + geoServerUsername.setValue(null); + geoServerPassword.setValue(null); + geoServerWorkspace.setValue(null); + geoServerStore.setValue(null); + geoServerLayer.setValue(null); + geoServerStyle.setValue(null); + // set no quicklook radio button in GUI quicklookNone.setValue(true, true); } else { @@ -572,6 +663,34 @@ else if (imageTypeValue.equalsIgnoreCase("geotiff")) legendEnabled.setValue(true); else legendEnabled.setValue(false); + + // geoServerRestUrl + String geoServerRestUrlValue = getTagValue(dom, "geoServerRestUrl"); + geoServerRestUrl.setValue(geoServerRestUrlValue); + + // geoServerUsername + String geoServerUsernameValue = getTagValue(dom, "geoServerUsername"); + geoServerUsername.setValue(geoServerUsernameValue); + + // geoServerPassword + String geoServerPasswordValue = getTagValue(dom, "geoServerPassword"); + geoServerPassword.setValue(geoServerPasswordValue); + + // geoServerWorkspace + String geoServerWorkspaceValue = getTagValue(dom, "geoServerWorkspace"); + geoServerWorkspace.setValue(geoServerWorkspaceValue); + + // geoServerStore + String geoServerStoreValue = getTagValue(dom, "geoServerStore"); + geoServerStore.setValue(geoServerStoreValue); + + // geoServerLayer + String geoServerLayerValue = getTagValue(dom, "geoServerLayer"); + geoServerLayer.setValue(geoServerLayerValue); + + // geoServerStyle + String geoServerStyleValue = getTagValue(dom, "geoServerStyle"); + geoServerStyle.setValue(geoServerStyleValue); } } diff --git a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml index 46a09bc12..3ab85b874 100755 --- a/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml +++ b/calvalus-portal/src/main/resources/com/bc/calvalus/portal/client/QuicklookParametersForm.ui.xml @@ -307,6 +307,152 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   
+ GeoServer REST URL: + + +
    + + The URL to GeoServer's REST API (e.g. http://myhost/geoserver/rest/ + +
   
+ GeoServer username: + + +
    + + The username for GeoServer authentication + +
   
+ GeoServer password: + + +
    + + The password for GeoServer authentication + +
   
+ GeoServer workspace: + + +
    + + The GeoServer workspace's name + +
   
+ GeoServer store: + + +
    + + The GeoServer store's name + +
   
+ GeoServer layer: + + +
    + + The GeoServer layer's name + +
   
+ GeoServer style: + + +
    + + The GeoServer style name + +
+ + +
diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java new file mode 100644 index 000000000..b85f2a8ec --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2019 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.processing.analysis; + +import com.bc.calvalus.commons.CalvalusLogger; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.esa.snap.core.datamodel.Product; + +import java.io.*; +import java.net.Authenticator; +import java.net.HttpURLConnection; +import java.net.PasswordAuthentication; +import java.net.URL; +import java.util.logging.Logger; + +/** + * GeoServer REST API handler + * + * @author Declan + */ +public class GeoServer { + + private static final Logger LOGGER = CalvalusLogger.getLogger(); + + private final TaskAttemptContext context; + private final Product sourceProduct; + private final Quicklooks.QLConfig qlConfig; + + public GeoServer(TaskAttemptContext context, Product product, Quicklooks.QLConfig qlConfig) { + this.context = context; + this.sourceProduct = product; + this.qlConfig = qlConfig; + } + + public void uploadImage(InputStream inputStream, String imageName) throws IOException { + + String msg = String.format("Uploading product image '%s' to GeoServer.", imageName); + LOGGER.info(msg); + + String geoserverRestURL = this.qlConfig.getGeoServerRestUrl(); + String username = this.qlConfig.getGeoServerUsername(); + String password = this.qlConfig.getGeoServerPassword(); + String workspace = this.qlConfig.getGeoServerWorkspace(); + String store = this.qlConfig.getGeoServerStore(); + String layer = this.qlConfig.getGeoServerLayer(); + String style = this.qlConfig.getGeoServerStyle(); + + if (geoserverRestURL == null || geoserverRestURL.isEmpty()) { + throw new IllegalArgumentException("geoserverRestURL is empty"); + } + geoserverRestURL = geoserverRestURL.trim(); + while (geoserverRestURL.endsWith("/")) { + geoserverRestURL = geoserverRestURL.substring(0, geoserverRestURL.length() - 1); + } + + if (workspace == null || workspace.isEmpty()) { + throw new IllegalArgumentException("workspace is empty"); + } + + if (store == null || store.isEmpty()) { + store = imageName; + } + else { + store.trim(); + } + + if (layer == null || layer.isEmpty()) { + layer = imageName; + } + else { + layer.trim(); + } + + if (username != null && !username.isEmpty() && password != null) { + Authenticator.setDefault(new Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password.toCharArray()); + } + }); + } + + // other functionality for future (not required at present) + //getCoverageStore(geoserverURL, workspace, layerID); + //deleteCoverageStore(geoserverURL, workspace, layerID); + + putCoverageStoreSingleGeoTiff(inputStream, geoserverRestURL, workspace, store, layer); + + if (style != null && !style.isEmpty()) { + putLayerStyle(geoserverRestURL, workspace, store, style); + } + } + + /** + * Get a coverage store named {storeName} in the {workspace} workspace + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be retrieved + */ + public void getCoverageStore(String geoserverRestURL, String workspace, String store) throws IOException { + String getCoverageStoreURL = String.format("%s/workspaces/%s/coveragestores/%s", geoserverRestURL, workspace, store); + LOGGER.info("Getting GeoServer coverage store: " + getCoverageStoreURL); + + URL url = new URL(getCoverageStoreURL); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "application/xml"); + conn.setRequestProperty("charset", "utf-8"); + conn.setRequestMethod("GET"); + + int responseCode = conn.getResponseCode(); + String responseMessage = conn.getResponseMessage(); + LOGGER.info("GeoServer response: " + responseCode + ": " + responseMessage); + conn.disconnect(); + } + + /** + * Delete a coverage store named {storeName} in the {workspace} workspace + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be deleted + */ + public void deleteCoverageStore(String geoserverRestURL, String workspace, String store) throws IOException { + String deleteCoverageStoreURL = String.format("%/workspaces/%s/coveragestores/%s?recurse=true&purge=all", geoserverRestURL, workspace, store); + LOGGER.info("Deleting GeoServer coverage store: " + deleteCoverageStoreURL); + + URL url = new URL(deleteCoverageStoreURL); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + //conn.setRequestProperty("Content-Type", "application/xml"); + //conn.setRequestProperty("charset", "utf-8"); + conn.setRequestMethod("DELETE"); + + int responseCode = conn.getResponseCode(); + String responseMessage = conn.getResponseMessage(); + LOGGER.info("GeoServer response: " + responseCode + ": " + responseMessage); + conn.disconnect(); + } + + /** + * Creates or overwrites a single coverage store by uploading its GeoTIFF raster data file + * + * @param inputStream an InputStream to the GeoTIFF for updating + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param store the name of the store to be created or overwritten + * @param layer the name of the new layer + */ + public void putCoverageStoreSingleGeoTiff(InputStream inputStream, String geoserverRestURL, String workspace, String store, String layer) throws IOException { + String putCoverageStoreURL = String.format("%s/workspaces/%s/coveragestores/%s/file.geotiff?coverageName=%s", geoserverRestURL, workspace, store, layer); + LOGGER.info("Uploading GeoTIFF to the GeoServer coverage store: " + putCoverageStoreURL); + + byte[] geoTiffPayload = IOUtils.toByteArray(inputStream); + LOGGER.info("GeoTIFF size (bytes): " + geoTiffPayload.length); + + URL url = new URL(putCoverageStoreURL); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoInput(true); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "text/plain"); + //conn.setRequestProperty("charset", "utf-8"); + conn.setRequestMethod("PUT"); + + DataOutputStream out = new DataOutputStream(conn.getOutputStream()); + out.write(geoTiffPayload); + out.flush(); + out.close(); + + int responseCode = conn.getResponseCode(); + String responseMessage = conn.getResponseMessage(); + LOGGER.info("GeoServer response: " + responseCode + ": " + responseMessage); + conn.disconnect(); + } + + /** + * Modify a layer's style + * + * @param geoserverRestURL the root URL to GeoServer + * @param workspace the name of the workspace containing the coverage stores + * @param layer the name of the layer to modify + * @param style the default style to use for this layer + */ + public void putLayerStyle(String geoserverRestURL, String workspace, String layer, String style) throws IOException { + String payload = "" + style + ""; + String uploadStyleURL = String.format("%s/workspaces/%s/layers/%s", geoserverRestURL, workspace, layer); + LOGGER.info("Updating layer style to '" + style + "':" + uploadStyleURL); + + URL url = new URL(uploadStyleURL); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoInput(true); + conn.setDoOutput(true); + conn.setInstanceFollowRedirects(false); + conn.setUseCaches(false); + conn.setRequestProperty("Content-Type", "application/xml"); + //conn.setRequestProperty("charset", "utf-8"); + conn.setRequestMethod("PUT"); + + DataOutputStream out = new DataOutputStream(conn.getOutputStream()); + out.writeBytes(payload); + out.flush(); + out.close(); + + int responseCode = conn.getResponseCode(); + String responseMessage = conn.getResponseMessage(); + LOGGER.info("GeoServer response: " + responseCode + ": " + responseMessage); + conn.disconnect(); + } +} diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index ade25a85e..d7fdb239e 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -24,6 +24,7 @@ import com.bc.calvalus.processing.l2.L2FormattingMapper; import com.bc.ceres.core.ProgressMonitor; import com.bc.ceres.core.SubProgressMonitor; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; @@ -48,7 +49,9 @@ import javax.imageio.ImageIO; import java.awt.image.RenderedImage; +import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.InputStream; import java.io.IOException; import java.io.OutputStream; import java.util.logging.Logger; @@ -75,18 +78,25 @@ public void run(Mapper.Context context) throws IOException, InterruptedException final String productName = FileUtils.getFilenameWithoutExtension(inputFileName); final Quicklooks.QLConfig[] configs = Quicklooks.get(context.getConfiguration()); for (Quicklooks.QLConfig config : configs) { - final String imageFileName; + final String imageBaseName; if (context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REGEX) != null && context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REPLACEMENT) != null) { if (configs.length == 1) { - imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName); + imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName); } else { - imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName) + "_" + config.getBandName(); + imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), inputFileName) + "_" + config.getBandName(); } } else { - imageFileName = productName + "_" + config.getBandName(); + imageBaseName = productName + "_" + config.getBandName(); + } + createQuicklook(product, imageBaseName, context, config); + if( config.getGeoServerRestUrl() != null ) { + // upload geoTiff to GeoServer + GeoServer geoserver = new GeoServer(context, product, config); + String imageFilename = QLMapper.getImageFileName(imageBaseName, config); + InputStream inputStream = QLMapper.createInputStream(context, imageFilename); + geoserver.uploadImage(inputStream, imageBaseName); } - createQuicklook(product, imageFileName, context, config); } } } finally { @@ -95,13 +105,14 @@ public void run(Mapper.Context context) throws IOException, InterruptedException } } - public static void createQuicklook(Product product, String imageFileName, Mapper.Context context, + public static void createQuicklook(Product product, String imageBaseName, Mapper.Context context, Quicklooks.QLConfig config) throws IOException, InterruptedException { // try { + String imageFileName = getImageFileName(imageBaseName, config); RenderedImage quicklookImage = new QuicklookGenerator(context, product, config).createImage(); if (quicklookImage != null) { if( isGeoTiff(config)) { - OutputStream outputStream = createOutputStream(context, imageFileName + ".tiff"); + OutputStream outputStream = createOutputStream(context, imageFileName); LOGGER.info("outputStream: " + outputStream.toString()); OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); @@ -140,7 +151,7 @@ public static void createQuicklook(Product product, String imageFileName, Mapper } } else { - OutputStream outputStream = createOutputStream(context, imageFileName + "." + config.getImageType()); + OutputStream outputStream = createOutputStream(context, imageFileName); LOGGER.info("outputStream: " + outputStream.toString()); OutputStream pmOutputStream = new BytesCountingOutputStream(outputStream, context); try { @@ -156,16 +167,37 @@ public static void createQuicklook(Product product, String imageFileName, Mapper // } } + + public static String getImageFileName(String imageBaseName, Quicklooks.QLConfig qlConfig) { + if( isGeoTiff(qlConfig)) { + return imageBaseName + ".tiff"; + } + else { + return imageBaseName + qlConfig.getImageType(); + } + } + + public static Path getWorkOutputFilePath(Mapper.Context context, String fileName) throws IOException, InterruptedException { + return new Path(FileOutputFormat.getWorkOutputPath(context), fileName); + } + private static boolean isGeoTiff(Quicklooks.QLConfig qlConfig) { return "geotiff".equalsIgnoreCase(qlConfig.getImageType()); } private static OutputStream createOutputStream(Mapper.Context context, String fileName) throws IOException, InterruptedException { - Path path = new Path(FileOutputFormat.getWorkOutputPath(context), fileName); + Path path = getWorkOutputFilePath(context, fileName); + LOGGER.info("createOutputStream: path = " + path); final FSDataOutputStream fsDataOutputStream = path.getFileSystem(context.getConfiguration()).create(path); return new BufferedOutputStream(fsDataOutputStream); } + public static InputStream createInputStream(Mapper.Context context, String fileName) throws IOException, InterruptedException { + Path path = getWorkOutputFilePath(context, fileName); + LOGGER.info("createInputStream: path = " + path); + final FSDataInputStream fsDataInputStream = path.getFileSystem(context.getConfiguration()).open(path); + return new BufferedInputStream(fsDataInputStream); + } private static class BytesCountingOutputStream extends OutputStream { diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java index d3345d1e9..8d8892302 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/Quicklooks.java @@ -76,6 +76,21 @@ public static class QLConfig { @Parameter private String shapefileURL; + @Parameter + private String geoServerRestUrl; + @Parameter + private String geoServerUsername; + @Parameter + private String geoServerPassword; + @Parameter + private String geoServerWorkspace; + @Parameter + private String geoServerStore; + @Parameter + private String geoServerLayer; + @Parameter + private String geoServerStyle; + public String getImageType() { return imageType; } @@ -155,5 +170,34 @@ public void setLegendEnabled(boolean legendEnabled) { public void setOverlayURL(String overlayURL) { this.overlayURL = overlayURL; } + + public String getGeoServerRestUrl() { + return geoServerRestUrl; + } + + public String getGeoServerUsername() { + return geoServerUsername; + } + + public String getGeoServerPassword() { + return geoServerPassword; + } + + public String getGeoServerWorkspace() { + return geoServerWorkspace; + } + + public String getGeoServerStore() { + return geoServerStore; + } + + public String getGeoServerLayer() { + return geoServerLayer; + } + + public String getGeoServerStyle() { + return geoServerStyle; + } + } } diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java index be0168779..cf0d461aa 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/L2FormattingMapper.java @@ -21,6 +21,7 @@ import com.bc.calvalus.processing.JobConfigNames; import com.bc.calvalus.processing.ProcessorAdapter; import com.bc.calvalus.processing.ProcessorFactory; +import com.bc.calvalus.processing.analysis.GeoServer; import com.bc.calvalus.processing.analysis.QLMapper; import com.bc.calvalus.processing.analysis.Quicklooks; import com.bc.calvalus.processing.hadoop.ProgressSplitProgressMonitor; @@ -39,6 +40,7 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.text.DateFormat; import java.util.*; import java.util.logging.Level; @@ -163,22 +165,36 @@ private void writeQuicklooks(Mapper.Context context, Configuration jobConfig, St List qlConfigList = getValidQlConfigs(jobConfig); for (Quicklooks.QLConfig qlConfig : qlConfigList) { - String imageFileName; + String imageBaseName; // if (context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REGEX) != null // && context.getConfiguration().get(JobConfigNames.CALVALUS_OUTPUT_REPLACEMENT) != null) { -// imageFileName = L2FormattingMapper.getProductName(context.getConfiguration(), productName); +// imageBaseName = L2FormattingMapper.getProductName(context.getConfiguration(), productName); // } else { - imageFileName = productName; + imageBaseName = productName; // } if (qlConfigList.size() > 1) { - imageFileName = imageFileName + "_" + qlConfig.getBandName(); + imageBaseName = imageBaseName + "_" + qlConfig.getBandName(); } + try { - QLMapper.createQuicklook(targetProduct, imageFileName, context, qlConfig); + QLMapper.createQuicklook(targetProduct, imageBaseName, context, qlConfig); } catch (Exception e) { String msg = String.format("Could not create quicklook image '%s'.", qlConfig.getBandName()); LOG.log(Level.WARNING, msg, e); } + + if( qlConfig.getGeoServerRestUrl() != null ) { + // upload geoTiff to GeoServer + try { + GeoServer geoserver = new GeoServer(context, targetProduct, qlConfig); + String imageFilename = QLMapper.getImageFileName(imageBaseName, qlConfig); + InputStream inputStream = QLMapper.createInputStream(context, imageFilename); + geoserver.uploadImage(inputStream, imageBaseName); + } catch (Exception e) { + String msg = String.format("Could not upload quicklook image '%s' to GeoServer.", qlConfig.getBandName()); + LOG.log(Level.WARNING, msg, e); + } + } } LOG.info("Finished creating quicklooks."); } From adcf58224cb7054d6bcb1b6b9df7ebf004006a8d Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Fri, 25 Jan 2019 21:35:29 +0100 Subject: [PATCH 22/25] Bug fix for updating a layer's style in GeoServer --- .../java/com/bc/calvalus/processing/analysis/GeoServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java index b85f2a8ec..e76401ba3 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/GeoServer.java @@ -101,7 +101,7 @@ protected PasswordAuthentication getPasswordAuthentication() { putCoverageStoreSingleGeoTiff(inputStream, geoserverRestURL, workspace, store, layer); if (style != null && !style.isEmpty()) { - putLayerStyle(geoserverRestURL, workspace, store, style); + putLayerStyle(geoserverRestURL, workspace, layer, style); } } @@ -205,7 +205,7 @@ public void putCoverageStoreSingleGeoTiff(InputStream inputStream, String geoser public void putLayerStyle(String geoserverRestURL, String workspace, String layer, String style) throws IOException { String payload = "" + style + ""; String uploadStyleURL = String.format("%s/workspaces/%s/layers/%s", geoserverRestURL, workspace, layer); - LOGGER.info("Updating layer style to '" + style + "':" + uploadStyleURL); + LOGGER.info("Updating layer style to '" + style + "': " + uploadStyleURL); URL url = new URL(uploadStyleURL); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); From 7451c1e2f4deb253bacbe2be5d55c22df8edcad4 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Thu, 21 Feb 2019 16:22:07 +0100 Subject: [PATCH 23/25] Fix filename bug for PNG, JPEG and TIFF quicklook --- .../main/java/com/bc/calvalus/processing/analysis/QLMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java index d7fdb239e..fd9b1c529 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/analysis/QLMapper.java @@ -173,7 +173,7 @@ public static String getImageFileName(String imageBaseName, Quicklooks.QLConfig return imageBaseName + ".tiff"; } else { - return imageBaseName + qlConfig.getImageType(); + return imageBaseName + "." + qlConfig.getImageType(); } } From 20275a0ce4d26101fd63c572bc7f9b81b7979dec Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Fri, 22 Feb 2019 17:48:18 +0100 Subject: [PATCH 24/25] Fixes for 'mvn clean install' --- calvalus-processing/pom.xml | 9 ++------- calvalus-snap/pom.xml | 6 ++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/calvalus-processing/pom.xml b/calvalus-processing/pom.xml index 28d3218e5..7db82f1ec 100644 --- a/calvalus-processing/pom.xml +++ b/calvalus-processing/pom.xml @@ -19,12 +19,6 @@ - - edu.ucar - netcdf - 4.3.23 - - org.slf4j slf4j-api @@ -62,11 +56,12 @@ org.esa.snap snap-gpf + org.xeustechnologies jtar diff --git a/calvalus-snap/pom.xml b/calvalus-snap/pom.xml index 7da296f57..17e8aca47 100644 --- a/calvalus-snap/pom.xml +++ b/calvalus-snap/pom.xml @@ -82,6 +82,12 @@ org.esa.snapsnap-watermask org.esa.snapsnap-sta + + ncsa.hdfjhdfobj + ncsa.hdfjhdf4obj + ncsa.hdfjhdf5obj + ncsa.hdflib-hdf + org.esa.s3tbxs3tbx-alos-reader org.esa.s3tbxs3tbx-atsr-reader org.esa.s3tbxs3tbx-avhrr-reader From f9fab904c492d6a2844997673834e884e81786a8 Mon Sep 17 00:00:00 2001 From: Declan Dunne Date: Fri, 22 Feb 2019 17:52:17 +0100 Subject: [PATCH 25/25] Fixes for 'mvn clean install' --- calvalus-snap/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/calvalus-snap/pom.xml b/calvalus-snap/pom.xml index 17e8aca47..7da296f57 100644 --- a/calvalus-snap/pom.xml +++ b/calvalus-snap/pom.xml @@ -82,12 +82,6 @@ org.esa.snapsnap-watermask org.esa.snapsnap-sta - - ncsa.hdfjhdfobj - ncsa.hdfjhdf4obj - ncsa.hdfjhdf5obj - ncsa.hdflib-hdf - org.esa.s3tbxs3tbx-alos-reader org.esa.s3tbxs3tbx-atsr-reader org.esa.s3tbxs3tbx-avhrr-reader