diff --git a/cdm/core/build.gradle b/cdm/core/build.gradle index f02decb9ff..2ef449b1be 100644 --- a/cdm/core/build.gradle +++ b/cdm/core/build.gradle @@ -8,7 +8,7 @@ apply from: "$rootDir/gradle/any/protobuf.gradle" dependencies { api enforcedPlatform(project(':netcdf-java-platform')) - implementation 'commons-math:commons-math' + implementation 'org.apache.commons:commons-math3' testImplementation enforcedPlatform(project(':netcdf-java-testing-platform')) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 5581a1faf4..c12a232341 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -34,6 +34,8 @@ */ public class VariableDS extends Variable implements VariableEnhanced, EnhanceScaleMissingUnsigned { + + /** * Constructor when there's no underlying variable. * You must also set the values by doing one of: @@ -269,8 +271,10 @@ Array convert(Array data, Set enhancements) { // datatype of the result depends on what enhancements were applied DataType convertedType = data.getDataType(); + // TODO: change to a provider for extensible Enhancements List toApply = new ArrayList<>(); + if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) { toApply.add(unsignedConversion); convertedType = unsignedConversion.getOutType(); @@ -283,15 +287,9 @@ Array convert(Array data, Set enhancements) { toApply.add(scaleOffset); convertedType = scaleOffset.getScaledOffsetType(); } - if (enhancements.contains(Enhance.ApplyStandardizer) && standardizer != null) { - toApply.add(standardizer); - } - if (enhancements.contains(Enhance.ApplyNormalizer) && normalizer != null) { - toApply.add(normalizer); - } - if (enhancements.contains(Enhance.ApplyClassifier) && classifier != null) { - toApply.add(classifier); - } + + toApply.addAll(loadedEnhancements); + double[] dataArray = (double[]) data.get1DJavaArray(DataType.DOUBLE); @@ -866,9 +864,8 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset // TODO make immutable in version 6 private UnsignedConversion unsignedConversion; private ScaleOffset scaleOffset; - private Standardizer standardizer; - private Normalizer normalizer; - private Classifier classifier; + private List loadedEnhancements = new ArrayList<>(); + private ConvertMissing convertMissing; private Set enhanceMode = EnumSet.noneOf(Enhance.class); // The set of enhancements that were made. @@ -935,18 +932,14 @@ private void createEnhancements() { } this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; } - Attribute standardizerAtt = findAttribute(CDM.STANDARDIZE); - if (standardizerAtt != null && this.enhanceMode.contains(Enhance.ApplyStandardizer) && dataType.isFloatingPoint()) { - this.standardizer = Standardizer.createFromVariable(this); - } - Attribute normalizerAtt = findAttribute(CDM.NORMALIZE); - if (normalizerAtt != null && this.enhanceMode.contains(Enhance.ApplyNormalizer) && dataType.isFloatingPoint()) { - this.normalizer = Normalizer.createFromVariable(this); - } - Attribute classifierAtt = findAttribute(CDM.CLASSIFY); - if (classifierAtt != null && this.enhanceMode.contains(Enhance.ApplyClassifier) && dataType.isNumeric()) { - this.classifier = Classifier.createFromVariable(this); + for (Enhance enhance : this.enhanceMode) { + for (EnhancementProvider service : ServiceLoader.load(EnhancementProvider.class)) { + if (service.appliesTo(enhance, this.attributes(), dataType)) { + loadedEnhancements.add(service.create(this)); + } + } } + } public Builder toBuilder() { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java b/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java index ab3cae4ad2..1734400be2 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java @@ -1,20 +1,28 @@ package ucar.nc2.filter; - import ucar.ma2.Array; import ucar.ma2.IndexIterator; -import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.Attribute; +import ucar.nc2.dataset.NetcdfDataset.Enhance; +import ucar.nc2.dataset.VariableDS; import ucar.nc2.util.Misc; import java.util.ArrayList; import java.util.List; +import ucar.ma2.*; +import ucar.nc2.*; public class Classifier implements Enhancement { + + private String[] AttCat; private List rules = new ArrayList<>(); + private static String name = "Classifier"; + + + public Classifier() { this.AttCat = new String[0]; this.rules = new ArrayList<>(); @@ -27,7 +35,8 @@ public Classifier(String[] AttCat) { } // Factory method to create a Classifier from a Variable - public static Classifier createFromVariable(Variable var) { + + public static Classifier createFromVariable(VariableDS var) { List attributes = var.attributes().getAttributes(); for (Attribute attribute : attributes) { @@ -116,4 +125,18 @@ public static int[] stringToIntArray(String str) { return intArray; } + public static class Provider implements EnhancementProvider { + @Override + public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) { + return enhance == Enhance.ApplyClassifier && attributes.findAttribute(CDM.CLASSIFY) != null && dt.isNumeric(); + } + + @Override + public Classifier create(VariableDS var) { + return createFromVariable(var); + } + } + } + + diff --git a/cdm/core/src/main/java/ucar/nc2/filter/EnhancementProvider.java b/cdm/core/src/main/java/ucar/nc2/filter/EnhancementProvider.java new file mode 100644 index 0000000000..a8cf2fdb04 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/EnhancementProvider.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2021 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.filter; + +import ucar.ma2.*; +import ucar.nc2.*; +import ucar.nc2.dataset.NetcdfDataset.Enhance; +import ucar.nc2.dataset.VariableDS; + + +/** + * A Service Provider of {@link Enhancement}. + */ +public interface EnhancementProvider { + + boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt); + + Enhancement create(VariableDS var); + + +} + + diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java b/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java index 688c94090f..fcae8710c4 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java @@ -3,11 +3,15 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; - -import org.apache.commons.math.stat.descriptive.SummaryStatistics; +import ucar.nc2.constants.CDM; +import java.util.Set; +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; +import ucar.nc2.AttributeContainer; +import ucar.nc2.Variable; +import ucar.nc2.dataset.NetcdfDataset.Enhance; import ucar.nc2.dataset.VariableDS; public class Normalizer implements Enhancement { @@ -15,6 +19,7 @@ public class Normalizer implements Enhancement { private final ScaleOffset scaleOffset; private final double minimum; private final double range; // maximum - minimum + private static String name = "Normalizer"; public static Normalizer createFromVariable(VariableDS var) { try { @@ -72,4 +77,22 @@ public double getRange() { return range; } + public static class Provider implements EnhancementProvider { + + + @Override + public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) { + return enhance == Enhance.ApplyNormalizer && attributes.findAttribute(CDM.NORMALIZE) != null + && dt.isFloatingPoint(); + } + + @Override + public Normalizer create(VariableDS var) { + return Normalizer.createFromVariable(var); + } + + + + } + } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 21dd426e5a..4f7de9ff11 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -290,5 +290,8 @@ public int getId() { public Filter create(Map properties) { return new ScaleOffset(properties); } + } + + } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java b/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java index 01613ba8c8..5bf8948b29 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Standardizer.java @@ -3,11 +3,15 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import ucar.nc2.constants.CDM; -import org.apache.commons.math.stat.descriptive.SummaryStatistics; +import java.util.Set; +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; +import ucar.nc2.AttributeContainer; +import ucar.nc2.dataset.NetcdfDataset.Enhance; import ucar.nc2.dataset.VariableDS; public class Standardizer implements Enhancement { @@ -15,6 +19,7 @@ public class Standardizer implements Enhancement { private final ScaleOffset scaleOffset; private final double mean; private final double stdDev; + private static String name = "Standardizer"; public static Standardizer createFromVariable(VariableDS var) { try { @@ -71,5 +76,22 @@ public double getMean() { public double getStdDev() { return stdDev; } + + + + public static class Provider implements EnhancementProvider { + + + @Override + public boolean appliesTo(Enhance enhance, AttributeContainer attributes, DataType dt) { + return enhance == Enhance.ApplyStandardizer && attributes.findAttribute(CDM.STANDARDIZE) != null + && dt.isFloatingPoint(); + } + + @Override + public Standardizer create(VariableDS var) { + return createFromVariable(var); + } + } } diff --git a/cdm/core/src/main/resources/META-INF/services/ucar.nc2.filter.EnhancementProvider b/cdm/core/src/main/resources/META-INF/services/ucar.nc2.filter.EnhancementProvider new file mode 100644 index 0000000000..4fc1725949 --- /dev/null +++ b/cdm/core/src/main/resources/META-INF/services/ucar.nc2.filter.EnhancementProvider @@ -0,0 +1,6 @@ +ucar.nc2.filter.Classifier$Provider +ucar.nc2.filter.Standardizer$Provider +ucar.nc2.filter.Normalizer$Provider + + + diff --git a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java index a61d43d2c8..089a5de2f3 100644 --- a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java +++ b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java @@ -12,6 +12,7 @@ import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.dataset.VariableDS; import ucar.nc2.filter.Classifier; import ucar.unidata.util.test.TestDir; @@ -33,7 +34,7 @@ public void testEnhanceClassifier_Ints() throws IOException { assertThat((Object) classifySpecs).isNotNull(); assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); Array Data = classifySpecs.read(); - Classifier classifier = Classifier.createFromVariable(classifySpecs); + Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs); int[] ClassifiedArray = classifier.classifyWithAttributes(Data); assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue(); @@ -53,7 +54,7 @@ public void testEnhanceClassifier_Floats() throws IOException { assertThat((Object) classifySpecs).isNotNull(); assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); Array Data = classifySpecs.read(); - Classifier classifier = Classifier.createFromVariable(classifySpecs); + Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs); int[] ClassifiedArray = classifier.classifyWithAttributes(Data); assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue(); @@ -73,7 +74,7 @@ public void testEnhanceClassifier_classification() throws IOException { assertThat((Object) classifySpecs).isNotNull(); assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); Array Data = classifySpecs.read(); - Classifier classifier = Classifier.createFromVariable(classifySpecs); + Classifier classifier = Classifier.createFromVariable((VariableDS) classifySpecs); int[] ClassifiedArray = classifier.classifyWithAttributes(Data); assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), CLASSIFICATION_TEST)).isTrue(); diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index b1d8b8b439..51a986c523 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -37,7 +37,7 @@ dependencies { api 'com.beust:jcommander:1.78' // cdm-core - api 'commons-math:commons-math:1.2' + api 'org.apache.commons:commons-math3:3.6.1' // cdm-grib api 'edu.ucar:jj2000:5.4'