diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java b/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java index 650b273d59..ab3cae4ad2 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Classifier.java @@ -1,45 +1,63 @@ package ucar.nc2.filter; -import java.io.IOException; + import ucar.ma2.Array; -import ucar.ma2.DataType; import ucar.ma2.IndexIterator; -import ucar.nc2.dataset.VariableDS; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.Attribute; +import ucar.nc2.util.Misc; +import java.util.ArrayList; +import java.util.List; + public class Classifier implements Enhancement { - private Classifier classifier = null; - private static Classifier emptyClassifier; - private int classifiedVal; - private int[] classifiedArray; - - public static Classifier createFromVariable(VariableDS var) { - try { - Array arr = var.read(); - // DataType type = var.getDataType(); - return emptyClassifier(); - } catch (IOException e) { - return emptyClassifier(); - } + private String[] AttCat; + private List rules = new ArrayList<>(); + + public Classifier() { + this.AttCat = new String[0]; + this.rules = new ArrayList<>(); } - public static Classifier emptyClassifier() { - emptyClassifier = new Classifier(); - return emptyClassifier; + // Constructor with attributes + public Classifier(String[] AttCat) { + this.AttCat = AttCat; + this.rules = loadClassificationRules(); } - /** Enough of a constructor */ - public Classifier() {} + // Factory method to create a Classifier from a Variable + public static Classifier createFromVariable(Variable var) { + List attributes = var.attributes().getAttributes(); + + for (Attribute attribute : attributes) { + // check like this, or something else? + if (attribute == var.attributes().findAttribute(CDM.CLASSIFY)) { + String[] sets = attribute.getStringValue().split(";"); + for (int i = 0; i < sets.length; i++) { + // trim and clean so it's ready + sets[i] = sets[i].trim(); + } + return new Classifier(sets); + } + } + + return new Classifier(); + + } + + - /** Classify double array */ - public int[] classifyDoubleArray(Array arr) { + public int[] classifyWithAttributes(Array arr) { int[] classifiedArray = new int[(int) arr.getSize()]; - int i = 0; IndexIterator iterArr = arr.getIndexIterator(); + int i = 0; while (iterArr.hasNext()) { Number value = (Number) iterArr.getObjectNext(); if (!Double.isNaN(value.doubleValue())) { - - classifiedArray[i] = classifyArray(value.doubleValue()); + classifiedArray[i] = classifyArrayAttribute(value.doubleValue()); + } else { + classifiedArray[i] = Integer.MIN_VALUE; } i++; } @@ -48,23 +66,54 @@ public int[] classifyDoubleArray(Array arr) { - /** for a single double */ - public int classifyArray(double val) { - if (val >= 0) { - classifiedVal = 1; - } else { - classifiedVal = 0; + public int classifyArrayAttribute(double val) { + for (int[] rule : rules) { + if (val > rule[0] && val <= rule[1] + Misc.defaultMaxRelativeDiffFloat) { + return rule[2]; + } } + // Return min possible int if no rule matches + return Integer.MIN_VALUE; + } - return classifiedVal; + // Method to load classification rules from the attributes + private List loadClassificationRules() { + for (String rules : this.AttCat) { + int[] rule = stringToIntArray(rules); + this.rules.add(rule); + } + return rules; } @Override public double convert(double val) { - return emptyClassifier.classifyArray(val); + return classifyArrayAttribute(val); } + public static int[] stringToIntArray(String str) { + String[] stringArray = str.split(" "); // Split the string by spaces + int[] intArray = new int[stringArray.length]; // Create an array to hold the parsed integers + + for (int i = 0; i < stringArray.length; i++) { + + double value = Double.parseDouble(stringArray[i]); // Parse each string to a double + + if (Double.isNaN(value)) { + // Check if the entry is NaN and assign Integer.MIN_VALUE or Integer.MAX_VALUE based on the index + if (i == 0) { + intArray[i] = Integer.MIN_VALUE; + } else if (i == 1) { + intArray[i] = Integer.MAX_VALUE; + } else { + intArray[i] = -99999; // Default value for other indices if needed + } + } else { + intArray[i] = (int) value; // Convert the value to int if it is not NaN + } -} + } + return intArray; + } +} diff --git a/cdm/core/src/test/data/ncml/enhance/testAddToClassifier.ncml b/cdm/core/src/test/data/ncml/enhance/testAddToClassifier.ncml new file mode 100644 index 0000000000..8938e30e60 --- /dev/null +++ b/cdm/core/src/test/data/ncml/enhance/testAddToClassifier.ncml @@ -0,0 +1,29 @@ + + + + + + + + + -500000 NaN -10 0 1 2 3 11 25 29 NaN 100 150 121 102 199999 12211 + + + + + 1 -2 0 4 -5 + + + + + 1.0 -2.0 0.0 4.0 -5.0 + + + + + + + diff --git a/cdm/core/src/test/data/ncml/enhance/testClassifier.ncml b/cdm/core/src/test/data/ncml/enhance/testClassifier.ncml deleted file mode 100644 index b649675d99..0000000000 --- a/cdm/core/src/test/data/ncml/enhance/testClassifier.ncml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - 1.0 2.0 3.0 4.0 5.0 - - - - - -1.0 -2.0 -3.0 -4.0 -5.0 - - - - - 1.0 -2.0 0.0 4.0 -5.0 - - - - - 1.0 2.0 3.0 4.0 5.0 - - - - - -1.0 -2.0 -3.0 -4.0 -5.0 - - - - 1.0 -2.0 0.0 4.0 -5.0 - - - - - 1 2 3 4 5 - - - - - -1.0 -2.0 -3.0 -4.0 -5.0 - - - - 1.0 -2.0 0.0 4.0 -5.0 - - - - - diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestClassifier.java b/cdm/core/src/test/java/ucar/nc2/filter/TestClassifier.java deleted file mode 100644 index 967861064e..0000000000 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestClassifier.java +++ /dev/null @@ -1,51 +0,0 @@ -package ucar.nc2.filter; - -import static org.junit.Assert.*; -import org.junit.Test; -import ucar.ma2.Array; - - -public class TestClassifier { - - - - /** test doubles */ - @Test - public void testClassifyDoubleArray_AllPositive() { - Classifier classifier = new Classifier(); - double[] input = {1.1, 2.2, 3.3}; - int[] expected = {1, 1, 1}; - Array DATA = Array.makeFromJavaArray(input); - assertArrayEquals(expected, classifier.classifyDoubleArray(DATA)); - } - - @Test - public void testClassifyDoubleArray_AllNegative() { - Classifier classifier = new Classifier(); - double[] input = {-1.1, -2.2, -3.3}; - int[] expected = {0, 0, 0}; - Array DATA = Array.makeFromJavaArray(input); - assertArrayEquals(expected, classifier.classifyDoubleArray(DATA)); - } - - @Test - public void testClassifyDoubleArray_Mixed() { - Classifier classifier = new Classifier(); - double[] input = {-1.1, 2.2, -3.3, 4.4}; - int[] expected = {0, 1, 0, 1}; - Array DATA = Array.makeFromJavaArray(input); - assertArrayEquals(expected, classifier.classifyDoubleArray(DATA)); - } - - @Test - public void testClassifyDoubleArray_WithZero() { - Classifier classifier = new Classifier(); - double[] input = {0.0, -1.1, 1.1, 0.0, 0.0, 0.0}; - int[] expected = {1, 0, 1, 1, 1, 1}; - Array DATA = Array.makeFromJavaArray(input); - assertArrayEquals(expected, classifier.classifyDoubleArray(DATA)); - } - - - -} diff --git a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java index 00fac9b05d..a61d43d2c8 100644 --- a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java +++ b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhanceClassifier.java @@ -2,114 +2,86 @@ import static com.google.common.truth.Truth.assertThat; import static ucar.ma2.MAMath.nearlyEquals; - import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import ucar.ma2.Array; import ucar.ma2.DataType; +import ucar.nc2.Attribute; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.filter.Classifier; import ucar.unidata.util.test.TestDir; public class TestEnhanceClassifier { private static String dataDir = TestDir.cdmLocalTestDataDir + "ncml/enhance/"; - - public static final int[] all_ones = {1, 1, 1, 1, 1}; - public static final Array DATA_all_ones = Array.makeFromJavaArray(all_ones); - public static final int[] all_zeroes = {0, 0, 0, 0, 0}; - public static final Array DATA_all_zeroes = Array.makeFromJavaArray(all_zeroes); - public static final int[] mixNumbers = {1, 0, 1, 1, 0}; + public static final int[] mixNumbers = {1, 0, 0, 1, 0}; public static final Array DATA_mixNumbers = Array.makeFromJavaArray(mixNumbers); + public static final int[] Classification_test = + {0, -2147483648, 0, 0, 10, 10, 10, 100, 100, 100, -2147483648, 100, 1000, 1000, 1000, 1000, 1000}; + public static final Array CLASSIFICATION_TEST = Array.makeFromJavaArray(Classification_test); - - /** test on doubles, all positives, all negatives and a mixed array */ @Test - public void testEnhanceClassifier_doubles() throws IOException { - try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testClassifier.ncml", true, null)) { - Variable doublePositives = ncfile.findVariable("doublePositives"); - assertThat((Object) doublePositives).isNotNull(); - assertThat(doublePositives.getDataType()).isEqualTo(DataType.DOUBLE); - assertThat(doublePositives.attributes().hasAttribute("classify")).isTrue(); - Array dataDoubles = doublePositives.read(); - assertThat(nearlyEquals(dataDoubles, DATA_all_ones)).isTrue(); - - Variable doubleNegatives = ncfile.findVariable("doubleNegatives"); - assertThat((Object) doubleNegatives).isNotNull(); - assertThat(doubleNegatives.getDataType()).isEqualTo(DataType.DOUBLE); - assertThat(doubleNegatives.attributes().hasAttribute("classify")).isTrue(); - Array datadoubleNegatives = doubleNegatives.read(); - assertThat(nearlyEquals(datadoubleNegatives, DATA_all_zeroes)).isTrue(); - - Variable doubleMix = ncfile.findVariable("doubleMix"); - assertThat((Object) doubleMix).isNotNull(); - assertThat(doubleMix.getDataType()).isEqualTo(DataType.DOUBLE); - assertThat(doubleMix.attributes().hasAttribute("classify")).isTrue(); - Array datadoubleMix = doubleMix.read(); - assertThat(nearlyEquals(datadoubleMix, DATA_mixNumbers)).isTrue(); + public void testEnhanceClassifier_Ints() throws IOException { - } + try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testAddToClassifier.ncml", true, null)) { + Variable classifySpecs = ncfile.findVariable("classify_ints"); + assertThat((Object) classifySpecs).isNotNull(); + assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); + Array Data = classifySpecs.read(); + Classifier classifier = Classifier.createFromVariable(classifySpecs); + int[] ClassifiedArray = classifier.classifyWithAttributes(Data); + assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue(); + + + } catch (IOException e) { + throw new RuntimeException(e); + } } - /** test on floats, all positives, all negatives and a mixed array */ @Test - public void testEnhanceClassifier_floats() throws IOException { - try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testClassifier.ncml", true, null)) { - - Variable floatPositives = ncfile.findVariable("floatPositives"); - assertThat((Object) floatPositives).isNotNull(); - assertThat(floatPositives.getDataType()).isEqualTo(DataType.FLOAT); - assertThat(floatPositives.attributes().hasAttribute("classify")).isTrue(); - Array datafloats = floatPositives.read(); - assertThat(nearlyEquals(datafloats, DATA_all_ones)).isTrue(); - - Variable floatNegatives = ncfile.findVariable("floatNegatives"); - assertThat((Object) floatNegatives).isNotNull(); - assertThat(floatNegatives.getDataType()).isEqualTo(DataType.FLOAT); - assertThat(floatNegatives.attributes().hasAttribute("classify")).isTrue(); - Array datafloatNegatives = floatNegatives.read(); - assertThat(nearlyEquals(datafloatNegatives, DATA_all_zeroes)).isTrue(); - - Variable floatMix = ncfile.findVariable("floatMix"); - assertThat((Object) floatMix).isNotNull(); - assertThat(floatMix.getDataType()).isEqualTo(DataType.FLOAT); - assertThat(floatMix.attributes().hasAttribute("classify")).isTrue(); - Array datafloatsMix = floatMix.read(); - assertThat(nearlyEquals(datafloatsMix, DATA_mixNumbers)).isTrue(); + public void testEnhanceClassifier_Floats() throws IOException { + + try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testAddToClassifier.ncml", true, null)) { + Variable classifySpecs = ncfile.findVariable("classify_floats"); + assertThat((Object) classifySpecs).isNotNull(); + assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); + Array Data = classifySpecs.read(); + Classifier classifier = Classifier.createFromVariable(classifySpecs); + int[] ClassifiedArray = classifier.classifyWithAttributes(Data); + assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), DATA_mixNumbers)).isTrue(); + + + } catch (IOException e) { + throw new RuntimeException(e); } } - /** enhance is not applied to Integers, so we expect the same values after application */ @Test - public void testEnhanceClassifier_integers() throws IOException { - - try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testClassifier.ncml", true, null)) { - Variable IntegerPositives = ncfile.findVariable("intPositives"); - assertThat((Object) IntegerPositives).isNotNull(); - assertThat(IntegerPositives.getDataType()).isEqualTo(DataType.INT); - assertThat(IntegerPositives.attributes().hasAttribute("classify")).isTrue(); - Array dataIntegers = IntegerPositives.read(); - assertThat(nearlyEquals(dataIntegers, DATA_all_ones)).isTrue(); - - Variable intNegatives = ncfile.findVariable("intNegatives"); - assertThat((Object) intNegatives).isNotNull(); - assertThat(intNegatives.getDataType()).isEqualTo(DataType.INT); - assertThat(intNegatives.attributes().hasAttribute("classify")).isTrue(); - Array dataintNegatives = intNegatives.read(); - assertThat(nearlyEquals(dataintNegatives, DATA_all_zeroes)).isTrue(); - - Variable intMix = ncfile.findVariable("intMix"); - assertThat((Object) intMix).isNotNull(); - assertThat(intMix.getDataType()).isEqualTo(DataType.INT); - assertThat(intMix.attributes().hasAttribute("classify")).isTrue(); - Array dataintMix = intMix.read(); - assertThat(nearlyEquals(dataintMix, DATA_mixNumbers)).isTrue(); + public void testEnhanceClassifier_classification() throws IOException { + + try (NetcdfFile ncfile = NetcdfDatasets.openDataset(dataDir + "testAddToClassifier.ncml", true, null)) { + + Variable classifySpecs = ncfile.findVariable("class_specs"); + assertThat((Object) classifySpecs).isNotNull(); + assertThat(!classifySpecs.attributes().isEmpty()).isTrue(); + Array Data = classifySpecs.read(); + Classifier classifier = Classifier.createFromVariable(classifySpecs); + int[] ClassifiedArray = classifier.classifyWithAttributes(Data); + assertThat(nearlyEquals(Array.makeFromJavaArray(ClassifiedArray), CLASSIFICATION_TEST)).isTrue(); + + + } catch (IOException e) { + throw new RuntimeException(e); } } + }