From 9b73ffc08747a439a886bbf3ed22ec3f896cc0e5 Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Fri, 16 Sep 2022 15:52:36 -0700 Subject: [PATCH] Throw and handle InvalidRangeException when trying to create a variable with a zero dimension --- .../nc2/internal/iosp/hdf5/H5headerNew.java | 295 +++++++++--------- 1 file changed, 156 insertions(+), 139 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java index 4666d52008..ec0a345211 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java @@ -1247,146 +1247,154 @@ else if (dataType == DataType.ENUM2) } private Variable.Builder makeVariable(Group.Builder parentGroup, DataObjectFacade facade) throws IOException { - - Vinfo vinfo = new Vinfo(facade); - if (vinfo.getNCDataType() == null) { - log.debug("SKIPPING DataType= " + vinfo.typeInfo.hdfType + " for variable " + facade.name); - return null; - } - - Attribute fillAttribute = null; - for (HeaderMessage mess : facade.dobj.messages) { - if (mess.mtype == MessageType.FillValue) { - MessageFillValue fvm = (MessageFillValue) mess.messData; - if (fvm.hasFillValue) - vinfo.fillValue = fvm.value; - } else if (mess.mtype == MessageType.FillValueOld) { - MessageFillValueOld fvm = (MessageFillValueOld) mess.messData; - if (fvm.size > 0) - vinfo.fillValue = fvm.value; + try { + Vinfo vinfo = new Vinfo(facade); + if (vinfo.getNCDataType() == null) { + log.debug("SKIPPING DataType= " + vinfo.typeInfo.hdfType + " for variable " + facade.name); + return null; } - Object fillValue = vinfo.getFillValueNonDefault(); - if (fillValue != null) { - Object defFillValue = N3iosp.getFillValueDefault(vinfo.typeInfo.dataType); - if (!fillValue.equals(defFillValue)) - fillAttribute = new Attribute(CDM.FILL_VALUE, (Number) fillValue, vinfo.typeInfo.unsigned); + Attribute fillAttribute = null; + for (HeaderMessage mess : facade.dobj.messages) { + if (mess.mtype == MessageType.FillValue) { + MessageFillValue fvm = (MessageFillValue) mess.messData; + if (fvm.hasFillValue) + vinfo.fillValue = fvm.value; + } else if (mess.mtype == MessageType.FillValueOld) { + MessageFillValueOld fvm = (MessageFillValueOld) mess.messData; + if (fvm.size > 0) + vinfo.fillValue = fvm.value; + } + + Object fillValue = vinfo.getFillValueNonDefault(); + if (fillValue != null) { + Object defFillValue = N3iosp.getFillValueDefault(vinfo.typeInfo.dataType); + if (!fillValue.equals(defFillValue)) + fillAttribute = new Attribute(CDM.FILL_VALUE, (Number) fillValue, vinfo.typeInfo.unsigned); + } } - } - long dataAddress = facade.dobj.msl.dataAddress; + long dataAddress = facade.dobj.msl.dataAddress; - // deal with unallocated data - if (dataAddress == -1) { - vinfo.useFillValue = true; + // deal with unallocated data + if (dataAddress == -1) { + vinfo.useFillValue = true; - // if didnt find, use zeroes !! - if (vinfo.fillValue == null) { - vinfo.fillValue = new byte[vinfo.typeInfo.dataType.getSize()]; + // if didnt find, use zeroes !! + if (vinfo.fillValue == null) { + vinfo.fillValue = new byte[vinfo.typeInfo.dataType.getSize()]; + } } - } - Variable.Builder vb; - Structure.Builder sb = null; - if (facade.dobj.mdt.type == 6) { // Compound - String vname = facade.name; - vb = sb = Structure.builder().setName(vname); - vb.setParentGroupBuilder(parentGroup); - if (!makeVariableShapeAndType(parentGroup, sb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList)) - return null; - addMembersToStructure(parentGroup, sb, facade.dobj.mdt); - sb.setElementSize(facade.dobj.mdt.byteSize); + Variable.Builder vb; + Structure.Builder sb = null; + if (facade.dobj.mdt.type == 6) { // Compound + String vname = facade.name; + vb = sb = Structure.builder().setName(vname); + vb.setParentGroupBuilder(parentGroup); + if (!makeVariableShapeAndType(parentGroup, sb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList)) + return null; + addMembersToStructure(parentGroup, sb, facade.dobj.mdt); + sb.setElementSize(facade.dobj.mdt.byteSize); - } else { - String vname = facade.name; - if (vname.startsWith(Nc4.NETCDF4_NON_COORD)) - vname = vname.substring(Nc4.NETCDF4_NON_COORD.length()); // skip prefix - vb = Variable.builder().setName(vname); - vb.setParentGroupBuilder(parentGroup); - if (!makeVariableShapeAndType(parentGroup, vb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList)) - return null; - - // special case of variable length strings - if (vb.dataType == DataType.STRING) - vb.setElementSize(16); // because the array has elements that are HeapIdentifier - else if (vb.dataType == DataType.OPAQUE) // special case of opaque - vb.setElementSize(facade.dobj.mdt.getBaseSize()); - } + } else { + String vname = facade.name; + if (vname.startsWith(Nc4.NETCDF4_NON_COORD)) + vname = vname.substring(Nc4.NETCDF4_NON_COORD.length()); // skip prefix + vb = Variable.builder().setName(vname); + vb.setParentGroupBuilder(parentGroup); + if (!makeVariableShapeAndType(parentGroup, vb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList)) + return null; + + // special case of variable length strings + if (vb.dataType == DataType.STRING) + vb.setElementSize(16); // because the array has elements that are HeapIdentifier + else if (vb.dataType == DataType.OPAQUE) // special case of opaque + vb.setElementSize(facade.dobj.mdt.getBaseSize()); + } - vb.setSPobject(vinfo); + vb.setSPobject(vinfo); - // look for attributes - List fatts = filterAttributes(facade.dobj.attributes); - for (MessageAttribute matt : fatts) { - try { - makeAttributes(sb, matt, vb.getAttributeContainer()); - } catch (InvalidRangeException e) { - throw new IOException(e.getMessage()); + // look for attributes + List fatts = filterAttributes(facade.dobj.attributes); + for (MessageAttribute matt : fatts) { + try { + makeAttributes(sb, matt, vb.getAttributeContainer()); + } catch (InvalidRangeException e) { + throw new IOException(e.getMessage()); + } } - } - AttributeContainerMutable atts = vb.getAttributeContainer(); - processSystemAttributes(facade.dobj.messages, atts); - if (fillAttribute != null && atts.findAttribute(CDM.FILL_VALUE) == null) - vb.addAttribute(fillAttribute); - // if (vinfo.typeInfo.unsigned) - // v.addAttribute(new Attribute(CDM.UNSIGNED, "true")); - if (facade.dobj.mdt.type == 5) { - String desc = facade.dobj.mdt.opaque_desc; - if ((desc != null) && (!desc.isEmpty())) - vb.addAttribute(new Attribute("_opaqueDesc", desc)); - } + AttributeContainerMutable atts = vb.getAttributeContainer(); + processSystemAttributes(facade.dobj.messages, atts); + if (fillAttribute != null && atts.findAttribute(CDM.FILL_VALUE) == null) + vb.addAttribute(fillAttribute); + // if (vinfo.typeInfo.unsigned) + // v.addAttribute(new Attribute(CDM.UNSIGNED, "true")); + if (facade.dobj.mdt.type == 5) { + String desc = facade.dobj.mdt.opaque_desc; + if ((desc != null) && (!desc.isEmpty())) + vb.addAttribute(new Attribute("_opaqueDesc", desc)); + } - int[] shape = makeVariableShape(facade.dobj.mdt, facade.dobj.mds, facade.dimList); - if (vinfo.isChunked) { // make the data btree, but entries are not read in - vinfo.btree = new DataBTree(this, dataAddress, shape, vinfo.storageSize, memTracker); + int[] shape = makeVariableShape(facade.dobj.mdt, facade.dobj.mds, facade.dimList); + if (vinfo.isChunked) { // make the data btree, but entries are not read in + vinfo.btree = new DataBTree(this, dataAddress, shape, vinfo.storageSize, memTracker); - if (vinfo.isChunked) { // add an attribute describing the chunk size - List chunksize = new ArrayList<>(); - for (int i = 0; i < vinfo.storageSize.length - 1; i++) // skip last one - its the element size - chunksize.add(vinfo.storageSize[i]); - vb.addAttribute(Attribute.builder(CDM.CHUNK_SIZES).setValues((List) chunksize, true).build()); + if (vinfo.isChunked) { // add an attribute describing the chunk size + List chunksize = new ArrayList<>(); + for (int i = 0; i < vinfo.storageSize.length - 1; i++) // skip last one - its the element size + chunksize.add(vinfo.storageSize[i]); + vb.addAttribute(Attribute.builder(CDM.CHUNK_SIZES).setValues((List) chunksize, true).build()); + } } - } - if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 0)) { // object reference - // System.out.printf("new transform object Reference: facade= %s variable name=%s%n", facade.name, vb.shortName); - vb.setDataType(DataType.STRING); - Array rawData = vinfo.readArray(); - Array refData = findReferenceObjectNames(rawData); - vb.setCachedData(refData, true); // so H5iosp.read() is never called - vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are names of referenced Variables")); - } + if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 0)) { // object + // reference + // System.out.printf("new transform object Reference: facade= %s variable name=%s%n", facade.name, + // vb.shortName); + vb.setDataType(DataType.STRING); + Array rawData = vinfo.readArray(); + Array refData = findReferenceObjectNames(rawData); + vb.setCachedData(refData, true); // so H5iosp.read() is never called + vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are names of referenced Variables")); + } - if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 1)) { // region reference - if (warnings) - log.warn("transform region Reference: facade=" + facade.name + " variable name=" + vb.shortName); + if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 1)) { // region + // reference + if (warnings) + log.warn("transform region Reference: facade=" + facade.name + " variable name=" + vb.shortName); + + /* + * TODO doesnt work yet + * int nelems = (int) vb.getSize(); + * int heapIdSize = 12; + * for (int i = 0; i < nelems; i++) { + * H5header.RegionReference heapId = new RegionReference(vinfo.dataPos + heapIdSize * i); + * } + */ + + // fake data for now + vb.setDataType(DataType.LONG); + Array newData = Array.factory(DataType.LONG, shape); + vb.setCachedData(newData, true); // so H5iosp.read() is never called + vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are regions of referenced Variables")); + } - /* - * TODO doesnt work yet - * int nelems = (int) vb.getSize(); - * int heapIdSize = 12; - * for (int i = 0; i < nelems; i++) { - * H5header.RegionReference heapId = new RegionReference(vinfo.dataPos + heapIdSize * i); - * } - */ + // debugging + vinfo.setOwner(vb); + // TODO: handle logging warnings and unknown filters at the Filter level - // fake data for now - vb.setDataType(DataType.LONG); - Array newData = Array.factory(DataType.LONG, shape); - vb.setCachedData(newData, true); // so H5iosp.read() is never called - vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are regions of referenced Variables")); - } + if (debug1) { + log.debug("makeVariable " + vb.shortName + "; vinfo= " + vinfo); + } - // debugging - vinfo.setOwner(vb); - // TODO: handle logging warnings and unknown filters at the Filter level + return vb; - if (debug1) { - log.debug("makeVariable " + vb.shortName + "; vinfo= " + vinfo); + } catch (InvalidRangeException e) { + log.error(e.getMessage()); + return null; } - - return vb; } // convert an array of lons which are data object references to an array of strings, @@ -1434,29 +1442,34 @@ private Variable.Builder makeVariableMember(Group.Builder parentGroup, String na return null; } - if (mdt.type == 6) { - Structure.Builder sb = Structure.builder().setName(name).setParentGroupBuilder(parentGroup); - makeVariableShapeAndType(parentGroup, sb, mdt, null, vinfo, null); - addMembersToStructure(parentGroup, sb, mdt); - sb.setElementSize(mdt.byteSize); - - sb.setSPobject(vinfo); - vinfo.setOwner(sb); - return sb; - - } else { - Variable.Builder vb = Variable.builder().setName(name).setParentGroupBuilder(parentGroup); - makeVariableShapeAndType(parentGroup, vb, mdt, null, vinfo, null); + try { + if (mdt.type == 6) { + Structure.Builder sb = Structure.builder().setName(name).setParentGroupBuilder(parentGroup); + makeVariableShapeAndType(parentGroup, sb, mdt, null, vinfo, null); + addMembersToStructure(parentGroup, sb, mdt); + sb.setElementSize(mdt.byteSize); - // special case of variable length strings - if (vb.dataType == DataType.STRING) - vb.setElementSize(16); // because the array has elements that are HeapIdentifier - else if (vb.dataType == DataType.OPAQUE) // special case of opaque - vb.setElementSize(mdt.getBaseSize()); + sb.setSPobject(vinfo); + vinfo.setOwner(sb); + return sb; - vb.setSPobject(vinfo); - vinfo.setOwner(vb); - return vb; + } else { + Variable.Builder vb = Variable.builder().setName(name).setParentGroupBuilder(parentGroup); + makeVariableShapeAndType(parentGroup, vb, mdt, null, vinfo, null); + + // special case of variable length strings + if (vb.dataType == DataType.STRING) + vb.setElementSize(16); // because the array has elements that are HeapIdentifier + else if (vb.dataType == DataType.OPAQUE) // special case of opaque + vb.setElementSize(mdt.getBaseSize()); + + vb.setSPobject(vinfo); + vinfo.setOwner(vb); + return vb; + } + } catch (InvalidRangeException e) { + log.error(e.getMessage()); + return null; } } @@ -1525,7 +1538,7 @@ private int[] makeVariableShape(MessageDatatype mdt, MessageDataspace msd, Strin // set the type and shape of the Variable private boolean makeVariableShapeAndType(Group.Builder parent, Variable.Builder v, MessageDatatype mdt, - MessageDataspace msd, Vinfo vinfo, String dimNames) { + MessageDataspace msd, Vinfo vinfo, String dimNames) throws InvalidRangeException { int[] shape = makeVariableShape(mdt, msd, dimNames); @@ -1536,6 +1549,10 @@ private boolean makeVariableShapeAndType(Group.Builder parent, Variable.Builder else v.setDimensionsByName(dimNames); } else { + for (int i = 0; i < shape.length; i++) { + if ((shape[i] < 1) && (shape[i] != -1)) + throw new InvalidRangeException("shape[" + i + "]=" + shape[i] + " must be > 0"); + } v.setDimensionsAnonymous(shape); }