Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Throw and handle exception when trying to create a variable with a zero dimension #1083

Merged
merged 1 commit into from
Apr 12, 2024
Merged

Throw and handle exception when trying to create a variable with a zero dimension #1083

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
295 changes: 156 additions & 139 deletions cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1247,146 +1247,154 @@ else if (dataType == DataType.ENUM2)
}

private Variable.Builder makeVariable(Group.Builder parentGroup, DataObjectFacade facade) throws IOException {

Vinfo vinfo = new Vinfo(facade);
if (vinfo.getNCDataType() == null) {
log.debug("SKIPPING DataType= " + vinfo.typeInfo.hdfType + " for variable " + facade.name);
return null;
}

Attribute fillAttribute = null;
for (HeaderMessage mess : facade.dobj.messages) {
if (mess.mtype == MessageType.FillValue) {
MessageFillValue fvm = (MessageFillValue) mess.messData;
if (fvm.hasFillValue)
vinfo.fillValue = fvm.value;
} else if (mess.mtype == MessageType.FillValueOld) {
MessageFillValueOld fvm = (MessageFillValueOld) mess.messData;
if (fvm.size > 0)
vinfo.fillValue = fvm.value;
try {
Vinfo vinfo = new Vinfo(facade);
if (vinfo.getNCDataType() == null) {
log.debug("SKIPPING DataType= " + vinfo.typeInfo.hdfType + " for variable " + facade.name);
return null;
}

Object fillValue = vinfo.getFillValueNonDefault();
if (fillValue != null) {
Object defFillValue = N3iosp.getFillValueDefault(vinfo.typeInfo.dataType);
if (!fillValue.equals(defFillValue))
fillAttribute = new Attribute(CDM.FILL_VALUE, (Number) fillValue, vinfo.typeInfo.unsigned);
Attribute fillAttribute = null;
for (HeaderMessage mess : facade.dobj.messages) {
if (mess.mtype == MessageType.FillValue) {
MessageFillValue fvm = (MessageFillValue) mess.messData;
if (fvm.hasFillValue)
vinfo.fillValue = fvm.value;
} else if (mess.mtype == MessageType.FillValueOld) {
MessageFillValueOld fvm = (MessageFillValueOld) mess.messData;
if (fvm.size > 0)
vinfo.fillValue = fvm.value;
}

Object fillValue = vinfo.getFillValueNonDefault();
if (fillValue != null) {
Object defFillValue = N3iosp.getFillValueDefault(vinfo.typeInfo.dataType);
if (!fillValue.equals(defFillValue))
fillAttribute = new Attribute(CDM.FILL_VALUE, (Number) fillValue, vinfo.typeInfo.unsigned);
}
}
}

long dataAddress = facade.dobj.msl.dataAddress;
long dataAddress = facade.dobj.msl.dataAddress;

// deal with unallocated data
if (dataAddress == -1) {
vinfo.useFillValue = true;
// deal with unallocated data
if (dataAddress == -1) {
vinfo.useFillValue = true;

// if didnt find, use zeroes !!
if (vinfo.fillValue == null) {
vinfo.fillValue = new byte[vinfo.typeInfo.dataType.getSize()];
// if didnt find, use zeroes !!
if (vinfo.fillValue == null) {
vinfo.fillValue = new byte[vinfo.typeInfo.dataType.getSize()];
}
}
}

Variable.Builder vb;
Structure.Builder sb = null;
if (facade.dobj.mdt.type == 6) { // Compound
String vname = facade.name;
vb = sb = Structure.builder().setName(vname);
vb.setParentGroupBuilder(parentGroup);
if (!makeVariableShapeAndType(parentGroup, sb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList))
return null;
addMembersToStructure(parentGroup, sb, facade.dobj.mdt);
sb.setElementSize(facade.dobj.mdt.byteSize);
Variable.Builder vb;
Structure.Builder sb = null;
if (facade.dobj.mdt.type == 6) { // Compound
String vname = facade.name;
vb = sb = Structure.builder().setName(vname);
vb.setParentGroupBuilder(parentGroup);
if (!makeVariableShapeAndType(parentGroup, sb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList))
return null;
addMembersToStructure(parentGroup, sb, facade.dobj.mdt);
sb.setElementSize(facade.dobj.mdt.byteSize);

} else {
String vname = facade.name;
if (vname.startsWith(Nc4.NETCDF4_NON_COORD))
vname = vname.substring(Nc4.NETCDF4_NON_COORD.length()); // skip prefix
vb = Variable.builder().setName(vname);
vb.setParentGroupBuilder(parentGroup);
if (!makeVariableShapeAndType(parentGroup, vb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList))
return null;

// special case of variable length strings
if (vb.dataType == DataType.STRING)
vb.setElementSize(16); // because the array has elements that are HeapIdentifier
else if (vb.dataType == DataType.OPAQUE) // special case of opaque
vb.setElementSize(facade.dobj.mdt.getBaseSize());
}
} else {
String vname = facade.name;
if (vname.startsWith(Nc4.NETCDF4_NON_COORD))
vname = vname.substring(Nc4.NETCDF4_NON_COORD.length()); // skip prefix
vb = Variable.builder().setName(vname);
vb.setParentGroupBuilder(parentGroup);
if (!makeVariableShapeAndType(parentGroup, vb, facade.dobj.mdt, facade.dobj.mds, vinfo, facade.dimList))
return null;

// special case of variable length strings
if (vb.dataType == DataType.STRING)
vb.setElementSize(16); // because the array has elements that are HeapIdentifier
else if (vb.dataType == DataType.OPAQUE) // special case of opaque
vb.setElementSize(facade.dobj.mdt.getBaseSize());
}

vb.setSPobject(vinfo);
vb.setSPobject(vinfo);

// look for attributes
List<MessageAttribute> fatts = filterAttributes(facade.dobj.attributes);
for (MessageAttribute matt : fatts) {
try {
makeAttributes(sb, matt, vb.getAttributeContainer());
} catch (InvalidRangeException e) {
throw new IOException(e.getMessage());
// look for attributes
List<MessageAttribute> fatts = filterAttributes(facade.dobj.attributes);
for (MessageAttribute matt : fatts) {
try {
makeAttributes(sb, matt, vb.getAttributeContainer());
} catch (InvalidRangeException e) {
throw new IOException(e.getMessage());
}
}
}

AttributeContainerMutable atts = vb.getAttributeContainer();
processSystemAttributes(facade.dobj.messages, atts);
if (fillAttribute != null && atts.findAttribute(CDM.FILL_VALUE) == null)
vb.addAttribute(fillAttribute);
// if (vinfo.typeInfo.unsigned)
// v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));
if (facade.dobj.mdt.type == 5) {
String desc = facade.dobj.mdt.opaque_desc;
if ((desc != null) && (!desc.isEmpty()))
vb.addAttribute(new Attribute("_opaqueDesc", desc));
}
AttributeContainerMutable atts = vb.getAttributeContainer();
processSystemAttributes(facade.dobj.messages, atts);
if (fillAttribute != null && atts.findAttribute(CDM.FILL_VALUE) == null)
vb.addAttribute(fillAttribute);
// if (vinfo.typeInfo.unsigned)
// v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));
if (facade.dobj.mdt.type == 5) {
String desc = facade.dobj.mdt.opaque_desc;
if ((desc != null) && (!desc.isEmpty()))
vb.addAttribute(new Attribute("_opaqueDesc", desc));
}

int[] shape = makeVariableShape(facade.dobj.mdt, facade.dobj.mds, facade.dimList);
if (vinfo.isChunked) { // make the data btree, but entries are not read in
vinfo.btree = new DataBTree(this, dataAddress, shape, vinfo.storageSize, memTracker);
int[] shape = makeVariableShape(facade.dobj.mdt, facade.dobj.mds, facade.dimList);
if (vinfo.isChunked) { // make the data btree, but entries are not read in
vinfo.btree = new DataBTree(this, dataAddress, shape, vinfo.storageSize, memTracker);

if (vinfo.isChunked) { // add an attribute describing the chunk size
List<Integer> chunksize = new ArrayList<>();
for (int i = 0; i < vinfo.storageSize.length - 1; i++) // skip last one - its the element size
chunksize.add(vinfo.storageSize[i]);
vb.addAttribute(Attribute.builder(CDM.CHUNK_SIZES).setValues((List) chunksize, true).build());
if (vinfo.isChunked) { // add an attribute describing the chunk size
List<Integer> chunksize = new ArrayList<>();
for (int i = 0; i < vinfo.storageSize.length - 1; i++) // skip last one - its the element size
chunksize.add(vinfo.storageSize[i]);
vb.addAttribute(Attribute.builder(CDM.CHUNK_SIZES).setValues((List) chunksize, true).build());
}
}
}

if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 0)) { // object reference
// System.out.printf("new transform object Reference: facade= %s variable name=%s%n", facade.name, vb.shortName);
vb.setDataType(DataType.STRING);
Array rawData = vinfo.readArray();
Array refData = findReferenceObjectNames(rawData);
vb.setCachedData(refData, true); // so H5iosp.read() is never called
vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are names of referenced Variables"));
}
if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 0)) { // object
// reference
// System.out.printf("new transform object Reference: facade= %s variable name=%s%n", facade.name,
// vb.shortName);
vb.setDataType(DataType.STRING);
Array rawData = vinfo.readArray();
Array refData = findReferenceObjectNames(rawData);
vb.setCachedData(refData, true); // so H5iosp.read() is never called
vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are names of referenced Variables"));
}

if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 1)) { // region reference
if (warnings)
log.warn("transform region Reference: facade=" + facade.name + " variable name=" + vb.shortName);
if (transformReference && (facade.dobj.mdt.type == 7) && (facade.dobj.mdt.referenceType == 1)) { // region
// reference
if (warnings)
log.warn("transform region Reference: facade=" + facade.name + " variable name=" + vb.shortName);

/*
* TODO doesnt work yet
* int nelems = (int) vb.getSize();
* int heapIdSize = 12;
* for (int i = 0; i < nelems; i++) {
* H5header.RegionReference heapId = new RegionReference(vinfo.dataPos + heapIdSize * i);
* }
*/

// fake data for now
vb.setDataType(DataType.LONG);
Array newData = Array.factory(DataType.LONG, shape);
vb.setCachedData(newData, true); // so H5iosp.read() is never called
vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are regions of referenced Variables"));
}

/*
* TODO doesnt work yet
* int nelems = (int) vb.getSize();
* int heapIdSize = 12;
* for (int i = 0; i < nelems; i++) {
* H5header.RegionReference heapId = new RegionReference(vinfo.dataPos + heapIdSize * i);
* }
*/
// debugging
vinfo.setOwner(vb);
// TODO: handle logging warnings and unknown filters at the Filter level

// fake data for now
vb.setDataType(DataType.LONG);
Array newData = Array.factory(DataType.LONG, shape);
vb.setCachedData(newData, true); // so H5iosp.read() is never called
vb.addAttribute(new Attribute("_HDF5ReferenceType", "values are regions of referenced Variables"));
}
if (debug1) {
log.debug("makeVariable " + vb.shortName + "; vinfo= " + vinfo);
}

// debugging
vinfo.setOwner(vb);
// TODO: handle logging warnings and unknown filters at the Filter level
return vb;

if (debug1) {
log.debug("makeVariable " + vb.shortName + "; vinfo= " + vinfo);
} catch (InvalidRangeException e) {
log.error(e.getMessage());
return null;
}

return vb;
}

// convert an array of lons which are data object references to an array of strings,
Expand Down Expand Up @@ -1434,29 +1442,34 @@ private Variable.Builder makeVariableMember(Group.Builder parentGroup, String na
return null;
}

if (mdt.type == 6) {
Structure.Builder sb = Structure.builder().setName(name).setParentGroupBuilder(parentGroup);
makeVariableShapeAndType(parentGroup, sb, mdt, null, vinfo, null);
addMembersToStructure(parentGroup, sb, mdt);
sb.setElementSize(mdt.byteSize);

sb.setSPobject(vinfo);
vinfo.setOwner(sb);
return sb;

} else {
Variable.Builder vb = Variable.builder().setName(name).setParentGroupBuilder(parentGroup);
makeVariableShapeAndType(parentGroup, vb, mdt, null, vinfo, null);
try {
if (mdt.type == 6) {
Structure.Builder sb = Structure.builder().setName(name).setParentGroupBuilder(parentGroup);
makeVariableShapeAndType(parentGroup, sb, mdt, null, vinfo, null);
addMembersToStructure(parentGroup, sb, mdt);
sb.setElementSize(mdt.byteSize);

// special case of variable length strings
if (vb.dataType == DataType.STRING)
vb.setElementSize(16); // because the array has elements that are HeapIdentifier
else if (vb.dataType == DataType.OPAQUE) // special case of opaque
vb.setElementSize(mdt.getBaseSize());
sb.setSPobject(vinfo);
vinfo.setOwner(sb);
return sb;

vb.setSPobject(vinfo);
vinfo.setOwner(vb);
return vb;
} else {
Variable.Builder vb = Variable.builder().setName(name).setParentGroupBuilder(parentGroup);
makeVariableShapeAndType(parentGroup, vb, mdt, null, vinfo, null);

// special case of variable length strings
if (vb.dataType == DataType.STRING)
vb.setElementSize(16); // because the array has elements that are HeapIdentifier
else if (vb.dataType == DataType.OPAQUE) // special case of opaque
vb.setElementSize(mdt.getBaseSize());

vb.setSPobject(vinfo);
vinfo.setOwner(vb);
return vb;
}
} catch (InvalidRangeException e) {
log.error(e.getMessage());
return null;
}
}

Expand Down Expand Up @@ -1525,7 +1538,7 @@ private int[] makeVariableShape(MessageDatatype mdt, MessageDataspace msd, Strin

// set the type and shape of the Variable
private boolean makeVariableShapeAndType(Group.Builder parent, Variable.Builder v, MessageDatatype mdt,
MessageDataspace msd, Vinfo vinfo, String dimNames) {
MessageDataspace msd, Vinfo vinfo, String dimNames) throws InvalidRangeException {

int[] shape = makeVariableShape(mdt, msd, dimNames);

Expand All @@ -1536,6 +1549,10 @@ private boolean makeVariableShapeAndType(Group.Builder parent, Variable.Builder
else
v.setDimensionsByName(dimNames);
} else {
for (int i = 0; i < shape.length; i++) {
if ((shape[i] < 1) && (shape[i] != -1))
throw new InvalidRangeException("shape[" + i + "]=" + shape[i] + " must be > 0");
}
v.setDimensionsAnonymous(shape);
}

Expand Down