Skip to content

Commit

Permalink
Update cdfwrite to include a more intelligent error message (#257)
Browse files Browse the repository at this point in the history 
* Updating cdfwrite to throw an error when incorrect types are used
  • Loading branch information
@maxinelasp
maxinelasp authored May 14, 2024
1 parent dfb257b commit 614815e
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 18 deletions.
13 changes: 8 additions & 5 deletions cdflib/cdfwrite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2285,7 +2285,6 @@ def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata:
odata : byte stream
The stream of bytes to write to the CDF file
"""

recSize = self._datatype_size(data_type, num_elems) * num_values
# List or Tuple data
if isinstance(indata, list) or isinstance(indata, tuple):
Expand Down Expand Up @@ -2393,10 +2392,14 @@ def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata:
complex_data.append(indata.imag)
indata = complex_data
form = tofrom + str(recs * num_values * num_elems) + dt_string
if recs * num_values * num_elems > 1:
return recs, struct.pack(form, *indata)
else:
return recs, struct.pack(form, indata)
try:
if recs * num_values * num_elems > 1:
return recs, struct.pack(form, *indata)
else:
return recs, struct.pack(form, indata)
except struct.error:
raise ValueError("Unable to convert data to CDF format, data "
"object cannot be of type string.")

def _num_values(self, zVar: bool, varNum: int) -> int:
"""
Expand Down
51 changes: 38 additions & 13 deletions tests/test_cdfwrite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from typing import Any, Dict

import numpy as np
import pytest

from cdflib import cdfread, cdfwrite

Expand Down Expand Up @@ -43,7 +44,8 @@ def test_checksum(tmp_path):
varatts["Attribute1"] = 1
varatts["Attribute2"] = "500"

tfile.write_var(var_spec, var_attrs=varatts, var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
tfile.write_var(var_spec, var_attrs=varatts,
var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))

tfile.close()

Expand Down Expand Up @@ -250,13 +252,15 @@ def test_create_zvariables_with_attributes(tmp_path):
varatts["Attribute2"] = "500"

tfile = cdf_create(fn, {"rDim_sizes": [1]})
tfile.write_var(var_spec, var_attrs=varatts, var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
tfile.write_var(var_spec, var_attrs=varatts,
var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))

var_spec["Variable"] = "Variable2"
varatts2: Dict[str, Any] = {}
varatts2["Attribute1"] = 2
varatts2["Attribute2"] = "1000"
tfile.write_var(var_spec, var_attrs=varatts2, var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
tfile.write_var(var_spec, var_attrs=varatts2,
var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))

tfile.close()

Expand Down Expand Up @@ -348,7 +352,9 @@ def test_sparse_virtual_zvariable_blocking(tmp_path):
physical_records2 = np.linspace(20001, 30000, num=10000)
physical_records3 = np.linspace(50001, 60000, num=10000)
physical_records4 = np.linspace(70001, 140000, num=70000)
physical_records = np.concatenate((physical_records1, physical_records2, physical_records3, physical_records4)).astype(int)
physical_records = np.concatenate((physical_records1, physical_records2,
physical_records3, physical_records4)).astype(
int)
sparse_data = [physical_records, data]

tfile = cdf_create(fn, {"rDim_sizes": [1]})
Expand Down Expand Up @@ -384,7 +390,9 @@ def test_sparse_zvariable_blocking(tmp_path):
physical_records2 = np.linspace(20001, 30000, num=10000)
physical_records3 = np.linspace(50001, 60000, num=10000)
physical_records4 = np.linspace(70001, 140000, num=70000)
physical_records = np.concatenate((physical_records1, physical_records2, physical_records3, physical_records4)).astype(int)
physical_records = np.concatenate((physical_records1, physical_records2,
physical_records3, physical_records4)).astype(
int)
sparse_data = [physical_records, data]

tfile = cdf_create(fn, {"rDim_sizes": [1]})
Expand Down Expand Up @@ -414,7 +422,8 @@ def test_sparse_zvariable_pad(tmp_path):
var_spec["Rec_Vary"] = True
var_spec["Dim_Sizes"] = []
var_spec["Sparse"] = "pad_sparse"
data = [[200, 3000, 3100, 3500, 4000, 5000, 6000, 10000, 10001, 10002, 20000], np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])]
data = [[200, 3000, 3100, 3500, 4000, 5000, 6000, 10000, 10001, 10002, 20000],
np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])]

tfile = cdf_create(fn, {"rDim_sizes": [1]})
tfile.write_var(var_spec, var_data=data)
Expand Down Expand Up @@ -443,7 +452,8 @@ def test_sparse_zvariable_previous(tmp_path):
var_spec["Rec_Vary"] = True
var_spec["Dim_Sizes"] = []
var_spec["Sparse"] = "prev_sparse"
data = [[200, 3000, 3100, 3500, 4000, 5000, 6000, 10000, 10001, 10002, 20000], np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])]
data = [[200, 3000, 3100, 3500, 4000, 5000, 6000, 10000, 10001, 10002, 20000],
np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])]

tfile = cdf_create(fn, {"rDim_sizes": [1]})
tfile.write_var(var_spec, var_data=data)
Expand Down Expand Up @@ -476,7 +486,9 @@ def test_create_2d_rvariable(tmp_path):

tfile = cdf_create(fn, {"rDim_sizes": [2, 2]})
tfile.write_var(
var_spec, var_data=np.array([[[0, 1], [1, 2]], [[2, 3], [3, 4]], [[4, 5], [5, 6]], [[6, 7], [7, 8]], [[8, 9], [9, 10]]])
var_spec, var_data=np.array(
[[[0, 1], [1, 2]], [[2, 3], [3, 4]], [[4, 5], [5, 6]], [[6, 7], [7, 8]],
[[8, 9], [9, 10]]])
)
tfile.close()

Expand Down Expand Up @@ -510,7 +522,8 @@ def test_create_2d_rvariable_dimvary(tmp_path):

tfile = cdf_create(fn, {"rDim_sizes": [2, 20]})

tfile.write_var(var_spec, var_data=np.array([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]))
tfile.write_var(var_spec,
var_data=np.array([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]))

tfile.close()

Expand Down Expand Up @@ -541,14 +554,16 @@ def test_create_2d_r_and_z_variables(tmp_path):
var_spec["Dim_Vary"] = [True, False]

tfile = cdf_create(fn, {"rDim_sizes": [2, 20]})
tfile.write_var(var_spec, var_data=np.array([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]))
tfile.write_var(var_spec,
var_data=np.array([[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]))

var_spec["Variable"] = "Variable2"
var_spec["Var_Type"] = "zvariable"
varatts: Dict[str, Any] = {}
varatts["Attribute1"] = 2
varatts["Attribute2"] = "1000"
tfile.write_var(var_spec, var_attrs=varatts, var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
tfile.write_var(var_spec, var_attrs=varatts,
var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))

tfile.close()
# Open the file to read
Expand Down Expand Up @@ -590,7 +605,8 @@ def test_create_zvariables_with_attributes_to_convert(tmp_path):
varatts["Attribute3"] = [700, "CDF_INT8"]

tfile = cdf_create(fn, {"rDim_sizes": [1]})
tfile.write_var(var_spec, var_attrs=varatts, var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
tfile.write_var(var_spec, var_attrs=varatts,
var_data=np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))

tfile.close()

Expand Down Expand Up @@ -620,7 +636,8 @@ def test_create_zvariables_with_data_to_convert(tmp_path):
var_spec["Dim_Sizes"] = []

tfile = cdf_create(fn, {"rDim_sizes": [1]})
tfile.write_var(var_spec, var_data=[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])
tfile.write_var(var_spec,
var_data=[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])

tfile.close()

Expand All @@ -630,3 +647,11 @@ def test_create_zvariables_with_data_to_convert(tmp_path):
# Test CDF info
var = reader.varget("Variable1")
assert var[3] == 3


def test_convert_data_error(tmp_path):
indata = int(-9223372036854775808)
cdf = cdfwrite.CDF(tmp_path / "test.cdf", cdf_spec={"rDim_sizes": [1]})
with pytest.raises(ValueError):
# Data from list of strings with dimension "epoch"
output = cdf._convert_data(51, 1, 1, indata)

0 comments on commit 614815e

Please sign in to comment.