Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update of the Initialization of CParams, DParams, and Storage in Blosc2 #278

Merged
merged 4 commits into from
Oct 10, 2024
Merged

Update of the Initialization of CParams, DParams, and Storage in Blosc2 #278

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0603be
Update of the Initialization of CParams, DParams, and Storage in Blosc2
omaech Oct 7, 2024
5552be2
More updated examples
omaech Oct 8, 2024
7621087
More updated examples of CParams, DParams and Storage
omaech Oct 8, 2024
c71f8c6
All the examples are up to date
omaech Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions examples/btune.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

nchunks = 10
# Set the compression and decompression parameters, use BTUNE tuner
cparams = {"codec": blosc2.Codec.LZ4HC, "typesize": 4, "tuner": blosc2.Tuner.BTUNE}
dparams = {}
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4HC, typesize=4, tuner=blosc2.Tuner.BTUNE)
dparams = blosc2.DParams()
contiguous = True
urlpath = "filename"

storage = {"contiguous": contiguous, "urlpath": urlpath, "cparams": cparams, "dparams": dparams}
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode='a')
blosc2.remove_urlpath(urlpath)

# Set the Btune configuration to use
Expand All @@ -32,7 +32,7 @@

# Create the SChunk
data = np.arange(200 * 1000 * nchunks)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, data=data, **storage)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, data=data, cparams=cparams, dparams=dparams, storage=storage)

# Check data can be retrieved correctly
data2 = np.empty(data.shape, dtype=data.dtype)
Expand Down
2 changes: 1 addition & 1 deletion examples/filler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
schunk_dtype = np.dtype(np.float64)

# Set the compression parameters. We need nthreads=1 for this example.
cparams = {"typesize": schunk_dtype.itemsize, "nthreads": 1}
cparams = blosc2.CParams(typesize=schunk_dtype.itemsize, nthreads=1)

# Create empty SChunk
schunk = blosc2.SChunk(chunksize=chunk_len * schunk_dtype.itemsize, cparams=cparams)
Expand Down
7 changes: 1 addition & 6 deletions examples/pack_tensor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,7 @@

a = np.arange(1_000_000)

cparams = {
"codec": blosc2.Codec.ZSTD,
"clevel": 9,
"filters": [blosc2.Filter.BITSHUFFLE],
"filters_meta": [0],
}
cparams = blosc2.CParams(codec=blosc2.Codec.ZSTD, clevel=9, filters=[blosc2.Filter.BITSHUFFLE], filters_meta=[0])
cframe = blosc2.pack_tensor(a, cparams=cparams)
print("Length of packed array in bytes:", len(cframe))

Expand Down
8 changes: 4 additions & 4 deletions examples/postfilter1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@
output_dtype = np.dtype(np.float32)

# Set the compression and decompression parameters
cparams = {"codec": blosc2.Codec.LZ4, "typesize": 4}
dparams = {"nthreads": 1}
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4, typesize=4)
dparams = blosc2.DParams(nthreads=1)
contiguous = True
urlpath = None
storage = {"contiguous": contiguous, "urlpath": urlpath, "cparams": cparams, "dparams": dparams}
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode='a')
# Remove previous SChunk
blosc2.remove_urlpath(urlpath)
# Create and set data
data = np.arange(200 * 1000 * nchunks, dtype=input_dtype)
schunk = blosc2.SChunk(chunksize=200 * 1000 * input_dtype.itemsize, data=data, **storage)
schunk = blosc2.SChunk(chunksize=200 * 1000 * input_dtype.itemsize, data=data, cparams=cparams, dparams=dparams, storage=storage)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Line is too long. This could be avoided by triggering ruff (a linter and formatter) on each commit. See the first paragraph in https://github.com/Blosc/python-blosc2/blob/main/README_DEVELOPERS.md


out1 = np.empty(200 * 1000 * nchunks, dtype=input_dtype)
schunk.get_slice(0, 200 * 1000 * nchunks, out=out1)
Expand Down
8 changes: 4 additions & 4 deletions examples/postfilter2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@
output_dtype = np.int64 # output dtype has to be of the same size as input

# Set the compression and decompression parameters
cparams = {"codec": blosc2.Codec.LZ4, "typesize": input_dtype.itemsize}
dparams = {"nthreads": 1}
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4, typesize=input_dtype.itemsize)
dparams = blosc2.DParams(nthreads=1)
contiguous = True
urlpath = "filename"
storage = {"contiguous": contiguous, "urlpath": urlpath, "cparams": cparams, "dparams": dparams}
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode='a')
# Remove previous SChunk
blosc2.remove_urlpath(urlpath)
# Create and set data
chunkshape = 200 * 1000
data = np.arange(0, chunkshape * nchunks, dtype=input_dtype)
schunk = blosc2.SChunk(chunksize=chunkshape * input_dtype.itemsize, data=data, **storage)
schunk = blosc2.SChunk(chunksize=chunkshape * input_dtype.itemsize, data=data, cparams=cparams, dparams=dparams, storage=storage)

out1 = np.empty(chunkshape * nchunks, dtype=input_dtype)
schunk.get_slice(0, chunkshape * nchunks, out=out1)
Expand Down
8 changes: 4 additions & 4 deletions examples/postfilter3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@
input_dtype = np.dtype(np.int64)

# Set the compression and decompression parameters
cparams = {"codec": blosc2.Codec.LZ4, "typesize": input_dtype.itemsize}
dparams = {"nthreads": 1}
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4, typesize=input_dtype.itemsize)
dparams = blosc2.DParams(nthreads=1)
contiguous = False
urlpath = None
storage = {"contiguous": contiguous, "urlpath": urlpath, "cparams": cparams, "dparams": dparams}
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode='a')
# Remove previous SChunk
blosc2.remove_urlpath(urlpath)
# Create and set data
chunkshape = 20_000
data = np.zeros(chunkshape * nchunks, dtype=input_dtype)
schunk = blosc2.SChunk(chunksize=chunkshape * input_dtype.itemsize, data=data, **storage)
schunk = blosc2.SChunk(chunksize=chunkshape * input_dtype.itemsize, data=data, cparams=cparams, dparams=dparams, storage=storage)

out1 = np.empty(chunkshape * nchunks, dtype=input_dtype)
schunk.get_slice(0, chunkshape * nchunks, out=out1)
Expand Down
8 changes: 4 additions & 4 deletions examples/prefilter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
output_dtype = np.dtype(np.float32)

# Set the compression and decompression parameters
cparams = {"typesize": 4, "nthreads": 1}
dparams = {"nthreads": 4}
storage = {"cparams": cparams, "dparams": dparams}
cparams = blosc2.CParams(typesize=4, nthreads=1)
dparams = blosc2.DParams(nthreads=4)
storage = blosc2.Storage(mode='a')
# Create empty schunk
schunk = blosc2.SChunk(chunksize=200 * 1000 * input_dtype.itemsize, **storage)
schunk = blosc2.SChunk(chunksize=200 * 1000 * input_dtype.itemsize, cparams=cparams, dparams=dparams, storage=storage)


# Set prefilter with decorator
Expand Down
10 changes: 5 additions & 5 deletions examples/schunk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@

nchunks = 10
# Set the compression and decompression parameters
cparams = {"codec": blosc2.Codec.LZ4HC, "typesize": 4}
dparams = {}
cparams = blosc2.CParams(codec=blosc2.Codec.LZ4HC, typesize=4)
dparams = blosc2.DParams()
contiguous = True
urlpath = "filename"

storage = {"contiguous": contiguous, "urlpath": urlpath, "cparams": cparams, "dparams": dparams}
storage = blosc2.Storage(contiguous=contiguous, urlpath=urlpath, mode='a')
blosc2.remove_urlpath(urlpath)
numpy_meta = {b"dtype": str(np.dtype("int32"))}
test_meta = {b"lorem": 1234}
meta = {"numpy": numpy_meta, "test": test_meta}

# Create the empty SChunk
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, meta=meta, **storage)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, meta=meta, cparams=cparams, dparams=dparams)
# Append some chunks
for i in range(nchunks):
buffer = i * np.arange(200 * 1000, dtype="int32")
Expand Down Expand Up @@ -54,7 +54,7 @@

# Update a chunk compressing the data first
buffer = 11 * np.arange(200 * 1000, dtype="int32")
chunk = blosc2.compress2(buffer, **cparams)
chunk = blosc2.compress2(buffer, cparams=cparams)
schunk.update_chunk(7, chunk)

# Delete the 4th chunk
Expand Down
Loading