Is this page helpful?

nvCOMP Python API Basics#

[1]:

import numpy as np
import cupy as cp

Download example files

[2]:

import urllib.request

urllib.request.urlretrieve(
    "http://textfiles.com/etext/NONFICTION/locke-essay-113.txt", "locke-essay-113.txt"
)
urllib.request.urlretrieve(
    "http://textfiles.com/etext/FICTION/mobydick.txt", "mobydick.txt"
)

[2]:

('mobydick.txt', <http.client.HTTPMessage at 0x7bcdb829e3f0>)

Import nvComp python module and check versions#

[3]:

from nvidia import nvcomp

print("nvcomp version:", nvcomp.__version__)
print("nvcomp cuda version:", nvcomp.__cuda_version__)

nvcomp version: 6.0.0
nvcomp cuda version: 13040

Zero-copy import host array#

[4]:

ascending = np.arange(0, 4096, dtype=np.int32)
nvarr_h = nvcomp.as_array(ascending)

[5]:

print(ascending.__array_interface__)
print(nvarr_h.__array_interface__)
print(nvarr_h.__cuda_array_interface__)
print(nvarr_h.buffer_size)
print(nvarr_h.buffer_kind)
print(nvarr_h.ndim)
print(nvarr_h.dtype)
print(nvarr_h.shape)
print(nvarr_h.strides)
print(nvarr_h.item_size)
print(nvarr_h.size)

{'data': (336056080, False), 'strides': None, 'descr': [('', '<i4')], 'typestr': '<i4', 'shape': (4096,), 'version': 3}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (336056080, False), 'version': 3}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (336056080, False), 'version': 3, 'stream': 1}
16384
ArrayBufferKind.STRIDED_HOST
1
int32
(4096,)
(4,)
4
4096

Zero-copy import device array#

[6]:

data_gpu = cp.array(ascending)
nvarr_d = nvcomp.as_array(data_gpu)
print(data_gpu.__cuda_array_interface__)
print(nvarr_d.__cuda_array_interface__)
print(nvarr_d.buffer_kind)
print(nvarr_d.ndim)
print(nvarr_d.dtype)
print(nvarr_d.shape)
print(nvarr_d.strides)
print(nvarr_d.item_size)
print(nvarr_d.size)

{'shape': (4096,), 'typestr': '<i4', 'descr': [('', '<i4')], 'stream': 1, 'version': 3, 'strides': None, 'data': (136121881198592, False)}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (136121881198592, False), 'version': 3, 'stream': 1}
ArrayBufferKind.STRIDED_DEVICE
1
int32
(4096,)
(4,)
4
4096

Convert host array to device array#

[7]:

nvarr_d_cnv = nvarr_h.cuda()
print(nvarr_d_cnv.__cuda_array_interface__)

{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (12918456320, False), 'version': 3, 'stream': 1}

Convert device array to host array#

[8]:

nvarr_h_cnv = nvarr_d.cpu()
print(nvarr_h_cnv.__array_interface__)

{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (136121883312128, False), 'version': 3}

Encode single array#

Read text file

[9]:

with open("mobydick.txt", "rb") as f:
    text = f.read()

as_array supports python buffer protocol so we can pass text directly to it

[10]:

nvarr_txt_h = nvcomp.as_array(text)
print(nvarr_txt_h.__array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (352591296, False), 'version': 3}

Transfer to Device

[11]:

nvarr_txt_d = nvarr_txt_h.cuda()
print(nvarr_txt_d.__cuda_array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (12918473216, False), 'version': 3, 'stream': 1}

Create Codec

[12]:

lz4_codec = nvcomp.Codec(algorithm="LZ4")

Encode

[13]:

lz4_comp_arr = lz4_codec.encode(nvarr_txt_d)

[14]:

print(lz4_comp_arr.__cuda_array_interface__)
print(lz4_comp_arr.buffer_kind)

{'shape': (824829,), 'strides': None, 'typestr': '|i1', 'data': (12922738176, False), 'version': 3, 'stream': 352587808}
ArrayBufferKind.STRIDED_DEVICE

Array supports python buffer protocol so we can pass it to write function directly

[15]:

with open("mobydick.lz4", "wb") as f:
    f.write(lz4_comp_arr.cpu())

Passing directly objects with standard interfaces to encode function

[16]:

lz4_comp_arr = lz4_codec.encode(text)

Decode single array#

[17]:

nv_dec_array = lz4_codec.decode(lz4_comp_arr)

[18]:

print(nv_dec_array.__cuda_array_interface__)
print(nv_dec_array.buffer_kind)

{'shape': (1205404,), 'strides': None, 'typestr': '|i1', 'data': (12923563520, False), 'version': 3, 'stream': 352587808}
ArrayBufferKind.STRIDED_DEVICE

Compare decoded array with original and print first 400 positions of decoded array

[19]:

print("Is decoded equal to original?", bytes(nv_dec_array.cpu()) == bytes(nvarr_txt_h))
print(bytes(nv_dec_array.cpu())[:400].decode())

Is decoded equal to original? True
Preliminary Matter.

This text of Melville's Moby-Dick is based on the Hendricks House edition.
It was prepared by Professor Eugene F. Irey at the University of Colorado.
Any subsequent copies of this data must include this notice
and any publications resulting from analysis of this data must
include reference to Professor Irey's work.

Etymology  (Supplied by a late consumptive usher to a gra

Encode and decode with ANS codec, specified chunk size and checksum policy

[20]:

ans_codec = nvcomp.Codec(
    algorithm="ANS",
    uncomp_chunk_size=16,
    checksum_policy=nvcomp.ChecksumPolicy.COMPUTE_AND_VERIFY,
)
ans_comp_arr = ans_codec.encode(nvarr_d)

Define decode output type

[21]:

ans_deco_arr_int8 = ans_codec.decode(ans_comp_arr)
ans_deco_arr_uint32 = ans_codec.decode(ans_comp_arr, "<u4")

print(ans_deco_arr_int8.dtype)
print(ans_deco_arr_uint32.dtype)

int8
uint32

Codec specific options#

[22]:

gdeflate_ht_codec = nvcomp.Codec(algorithm="GDeflate", algorithm_type=0)
gdeflate_lt_codec = nvcomp.Codec(algorithm="Gdeflate", algorithm_type=1)

[23]:

%%timeit
gdeflate_ht_comp_arr = gdeflate_ht_codec.encode(text[:4096])

105 μs ± 1.96 μs per loop (mean ± std. dev. of 7 runs, 10,000 loops each)

[24]:

%%timeit
gdeflate_lt_comp_arr = gdeflate_lt_codec.encode(text[:4096])

593 μs ± 2.74 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

[25]:

gdeflate_ht_comp_arr = gdeflate_ht_codec.encode(text[:4096])
gdeflate_lt_comp_arr = gdeflate_lt_codec.encode(text[:4096])
print(
    "high-throughput, low compression ratio (default) - compressed size:",
    gdeflate_ht_comp_arr.size,
)
print(
    "low-throughput, high compression ratio - compressed size:",
    gdeflate_lt_comp_arr.size,
)

high-throughput, low compression ratio (default) - compressed size: 2656
low-throughput, high compression ratio - compressed size: 2512

Encode single array with multiple codecs#

[26]:

print("Uncompressed size is", nvarr_txt_d.buffer_size)
alogs = [
    "LZ4",
    "Snappy",
    "GDeflate",
    "Deflate",
    "Bitcomp",
    "ANS",
    "Zstd",
    "Cascaded",
    "Gzip",
]
encoded_files = []
for algorithm in alogs:
    codec = nvcomp.Codec(algorithm=algorithm)
    com_arr = codec.encode(nvarr_txt_d)
    print(
        "Compressed size for",
        algorithm,
        "is",
        com_arr.buffer_size,
        "({:.1%})".format(com_arr.buffer_size / nvarr_txt_d.buffer_size),
    )
    with open("mobydick.%s" % algorithm, "wb") as f:
        f.write(com_arr.cpu())
    encoded_files.append("mobydick.%s" % algorithm)

Uncompressed size is 1205404
Compressed size for LZ4 is 824829 (68.4%)
Compressed size for Snappy is 863143 (71.6%)
Compressed size for GDeflate is 622608 (51.7%)
Compressed size for Deflate is 619043 (51.4%)
Compressed size for Bitcomp is 986784 (81.9%)
Compressed size for ANS is 707120 (58.7%)
Compressed size for Zstd is 540779 (44.9%)
Compressed size for Cascaded is 1173008 (97.3%)
Compressed size for Gzip is 623541 (51.7%)

Decoding single arrays of various formats#

[27]:

codec = nvcomp.Codec()
for file_name in encoded_files:
    print(
        "Decoding",
        file_name,
    )
    with open(file_name, "rb") as f:
        comp_bytes = f.read()
    nv_dec_d = codec.decode(
        comp_bytes
    )  # since it supports buffer protocol we can pass comp_bytes directly
    print("is equal to original? -", bytes(nv_dec_d.cpu()) == bytes(nvarr_txt_h))

Decoding mobydick.LZ4
is equal to original? - True
Decoding mobydick.Snappy
is equal to original? - True
Decoding mobydick.GDeflate
is equal to original? - True
Decoding mobydick.Deflate
is equal to original? - True
Decoding mobydick.Bitcomp
is equal to original? - True
Decoding mobydick.ANS
is equal to original? - True
Decoding mobydick.Zstd
is equal to original? - True
Decoding mobydick.Cascaded
is equal to original? - True
Decoding mobydick.Gzip
is equal to original? - True

Encoding and decoding with various Bitstream Kinds#

[28]:

print("Uncompressed size is", nvarr_txt_d.buffer_size)
algos = ["LZ4", "Snappy", "Bitcomp", "ANS", "Zstd", "Cascaded", "Gzip"]
bitstreams = [
    nvcomp.BitstreamKind.NVCOMP_NATIVE,
    nvcomp.BitstreamKind.RAW,
    nvcomp.BitstreamKind.WITH_UNCOMPRESSED_SIZE,
]

for algorithm in algos:
    for bitstream_kind in bitstreams:
        codec = nvcomp.Codec(algorithm=algorithm, bitstream_kind=bitstream_kind)
        comp_arr = codec.encode(nvarr_txt_d)
        comp_ratio = comp_arr.buffer_size / nvarr_txt_d.buffer_size
        print(
            "Compressed size for",
            algorithm,
            "with bitstream",
            bitstream_kind,
            "is",
            comp_arr.buffer_size,
            "({:.1%})".format(comp_ratio),
        )
        decomp_array = codec.decode(comp_arr)
        print(
            "is equal to original? -",
            bytes(decomp_array.cpu()) == bytes(nvarr_txt_d.cpu()),
        )

Uncompressed size is 1205404
Compressed size for LZ4 with bitstream BitstreamKind.NVCOMP_NATIVE is 824829 (68.4%)
is equal to original? - True
Compressed size for LZ4 with bitstream BitstreamKind.RAW is 807075 (67.0%)
is equal to original? - True
Compressed size for LZ4 with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 807091 (67.0%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.NVCOMP_NATIVE is 863143 (71.6%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.RAW is 854105 (70.9%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 854121 (70.9%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.NVCOMP_NATIVE is 986784 (81.9%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.RAW is 985800 (81.8%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 985816 (81.8%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.NVCOMP_NATIVE is 707120 (58.7%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.RAW is 674554 (56.0%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 674570 (56.0%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.NVCOMP_NATIVE is 540779 (44.9%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.RAW is 527386 (43.8%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 527402 (43.8%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.NVCOMP_NATIVE is 1173008 (97.3%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.RAW is 1172464 (97.3%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 1172480 (97.3%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.NVCOMP_NATIVE is 623541 (51.7%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.RAW is 622854 (51.7%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 622870 (51.7%)
is equal to original? - True

Batch encoding#

[29]:

from pathlib import Path

nv_uncomp_arrays = nvcomp.as_arrays(
    [Path(fn).read_bytes() for fn in ["mobydick.txt", "locke-essay-113.txt"]]
)

[30]:

nv_comp_arrays = lz4_comp_arr = lz4_codec.encode(nv_uncomp_arrays)

[31]:

for i in range(len(nv_uncomp_arrays)):
    print(nv_uncomp_arrays[i].__cuda_array_interface__)
    print(nv_comp_arrays[i].__cuda_array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (410331136, False), 'version': 3, 'stream': 1}
{'shape': (824829,), 'strides': None, 'typestr': '|u1', 'data': (12953435648, False), 'version': 3, 'stream': 352587808}
{'shape': (1605768,), 'strides': None, 'typestr': '|u1', 'data': (411536592, False), 'version': 3, 'stream': 1}
{'shape': (978812,), 'strides': None, 'typestr': '|u1', 'data': (12954686464, False), 'version': 3, 'stream': 352587808}

Batch decoding#

[32]:

nv_dec_arrays = lz4_codec.decode(nv_comp_arrays)

Compare with original

[33]:

for i in range(len(nv_dec_arrays)):
    print(
        "Is decoded equal to original?",
        bytes(nv_uncomp_arrays[i].cpu()) == bytes(nv_dec_arrays[i].cpu()),
    )
    print("\n", bytes(nv_dec_arrays[i].cpu())[:400].decode())

Is decoded equal to original? True

 Preliminary Matter.

This text of Melville's Moby-Dick is based on the Hendricks House edition.
It was prepared by Professor Eugene F. Irey at the University of Colorado.
Any subsequent copies of this data must include this notice
and any publications resulting from analysis of this data must
include reference to Professor Irey's work.

Etymology  (Supplied by a late consumptive usher to a gra
Is decoded equal to original? True

                                       1690

                    AN ESSAY CONCERNING HUMAN UNDERSTANDING

                                 by John Locke

                       TO THE RIGHT HONOURABLE

            LORD THOMAS, EARL OF PEMBROKE AND MONTGOMERY,

                      BARRON HERBERT OF CARDIFF,

      LORD ROSS, OF KENDAL, PAR, FITZHUGH, MARMION, ST. QUINTIN,

          AND SHURLAND;

Batch encoding from one contiguous buffer#

as_arrays also accepts a single buffer sliced by an offset/size table, instead of a list of separate sources - useful when data is already packed into one allocation. Use Codec.get_required_compression_alignment() or Codec.get_required_decompression_alignment() to align the resulting buffer addresses for the operation that will consume them.

[34]:

compression_alignment = lz4_codec.get_required_compression_alignment()
chunk_stride = (
    (4096 + compression_alignment - 1) // compression_alignment
) * compression_alignment
chunk_a = b"Call me Ishmael."
chunk_b = b"An Essay Concerning Human Understanding"

packed = bytearray(chunk_stride * 2)
packed[0 : len(chunk_a)] = chunk_a
packed[chunk_stride : chunk_stride + len(chunk_b)] = chunk_b

offsets = [0, chunk_stride]
sizes = [len(chunk_a), len(chunk_b)]

nv_strided_arrays = nvcomp.as_arrays(bytes(packed), offsets, sizes)
print(bytes(nv_strided_arrays[0].cpu()))
print(bytes(nv_strided_arrays[1].cpu()))

b'Call me Ishmael.'
b'An Essay Concerning Human Understanding'

[35]:

nv_strided_comp = lz4_codec.encode(nv_strided_arrays)
nv_strided_dec = lz4_codec.decode(nv_strided_comp)
for i in range(len(nv_strided_dec)):
    print(
        "Is decoded equal to original?",
        bytes(nv_strided_dec[i].cpu()) == bytes(nv_strided_arrays[i].cpu()),
    )

Is decoded equal to original? True
Is decoded equal to original? True

Batch decoding and encoding various formats#

[36]:

algos = ["LZ4", "Snappy", "Bitcomp", "Cascaded", "Zstd", "ANS", "Gzip"]
bitstreams = [
    nvcomp.BitstreamKind.NVCOMP_NATIVE,
    nvcomp.BitstreamKind.RAW,
    nvcomp.BitstreamKind.WITH_UNCOMPRESSED_SIZE,
]
for algorithm in algos:
    for bitstream_kind in bitstreams:
        print("Algorithm:", algorithm, "BitstreamKind:", bitstream_kind)
        codec = nvcomp.Codec(algorithm=algorithm, bitstream_kind=bitstream_kind)
        nv_comp_arrays = codec.encode(nv_uncomp_arrays)
        nv_dec_arrays = codec.decode(nv_comp_arrays)
        for i in range(len(nv_dec_arrays)):
            print(" - File #", i)
            print("   -- Uncompressed size:", nv_uncomp_arrays[i].buffer_size)
            print(
                "   -- Compressed size:",
                nv_comp_arrays[i].buffer_size,
                "({:.1%})".format(
                    nv_comp_arrays[i].buffer_size / nv_uncomp_arrays[i].buffer_size
                ),
            )
            print(
                "   -- Is decoded equal to original?",
                bytes(nv_uncomp_arrays[i].cpu()) == bytes(nv_dec_arrays[i].cpu()),
            )

Algorithm: LZ4 BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 824829 (68.4%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 978812 (61.0%)
   -- Is decoded equal to original? True
Algorithm: LZ4 BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 807075 (67.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 964181 (60.0%)
   -- Is decoded equal to original? True
Algorithm: LZ4 BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 807091 (67.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 964197 (60.0%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 863143 (71.6%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 960225 (59.8%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 854105 (70.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 945120 (58.9%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 854121 (70.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 945136 (58.9%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 986784 (81.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1300348 (81.0%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 985800 (81.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1299060 (80.9%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 985816 (81.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1299076 (80.9%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1173008 (97.3%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1562572 (97.3%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1172464 (97.3%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1561884 (97.3%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1172480 (97.3%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1561900 (97.3%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 540779 (44.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 613539 (38.2%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 527386 (43.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 596140 (37.1%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 527402 (43.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 596156 (37.1%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 707120 (58.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 914728 (57.0%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 674554 (56.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 871066 (54.2%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 674570 (56.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 871082 (54.2%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 623541 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 753195 (46.9%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 622854 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 752324 (46.9%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 622870 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 752340 (46.9%)
   -- Is decoded equal to original? True

Checksum example#

Checksums can be used only with nvcomp.BitstreamKind.NVCOMP_NATIVE bitstream kind. They are computed before the data is compressed and after it is decompressed and validates that the decompressed data is equal to original.

Checksum can be used to prevent silent corruption, which can happen when data is corrupted but the decoding finishes without errors or when there is a bug in encoding or decoding implementation

Following example will show how to use them. Create codec which computes and verifies checksums:

[37]:

codec = nvcomp.Codec(
    algorithm="GDeflate",
    bitstream_kind=nvcomp.BitstreamKind.NVCOMP_NATIVE,
    checksum_policy=nvcomp.ChecksumPolicy.COMPUTE_AND_VERIFY,
)
nv_comp_arrays = codec.encode(nv_uncomp_arrays)

Verify that there are no errors when data is correct:

[38]:

nv_dec_arrays = codec.decode(nv_comp_arrays)
for i in range(len(nv_dec_arrays)):
    print(
        f"Is array {i} equal to original? -",
        bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
    )

Is array 0 equal to original? - True
Is array 1 equal to original? - True

Introduce artificial error in data and decode

[39]:

array_with_error = 1

# BatchArray entries are views into the batch, so corrupt a detached copy and
# build a new batch from it.
corrupted = [cp.asarray(nv_comp_arrays[i]).copy() for i in range(len(nv_comp_arrays))]
corrupted[array_with_error].view(cp.uint8)[1000] ^= cp.uint8(176)

nv_dec_arrays = codec.decode(nvcomp.as_arrays(corrupted))

Check which arrays were affected in decoding (should be only one). Checksums are only validated during the first access to the data, any following access will skip that check.

[40]:

for i in range(len(nv_dec_arrays)):
    try:
        print(
            f"Is array {i} equal to original? -",
            bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
        )
    except RuntimeError as err:
        print(f"error with decoding array {i}: {err}")
        print(
            f"Rerunning: Is array {i} equal to original? -",
            bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
        )

error with decoding array 0: Checksum doesn't match.
Rerunning: Is array 0 equal to original? - True
Is array 1 equal to original? - False

Pre-configure API#

compression_config and decompression_config separate the configuration step from encode/decode. This enables:

  • Sync-free decode when compressing and decompressing in the same process; no stream synchronization at decode time

  • Config reuse across multiple calls with identically-shaped inputs, avoiding per-call setup overhead

[41]:

codec = nvcomp.Codec(algorithm="LZ4")
# compression_config accepts the uncompressed size in bytes - no input array needed
comp_cfg = codec.compression_config(nvarr_txt_d.buffer_size)
compressed = codec.encode(nvarr_txt_d, compression_config=comp_cfg)
# Deriving decomp config from comp_cfg is fully sync-free: no header parse at decode time
decomp_cfg = codec.decompression_config(comp_cfg)
decoded = codec.decode(compressed, decompression_config=decomp_cfg)
print("Compressed:", compressed.buffer_size, "bytes")
print("Roundtrip OK:", bytes(decoded.cpu()) == bytes(nvarr_txt_d.cpu()))

Compressed: 824829 bytes
Roundtrip OK: True

[42]:

# Configs are shape-bound: any input of the same size can reuse them
n = 65536
codec = nvcomp.Codec(algorithm="Zstd")
comp_cfg = codec.compression_config(n)
decomp_cfg = codec.decompression_config(comp_cfg)
for label, data in [
    ("zeros ", np.zeros(n, dtype=np.uint8)),
    ("arange", np.arange(n, dtype=np.uint8)),
]:
    nv_d = nvcomp.as_array(data).cuda()
    compressed = codec.encode(nv_d, compression_config=comp_cfg)
    decoded = codec.decode(compressed, decompression_config=decomp_cfg)
    print(
        f"{label}: compressed {compressed.buffer_size:6d} bytes, roundtrip OK = {bytes(decoded.cpu()) == data.tobytes()}"
    )

zeros : compressed    100 bytes, roundtrip OK = True
arange: compressed    358 bytes, roundtrip OK = True

Pre-allocated output buffers#

By default, encode and decode allocate their output buffers internally. Passing a pre-allocated CuPy array via out= lets you control exactly where the output lands, which is useful for pipelines with a fixed memory pool or when you want zero additional allocations.

Use codec.get_max_comp_buffer_size(src) to query the worst-case compressed size and pre-allocate a CuPy buffer of that capacity.

[43]:

codec = nvcomp.Codec(algorithm="LZ4")
# Pre-allocate compression output - get_max_comp_buffer_size returns an upper bound
max_comp_size = codec.get_max_comp_buffer_size(nvarr_txt_d)
enc_buf = cp.zeros(max_comp_size, dtype=cp.uint8)
# encode writes into enc_buf; the returned array shares the same device pointer
compressed = codec.encode(nvarr_txt_d, out=nvcomp.as_array(enc_buf))
print(f"Compressed: {compressed.buffer_size} / {enc_buf.nbytes} bytes capacity")
# Pre-allocate decompression output - uncompressed size is known upfront
dec_buf = cp.zeros(nvarr_txt_d.buffer_size, dtype=cp.uint8)
decoded = codec.decode(compressed, out=nvcomp.as_array(dec_buf))
print("Roundtrip OK:", bytes(decoded.cpu()) == bytes(nvarr_txt_d.cpu()))

Compressed: 824829 / 1250576 bytes capacity
Roundtrip OK: True

[44]:

# Batch encode/decode with pre-allocated CuPy output buffers
codec = nvcomp.Codec(algorithm="LZ4")
enc_bufs = [
    cp.zeros(codec.get_max_comp_buffer_size(src), dtype=cp.uint8)
    for src in nv_uncomp_arrays
]
compressed_batch = codec.encode(nv_uncomp_arrays, out=nvcomp.as_arrays(enc_bufs))
dec_bufs = [cp.zeros(src.buffer_size, dtype=cp.uint8) for src in nv_uncomp_arrays]
decoded_batch = codec.decode(compressed_batch, out=nvcomp.as_arrays(dec_bufs))
for i, (dec, src) in enumerate(zip(decoded_batch, nv_uncomp_arrays)):
    print(
        f"File #{i}: compressed {compressed_batch[i].buffer_size} bytes, "
        f"roundtrip OK = {bytes(dec.cpu()) == bytes(src.cpu())}"
    )

File #0: compressed 824829 bytes, roundtrip OK = True
File #1: compressed 978812 bytes, roundtrip OK = True