nvCOMP Python API Basics

import numpy as np
import cupy as cp

Download example files

import urllib.request

urllib.request.urlretrieve(
    "http://textfiles.com/etext/NONFICTION/locke-essay-113.txt", "locke-essay-113.txt"
)
urllib.request.urlretrieve(
    "http://textfiles.com/etext/FICTION/mobydick.txt", "mobydick.txt"
)

('mobydick.txt', <http.client.HTTPMessage at 0x7f99a8e287d0>)

Import nvComp python module and check versions

from nvidia import nvcomp

print("nvcomp version:", nvcomp.__version__)
print("nvcomp cuda version:", nvcomp.__cuda_version__)

nvcomp version: 5.3.0
nvcomp cuda version: 12090

Zero-copy import host array

ascending = np.arange(0, 4096, dtype=np.int32)
nvarr_h = nvcomp.as_array(ascending)

print(ascending.__array_interface__)
print(nvarr_h.__array_interface__)
print(nvarr_h.__cuda_array_interface__)
print(nvarr_h.buffer_size)
print(nvarr_h.buffer_kind)
print(nvarr_h.ndim)
print(nvarr_h.dtype)
print(nvarr_h.shape)
print(nvarr_h.strides)
print(nvarr_h.item_size)
print(nvarr_h.size)

{'data': (55462976, False), 'strides': None, 'descr': [('', '<i4')], 'typestr': '<i4', 'shape': (4096,), 'version': 3}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (55462976, False), 'version': 3}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (55462976, False), 'version': 3, 'stream': 1}
16384
ArrayBufferKind.STRIDED_HOST
1
int32
(4096,)
(4,)
4
4096

Zero-copy import device array

data_gpu = cp.array(ascending)
nvarr_d = nvcomp.as_array(data_gpu)
print(data_gpu.__cuda_array_interface__)
print(nvarr_d.__cuda_array_interface__)
print(nvarr_d.buffer_kind)
print(nvarr_d.ndim)
print(nvarr_d.dtype)
print(nvarr_d.shape)
print(nvarr_d.strides)
print(nvarr_d.item_size)
print(nvarr_d.size)

{'shape': (4096,), 'typestr': '<i4', 'descr': [('', '<i4')], 'stream': 1, 'version': 3, 'strides': None, 'data': (140295897350144, False)}
{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (140295897350144, False), 'version': 3, 'stream': 1}
ArrayBufferKind.STRIDED_DEVICE
1
int32
(4096,)
(4,)
4
4096

Convert host array to device array

nvarr_d_cnv = nvarr_h.cuda()
print(nvarr_d_cnv.__cuda_array_interface__)

{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (13421772800, False), 'version': 3, 'stream': 1}

Convert device array to host array

nvarr_h_cnv = nvarr_d.cpu()
print(nvarr_h_cnv.__array_interface__)

{'shape': (4096,), 'strides': None, 'typestr': '<i4', 'data': (140295899463680, False), 'version': 3}

Encode single array

Read text file

with open("mobydick.txt", "rb") as f:
    text = f.read()

as_array supports python buffer protocol so we can pass text directly to it

nvarr_txt_h = nvcomp.as_array(text)
print(nvarr_txt_h.__array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (72943744, False), 'version': 3}

Transfer to Device

nvarr_txt_d = nvarr_txt_h.cuda()
print(nvarr_txt_d.__cuda_array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (13421789184, False), 'version': 3, 'stream': 1}

Create Codec

lz4_codec = nvcomp.Codec(algorithm="LZ4")

Encode

lz4_comp_arr = lz4_codec.encode(nvarr_txt_d)

print(lz4_comp_arr.__cuda_array_interface__)
print(lz4_comp_arr.buffer_kind)

{'shape': (824829,), 'strides': None, 'typestr': '|i1', 'data': (13426054144, False), 'version': 3, 'stream': 72940528}
ArrayBufferKind.STRIDED_DEVICE

Array supports python buffer protocol so we can pass it to write function directly

with open("mobydick.lz4", "wb") as f:
    f.write(lz4_comp_arr.cpu())

Passing directly objects with standard interfaces to encode function

lz4_comp_arr = lz4_codec.encode(text)

Decode single array

nv_dec_array = lz4_codec.decode(lz4_comp_arr)

print(nv_dec_array.__cuda_array_interface__)
print(nv_dec_array.buffer_kind)

{'shape': (1205404,), 'strides': None, 'typestr': '|i1', 'data': (13426878976, False), 'version': 3, 'stream': 72940528}
ArrayBufferKind.STRIDED_DEVICE

Compare decoded array with original and print first 400 positions of decoded array

print("Is decoded equal to original?", bytes(nv_dec_array.cpu()) == bytes(nvarr_txt_h))
print(bytes(nv_dec_array.cpu())[:400].decode())

Is decoded equal to original? True
Preliminary Matter.

This text of Melville's Moby-Dick is based on the Hendricks House edition.
It was prepared by Professor Eugene F. Irey at the University of Colorado.
Any subsequent copies of this data must include this notice
and any publications resulting from analysis of this data must
include reference to Professor Irey's work.

Etymology  (Supplied by a late consumptive usher to a gra

Encode and decode with ANS codec, specified chunk size and checksum policy

ans_codec = nvcomp.Codec(
    algorithm="ANS",
    uncomp_chunk_size=20,
    checksum_policy=nvcomp.ChecksumPolicy.COMPUTE_AND_VERIFY,
)
ans_comp_arr = ans_codec.encode(nvarr_d)

Define decode output type

ans_deco_arr_int8 = ans_codec.decode(ans_comp_arr)
ans_deco_arr_uint32 = ans_codec.decode(ans_comp_arr, "<u4")

print(ans_deco_arr_int8.dtype)
print(ans_deco_arr_uint32.dtype)

int8
uint32

Codec specific options

gdeflate_ht_codec = nvcomp.Codec(algorithm="GDeflate", algorithm_type=0)
gdeflate_lt_codec = nvcomp.Codec(algorithm="Gdeflate", algorithm_type=1)

%%timeit
gdeflate_ht_comp_arr = gdeflate_ht_codec.encode(text[:4096])

165 μs ± 887 ns per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

%%timeit
gdeflate_lt_comp_arr = gdeflate_lt_codec.encode(text[:4096])

915 μs ± 17.4 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

gdeflate_ht_comp_arr = gdeflate_ht_codec.encode(text[:4096])
gdeflate_lt_comp_arr = gdeflate_lt_codec.encode(text[:4096])
print(
    "high-throughput, low compression ratio (default) - compressed size:",
    gdeflate_ht_comp_arr.size,
)
print(
    "low-throughput, high compression ratio - compressed size:",
    gdeflate_lt_comp_arr.size,
)

high-throughput, low compression ratio (default) - compressed size: 2664
low-throughput, high compression ratio - compressed size: 2520

Encode single array with multiple codecs

print("Uncompressed size is", nvarr_txt_d.buffer_size)
alogs = [
    "LZ4",
    "Snappy",
    "GDeflate",
    "Deflate",
    "Bitcomp",
    "ANS",
    "Zstd",
    "Cascaded",
    "Gzip",
]
encoded_files = []
for algorithm in alogs:
    codec = nvcomp.Codec(algorithm=algorithm)
    com_arr = codec.encode(nvarr_txt_d)
    print(
        "Compressed size for",
        algorithm,
        "is",
        com_arr.buffer_size,
        "({:.1%})".format(com_arr.buffer_size / nvarr_txt_d.buffer_size),
    )
    with open("mobydick.%s" % algorithm, "wb") as f:
        f.write(com_arr.cpu())
    encoded_files.append("mobydick.%s" % algorithm)

Uncompressed size is 1205404
Compressed size for LZ4 is 824829 (68.4%)
Compressed size for Snappy is 863151 (71.6%)
Compressed size for GDeflate is 622616 (51.7%)
Compressed size for Deflate is 619051 (51.4%)
Compressed size for Bitcomp is 986776 (81.9%)
Compressed size for ANS is 735980 (61.1%)
Compressed size for Zstd is 540787 (44.9%)
Compressed size for Cascaded is 1205948 (100.0%)
Compressed size for Gzip is 623549 (51.7%)

Decoding single arrays of various formats

codec = nvcomp.Codec()
for file_name in encoded_files:
    print(
        "Decoding",
        file_name,
    )
    with open(file_name, "rb") as f:
        comp_bytes = f.read()
    nv_dec_d = codec.decode(
        comp_bytes
    )  # since it supports buffer protocol we can pass comp_bytes directly
    print("is equal to original? -", bytes(nv_dec_d.cpu()) == bytes(nvarr_txt_h))

Decoding mobydick.LZ4
is equal to original? - True
Decoding mobydick.Snappy
is equal to original? - True
Decoding mobydick.GDeflate
is equal to original? - True
Decoding mobydick.Deflate
is equal to original? - True
Decoding mobydick.Bitcomp
is equal to original? - True
Decoding mobydick.ANS
is equal to original? - True
Decoding mobydick.Zstd
is equal to original? - True
Decoding mobydick.Cascaded
is equal to original? - True
Decoding mobydick.Gzip
is equal to original? - True

Encoding and decoding with various Bitstream Kinds

print("Uncompressed size is", nvarr_txt_d.buffer_size)
algos = ["LZ4", "Snappy", "Bitcomp", "ANS", "Zstd", "Cascaded", "Gzip"]
bitstreams = [
    nvcomp.BitstreamKind.NVCOMP_NATIVE,
    nvcomp.BitstreamKind.RAW,
    nvcomp.BitstreamKind.WITH_UNCOMPRESSED_SIZE,
]

for algorithm in algos:
    for bitstream_kind in bitstreams:
        codec = nvcomp.Codec(algorithm=algorithm, bitstream_kind=bitstream_kind)
        comp_arr = codec.encode(nvarr_txt_d)
        comp_ratio = comp_arr.buffer_size / nvarr_txt_d.buffer_size
        print(
            "Compressed size for",
            algorithm,
            "with bitstream",
            bitstream_kind,
            "is",
            comp_arr.buffer_size,
            "({:.1%})".format(comp_ratio),
        )
        decomp_array = codec.decode(comp_arr)
        print(
            "is equal to original? -",
            bytes(decomp_array.cpu()) == bytes(nvarr_txt_d.cpu()),
        )

Uncompressed size is 1205404
Compressed size for LZ4 with bitstream BitstreamKind.NVCOMP_NATIVE is 824829 (68.4%)
is equal to original? - True
Compressed size for LZ4 with bitstream BitstreamKind.RAW is 807075 (67.0%)
is equal to original? - True
Compressed size for LZ4 with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 807079 (67.0%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.NVCOMP_NATIVE is 863151 (71.6%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.RAW is 854105 (70.9%)
is equal to original? - True
Compressed size for Snappy with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 854113 (70.9%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.NVCOMP_NATIVE is 986776 (81.9%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.RAW is 985800 (81.8%)
is equal to original? - True
Compressed size for Bitcomp with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 985808 (81.8%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.NVCOMP_NATIVE is 735980 (61.1%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.RAW is 680000 (56.4%)
is equal to original? - True
Compressed size for ANS with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 680008 (56.4%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.NVCOMP_NATIVE is 540787 (44.9%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.RAW is 527386 (43.8%)
is equal to original? - True
Compressed size for Zstd with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 527394 (43.8%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.NVCOMP_NATIVE is 1205948 (100.0%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.RAW is 1205412 (100.0%)
is equal to original? - True
Compressed size for Cascaded with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 1205420 (100.0%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.NVCOMP_NATIVE is 623549 (51.7%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.RAW is 622854 (51.7%)
is equal to original? - True
Compressed size for Gzip with bitstream BitstreamKind.WITH_UNCOMPRESSED_SIZE is 622862 (51.7%)
is equal to original? - True

Batch encoding

nv_uncomp_arrays = []
for fn in ["mobydick.txt", "locke-essay-113.txt"]:
    with open(fn, "rb") as f:
        text = f.read()
    nv_uncomp_arrays.append(nvcomp.as_array(text).cuda())

nv_comp_arrays = lz4_comp_arr = lz4_codec.encode(nv_uncomp_arrays)

for i in range(len(nv_uncomp_arrays)):
    print(nv_uncomp_arrays[i].__cuda_array_interface__)
    print(nv_comp_arrays[i].__cuda_array_interface__)

{'shape': (1205404,), 'strides': None, 'typestr': '|u1', 'data': (13450522112, False), 'version': 3, 'stream': 1}
{'shape': (824829,), 'strides': None, 'typestr': '|i1', 'data': (13441342976, False), 'version': 3, 'stream': 72940528}
{'shape': (1605768,), 'strides': None, 'typestr': '|u1', 'data': (13451727872, False), 'version': 3, 'stream': 1}
{'shape': (978812,), 'strides': None, 'typestr': '|i1', 'data': (13453334016, False), 'version': 3, 'stream': 72940528}

Batch decoding

nv_dec_arrays = lz4_codec.decode(nv_comp_arrays)

Compare with original

for i in range(len(nv_dec_arrays)):
    print(
        "Is decoded equal to original?",
        bytes(nv_uncomp_arrays[i].cpu()) == bytes(nv_dec_arrays[i].cpu()),
    )
    print("\n", bytes(nv_dec_arrays[i].cpu())[:400].decode())

Is decoded equal to original? True

 Preliminary Matter.

This text of Melville's Moby-Dick is based on the Hendricks House edition.
It was prepared by Professor Eugene F. Irey at the University of Colorado.
Any subsequent copies of this data must include this notice
and any publications resulting from analysis of this data must
include reference to Professor Irey's work.

Etymology  (Supplied by a late consumptive usher to a gra
Is decoded equal to original? True

                                       1690

                    AN ESSAY CONCERNING HUMAN UNDERSTANDING

                                 by John Locke

                       TO THE RIGHT HONOURABLE

            LORD THOMAS, EARL OF PEMBROKE AND MONTGOMERY,

                      BARRON HERBERT OF CARDIFF,

      LORD ROSS, OF KENDAL, PAR, FITZHUGH, MARMION, ST. QUINTIN,

          AND SHURLAND;

Batch decoding and encoding various formats

algos = ["LZ4", "Snappy", "Bitcomp", "Cascaded", "Zstd", "ANS", "Gzip"]
bitstreams = [
    nvcomp.BitstreamKind.NVCOMP_NATIVE,
    nvcomp.BitstreamKind.RAW,
    nvcomp.BitstreamKind.WITH_UNCOMPRESSED_SIZE,
]
for algorithm in algos:
    for bitstream_kind in bitstreams:
        print("Algorithm:", algorithm, "BitstreamKind:", bitstream_kind)
        codec = nvcomp.Codec(algorithm=algorithm, bitstream_kind=bitstream_kind)
        nv_comp_arrays = codec.encode(nv_uncomp_arrays)
        nv_dec_arrays = codec.decode(nv_comp_arrays)
        for i in range(len(nv_dec_arrays)):
            print(" - File #", i)
            print("   -- Uncompressed size:", nv_uncomp_arrays[i].buffer_size)
            print(
                "   -- Compressed size:",
                nv_comp_arrays[i].buffer_size,
                "({:.1%})".format(
                    nv_comp_arrays[i].buffer_size / nv_uncomp_arrays[i].buffer_size
                ),
            )
            print(
                "   -- Is decoded equal to original?",
                bytes(nv_uncomp_arrays[i].cpu()) == bytes(nv_dec_arrays[i].cpu()),
            )

Algorithm: LZ4 BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 824829 (68.4%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 978812 (61.0%)
   -- Is decoded equal to original? True
Algorithm: LZ4 BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 807075 (67.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 964181 (60.0%)
   -- Is decoded equal to original? True
Algorithm: LZ4 BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 807079 (67.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 964185 (60.0%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 863151 (71.6%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 960233 (59.8%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 854105 (70.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 945120 (58.9%)
   -- Is decoded equal to original? True
Algorithm: Snappy BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 854113 (70.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 945128 (58.9%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 986776 (81.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1300340 (81.0%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 985800 (81.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1299060 (80.9%)
   -- Is decoded equal to original? True
Algorithm: Bitcomp BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 985808 (81.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1299068 (80.9%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1205948 (100.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1606456 (100.0%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1205412 (100.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1605776 (100.0%)
   -- Is decoded equal to original? True
Algorithm: Cascaded BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 1205420 (100.0%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 1605784 (100.0%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 540787 (44.9%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 613547 (38.2%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 527386 (43.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 596140 (37.1%)
   -- Is decoded equal to original? True
Algorithm: Zstd BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 527394 (43.8%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 596148 (37.1%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 735980 (61.1%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 954784 (59.5%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 680000 (56.4%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 879632 (54.8%)
   -- Is decoded equal to original? True
Algorithm: ANS BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 680008 (56.4%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 879640 (54.8%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.NVCOMP_NATIVE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 623549 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 753203 (46.9%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.RAW
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 622854 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 752324 (46.9%)
   -- Is decoded equal to original? True
Algorithm: Gzip BitstreamKind: BitstreamKind.WITH_UNCOMPRESSED_SIZE
 - File # 0
   -- Uncompressed size: 1205404
   -- Compressed size: 622862 (51.7%)
   -- Is decoded equal to original? True
 - File # 1
   -- Uncompressed size: 1605768
   -- Compressed size: 752332 (46.9%)
   -- Is decoded equal to original? True

Checksum example

Checksums can be used only with nvcomp.BitstreamKind.NVCOMP_NATIVE bitstream kind. They are computed before the data is compressed and after it is decompressed and validates that the decompressed data is equal to original.

Checksum can be used to prevent silent corruption, which can happen when data is corrupted but the decoding finishes without errors or when there is a bug in encoding or decoding implementation

Following example will show how to use them. Create codec which computes and verifies checksums:

codec = nvcomp.Codec(
    algorithm="GDeflate",
    bitstream_kind=nvcomp.BitstreamKind.NVCOMP_NATIVE,
    checksum_policy=nvcomp.ChecksumPolicy.COMPUTE_AND_VERIFY,
)
nv_comp_arrays = codec.encode(nv_uncomp_arrays)

Verify that there are no errors when data is correct:

nv_dec_arrays = codec.decode(nv_comp_arrays)
for i in range(len(nv_dec_arrays)):
    print(
        f"Is array {i} equal to original? -",
        bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
    )

Is array 0 equal to original? - True
Is array 1 equal to original? - True

Introduce artificial error in data and decode

array_with_error = 1

cupy_array = cp.asarray(nv_comp_arrays[array_with_error])
cupy_array.view(cp.uint8)[1000] ^= cp.uint8(176)

nv_comp_arrays[array_with_error] = nvcomp.as_array(cupy_array)

nv_dec_arrays = codec.decode(nv_comp_arrays)

Check which arrays were affected in decoding (should be only one). Checksums are only validated during the first access to the data, any following access will skip that check.

for i in range(len(nv_dec_arrays)):
    try:
        print(
            f"Is array {i} equal to original? -",
            bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
        )
    except RuntimeError as err:
        print(f"error with decoding array {i}: {err}")
        print(
            f"Rerunning: Is array {i} equal to original? -",
            bytes(nv_dec_arrays[i].cpu()) == bytes(nv_uncomp_arrays[i].cpu()),
        )

Is array 0 equal to original? - True
error with decoding array 1: Checksum doesn't match.
Rerunning: Is array 1 equal to original? - False

Pre-configure API

compression_config and decompression_config separate the configuration step from encode/decode. This enables:

• Sync-free decode when compressing and decompressing in the same process; no stream synchronization at decode time

• Config reuse across multiple calls with identically-shaped inputs, avoiding per-call setup overhead

codec = nvcomp.Codec(algorithm="LZ4")
# compression_config accepts the uncompressed size in bytes - no input array needed
comp_cfg = codec.compression_config(nvarr_txt_d.buffer_size)
compressed = codec.encode(nvarr_txt_d, compression_config=comp_cfg)
# Deriving decomp config from comp_cfg is fully sync-free: no header parse at decode time
decomp_cfg = codec.decompression_config(comp_cfg)
decoded = codec.decode(compressed, decompression_config=decomp_cfg)
print("Compressed:", compressed.buffer_size, "bytes")
print("Roundtrip OK:", bytes(decoded.cpu()) == bytes(nvarr_txt_d.cpu()))

Compressed: 824829 bytes
Roundtrip OK: True

# Configs are shape-bound: any input of the same size can reuse them
n = 65536
codec = nvcomp.Codec(algorithm="Zstd")
comp_cfg = codec.compression_config(n)
decomp_cfg = codec.decompression_config(comp_cfg)
for label, data in [
    ("zeros ", np.zeros(n, dtype=np.uint8)),
    ("arange", np.arange(n, dtype=np.uint8)),
]:
    nv_d = nvcomp.as_array(data).cuda()
    compressed = codec.encode(nv_d, compression_config=comp_cfg)
    decoded = codec.decode(compressed, decompression_config=decomp_cfg)
    print(
        f"{label}: compressed {compressed.buffer_size:6d} bytes, roundtrip OK = {bytes(decoded.cpu()) == data.tobytes()}"
    )

zeros : compressed    108 bytes, roundtrip OK = True
arange: compressed    366 bytes, roundtrip OK = True

Pre-allocated output buffers

By default, encode and decode allocate their output buffers internally. Passing a pre-allocated CuPy array via out= lets you control exactly where the output lands, which is useful for pipelines with a fixed memory pool or when you want zero additional allocations.

Use codec.get_max_comp_buffer_size(src) to query the worst-case compressed size and pre-allocate a CuPy buffer of that capacity.

codec = nvcomp.Codec(algorithm="LZ4")
# Pre-allocate compression output - get_max_comp_buffer_size returns an upper bound
max_comp_size = codec.get_max_comp_buffer_size(nvarr_txt_d)
enc_buf = cp.zeros(max_comp_size, dtype=cp.uint8)
# encode writes into enc_buf; the returned array shares the same device pointer
compressed = codec.encode(nvarr_txt_d, out=nvcomp.as_array(enc_buf))
print(f"Compressed: {compressed.buffer_size} / {enc_buf.nbytes} bytes capacity")
# Pre-allocate decompression output - uncompressed size is known upfront
dec_buf = cp.zeros(nvarr_txt_d.buffer_size, dtype=cp.uint8)
decoded = codec.decode(compressed, out=nvcomp.as_array(dec_buf))
print("Roundtrip OK:", bytes(decoded.cpu()) == bytes(nvarr_txt_d.cpu()))

Compressed: 824829 / 1250576 bytes capacity
Roundtrip OK: True

# Batch encode/decode with pre-allocated CuPy output buffers
codec = nvcomp.Codec(algorithm="LZ4")
enc_bufs = [
    cp.zeros(codec.get_max_comp_buffer_size(src), dtype=cp.uint8)
    for src in nv_uncomp_arrays
]
compressed_batch = codec.encode(
    nv_uncomp_arrays, out=[nvcomp.as_array(b) for b in enc_bufs]
)
dec_bufs = [cp.zeros(src.buffer_size, dtype=cp.uint8) for src in nv_uncomp_arrays]
decoded_batch = codec.decode(
    compressed_batch, out=[nvcomp.as_array(b) for b in dec_bufs]
)
for i, (dec, src) in enumerate(zip(decoded_batch, nv_uncomp_arrays)):
    print(
        f"File #{i}: compressed {compressed_batch[i].buffer_size} bytes, "
        f"roundtrip OK = {bytes(dec.cpu()) == bytes(src.cpu())}"
    )

File #0: compressed 824829 bytes, roundtrip OK = True
File #1: compressed 978812 bytes, roundtrip OK = True