Multi-GPU IVF PQ

1
cdef class Index

1
def trained(self)

1
cdef class IndexParams(SingleGpuIndexParams)

1
def __init__(self, *, distribution_mode="sharded", **kwargs)

1
def get_handle(self)

1
def distribution_mode(self)

1
cdef class SearchParams(SingleGpuSearchParams)

1
def __init__(self, *, n_probes=20, search_mode="load_balancer", merge_mode="merge_on_root_rank", n_rows_per_batch=1000, **kwargs)

1
def get_handle(self)

1
def search_mode(self)

1
def search_mode(self, value)

1
def merge_mode(self)

1
def merge_mode(self, value)

1
def n_rows_per_batch(self)

1
def n_rows_per_batch(self, value)

1
def build(IndexParams index_params, dataset, resources=None)

1
>>> import numpy as np
2
>>> from cuvs.neighbors.mg import ivf_pq
3
>>> n_samples = 50000
4
>>> n_features = 50
5
>>> n_queries = 1000
6
>>> k = 10
7
>>> # For multi-GPU IVF-PQ, use host (NumPy) arrays
8
>>> dataset = np.random.random_sample((n_samples, n_features)).astype(
9
...     np.float32)
10
>>> build_params = ivf_pq.IndexParams(metric="sqeuclidean")
11
>>> index = ivf_pq.build(build_params, dataset)
12
>>> distances, neighbors = ivf_pq.search(
13
...     ivf_pq.SearchParams(),
14
...     index, dataset, k)
15
>>> # Results are already in host memory (NumPy arrays)

1
def extend(Index index, new_vectors, new_indices=None, resources=None)

1
>>> import numpy as np
2
>>> from cuvs.neighbors.mg import ivf_pq
3
>>> n_samples = 50000
4
>>> n_features = 50
5
>>> n_new_vectors = 1000
6
>>> # For multi-GPU IVF-PQ, use host (NumPy) arrays
7
>>> dataset = np.random.random_sample((n_samples, n_features)).astype(
8
...     np.float32)
9
>>> new_vectors = np.random.random_sample(
10
...     (n_new_vectors, n_features)).astype(np.float32)
11
>>> new_indices = np.arange(n_samples, n_new_vectors, dtype=np.int64)
12
>>> build_params = ivf_pq.IndexParams(metric="sqeuclidean")
13
>>> index = ivf_pq.build(build_params, dataset)
14
>>> ivf_pq.extend(index, new_vectors, new_indices)

1
def search(SearchParams search_params, Index index, queries, k, neighbors=None, distances=None, resources=None)

1
>>> import numpy as np
2
>>> from cuvs.neighbors.mg import ivf_pq
3
>>> n_samples = 50000
4
>>> n_features = 50
5
>>> n_queries = 1000
6
>>> k = 10
7
>>> # For multi-GPU IVF-PQ, use host (NumPy) arrays
8
>>> dataset = np.random.random_sample((n_samples, n_features)).astype(
9
...     np.float32)
10
>>> queries = np.random.random_sample((n_queries, n_features)).astype(
11
...     np.float32)
12
>>> build_params = ivf_pq.IndexParams(metric="sqeuclidean")
13
>>> index = ivf_pq.build(build_params, dataset)
14
>>> distances, neighbors = ivf_pq.search(ivf_pq.SearchParams(),
15
...                                       index, queries, k)
16
>>> # Results are already in host memory (NumPy arrays)

1
def save(Index index, filename, resources=None)

1
>>> import numpy as np
2
>>> from cuvs.neighbors.mg import ivf_pq
3
>>> n_samples = 50000
4
>>> n_features = 50
5
>>> # For multi-GPU IVF-PQ, use host (NumPy) arrays
6
>>> dataset = np.random.random_sample((n_samples, n_features)).astype(
7
...     np.float32)
8
>>> build_params = ivf_pq.IndexParams(metric="sqeuclidean")
9
>>> index = ivf_pq.build(build_params, dataset)
10
>>> ivf_pq.save(index, "index.bin")

1
def load(filename, resources=None)

1
>>> from cuvs.neighbors.mg import ivf_pq
2
>>> index = ivf_pq.load("index.bin")  # doctest: +SKIP

1
def distribute(filename, resources=None)

1
>>> from cuvs.neighbors.mg import ivf_pq
2
>>> index = ivf_pq.distribute("single_gpu_index.bin")  # doctest: +SKIP