ai4med.libs.data package
-
class
Balancer(items, categorizer: ai4med.libs.data.balancer.Categorizer, items_per_category='max') Bases:
object-
balance()
-
-
class
Categorizer Bases:
abc.ABC-
abstract
get_category_id(item)
-
abstract
get_category_weight_factor(category_id)
-
abstract
-
class
Cacher Bases:
object-
contains(cache_id)
-
get(cache_id)
-
get_cache_ids()
-
initialize(cache_id)
-
put(cache_id, data)
-
replace(old_id, new_id, new_data)
-
saves_data()
-
-
class
NullCacher Bases:
ai4med.libs.data.cacher.Cacher-
get(cache_id)
-
put(cache_id, data)
-
replace(old_id, new_id, new_data)
-
saves_data()
-
-
class
MultiLabelCategorizer Bases:
ai4med.libs.data.balancer.Categorizer-
get_category_id(item)
-
get_category_weight_factor(category_id)
-
-
class
SingleTaskCategorizer(weights, default_weight=1) Bases:
ai4med.libs.data.balancer.Categorizer-
get_category_id(item)
-
get_category_weight_factor(category_id)
-
-
class
ChainTransformer(build_ctx: ai4med.common.build_ctx.BuildContext) Bases:
object-
get_stats()
-
transform(transforms, x)
-
-
class
DatalistManager(list_gen: ai4med.libs.data.list_gen.ListGenerator, transforms, build_ctx: ai4med.common.build_ctx.BuildContext) Bases:
object-
get_dataset_size()
-
get_list_generator()
-
initialize(state=- 1)
-
set_sharding(rank, num_shards, equal_shard_size=True, fixed_shard_data=False)
-
shutdown()
-
transform(transform_ctx: ai4med.common.transform_ctx.TransformContext)
-
-
class
DatalistManagerWithCache(list_gen: ai4med.libs.data.list_gen.ListGenerator, transforms, build_ctx: ai4med.common.build_ctx.BuildContext, cache_obj_count, replace_percent=0.1, caches_data=True) Bases:
ai4med.libs.data.datalist_manager.DatalistManager-
initialize(state=- 1)
-
set_sharding(rank, num_shards, equal_shard_size=True, fixed_shard_data=False)
-
shutdown()
-
transform(transform_ctx: ai4med.common.transform_ctx.TransformContext)
-
-
class
DeterministicTransformer(transforms, chain_transformer) Bases:
ai4med.libs.data.smart_cache.CachePreparer-
get_cache_id(data: dict)
-
prepare(data)
-
-
class
Dataset(datalist_manager, output_shape_dict, output_type_dict) Bases:
object-
get_dataset_size()
-
get_next_batch(session)
-
initialize(session, state=- 1)
-
static
make_dataset(datalist_manager: ai4med.libs.data.datalist_manager.DatalistManager, output_shape_dict, output_type_dict, batch_size=1, num_workers=1, prefetch_size=0, shuffle_size=0, repeat_count=None)
-
set_sharding(rank, num_shards, equal_shard_size=True, fixed_shard_data=False)
-
shutdown()
-
-
class
ListGenerator(items_list, output_types, extra_elements=None) Bases:
objectFilename generator.
This generator outputs a dict, which consists of items with any keys.
- Parameters
items_list – this is a list of data items. Each item is a dict keyed by the names of data elements.
-
get_item_at_index(index) Return an item at a specific index.
-
get_item_count()
-
get_items()
-
next() Generator requires a callable object, which returns one data sample.
-
set_items(items_list) Sets the item
- Parameters
items_list – this is a list of data items. Each item is a dict keyed by the names of data elements.
-
sample_weights_by_categories(items_list) “Calculates sample weights based on item count per class.
-
class
CachePreparer Bases:
abc.ABC-
get_cache_id(data)
-
abstract
prepare(data)
-
-
class
SmartCache(data_list, preparer, cache_count, replace_count, start_pos=0, cacher=None) Bases:
object-
get_all_data_ids()
-
get_cached_data_ids()
-
get_data(x)
-
get_id_map()
-
manage_replacement()
-
shutdown()
-
start()
-
update_cache()
-