`bridge.data.energon.energon_provider`#

Module Contents#

Classes#

EnergonProvider

Energon Provider.

API#

class bridge.data.energon.energon_provider.EnergonProvider#

Bases: megatron.bridge.data.utils.DatasetProvider

Energon Provider.

path: str#: None

image_processor: Optional[Any]#: None

seq_length: int#: None

micro_batch_size: int#: None

global_batch_size: int#: None

num_workers: torch.int_repr#: None

dataloader_type: str#: ‘external’

task_encoder: Optional[Any]#: None

enable_in_batch_packing: bool#: False

defer_in_batch_packing_to_step: bool#: False

pad_to_max_length: bool#: False

pad_to_multiple_of: int#: 128

in_batch_packing_pad_to_multiple_of: int#: 1

_sync_task_encoder_sequence_batching() → None#

build_datasets( context: megatron.bridge.data.utils.DatasetBuildContext, )#