aiq.eval.dataset_handler.dataset_downloader#

Attributes#

Classes#

DatasetDownloader

Download remote datasets using signed URLs or S3 credentials.

Module Contents#

logger#

class DatasetDownloader( dataset_config: aiq.data_models.dataset_handler.EvalDatasetConfig, )#

Download remote datasets using signed URLs or S3 credentials.

One DatasetDownloader object is needed for each dataset to be downloaded.

dataset_config#

_s3_client = None#

property s3_config#

property s3_client#: Lazy init the S3 client.

static ensure_directory_exists(file_path: str)#: Ensure the directory for the file exists.

download_with_signed_url( remote_file_path: str, local_file_path: str, timeout: int = 300, )#: Download a file using a signed URL.

download_with_boto3(remote_file_path: str, local_file_path: str)#: Download a file using boto3 and credentials.

static is_file_path_url(file_path: str) → bool#: Check if the file path is a URL.

download_file(remote_file_path: str, local_file_path: str)#: Download a file using the appropriate method.

download_dataset()#: Download datasets defined in the evaluation configuration.