nat.plugins.eval.runtime.remote_workflow#

Attributes#

`logger`
`DATA_PREFIX`
`INTERMEDIATE_DATA_PREFIX`

Classes#

EvaluationRemoteWorkflowHandler

Module Contents#

logger#

DATA_PREFIX = 'data: '#

INTERMEDIATE_DATA_PREFIX = 'intermediate_data: '#

class EvaluationRemoteWorkflowHandler( config: nat.data_models.evaluate_runtime.EvaluationRunConfig, max_concurrency: int, )#

config#

semaphore#

async run_workflow_remote_single( session: aiohttp.ClientSession, item: nat.data_models.evaluator.EvalInputItem, ) → None#: Sends a single input to the endpoint hosting the workflow and retrieves the response.

async run_workflow_remote_with_limits( session: aiohttp.ClientSession, item: nat.data_models.evaluator.EvalInputItem, progress_bar: tqdm.tqdm, ) → None#: Sends limited number of concurrent requests to a remote workflow and retrieves responses.

async run_workflow_remote( eval_input: nat.data_models.evaluator.EvalInput, ) → nat.data_models.evaluator.EvalInput#: Sends inputs to a workflow hosted on a remote endpoint.

async _retry_request(attempt: int, max_retries: int) → bool#

Sleep with exponential backoff if retry attempts remain.

Returns True if should retry, False if last attempt.