API ReferenceFull Library ReferenceNemo GymNemo Gym

nemo_gym.server_utils

View as Markdown

Module Contents

Classes

Functions

Data

DEFAULT_HEAD_SERVER_PORT

DISCONNECTED_CLIENT_OS_HELP_TEXT

DISCONNECTED_CLIENT_OS_PRINT_INTERVAL

IS_NEMO_GYM_FASTAPI_ENTRYPOINT_KEY_NAME

IS_NEMO_GYM_FASTAPI_WORKER_KEY_NAME

MAX_NUM_TRIES

NEMO_GYM_FASTAPI_NUM_WORKERS

SESSION_ID_KEY

ServerStatus

_GLOBAL_AIOHTTP_CLIENT

_GLOBAL_AIOHTTP_CLIENT_REQUEST_DEBUG

_NEMO_GYM_STARTED_RAY_CLUSTER

_NUM_CLIENT_OS_ERROR

_NUM_SERVER_DISCONNECTED_ERROR

API

class nemo_gym.server_utils.BaseServer()

Bases: BaseModel

All instances of BaseServer are queryable using ServerClient.

config
BaseRunServerInstanceConfig
nemo_gym.server_utils.BaseServer.load_config_from_global_config() -> nemo_gym.config_types.BaseRunServerInstanceConfig
classmethod
class nemo_gym.server_utils.GlobalAIOHTTPAsyncClientConfig()

Bases: BaseModel

global_aiohttp_client_request_debug
bool = False
global_aiohttp_connector_limit
int = 100 * 1024
global_aiohttp_connector_limit_per_host
int = 1024
class nemo_gym.server_utils.HeadServer()

Bases: BaseServer

_server_instances
List[dict] = []
config
BaseServerConfig
nemo_gym.server_utils.HeadServer.get_server_instances() -> typing.List[dict]
nemo_gym.server_utils.HeadServer.global_config_dict_yaml() -> str
async
nemo_gym.server_utils.HeadServer.run_webserver() -> typing.Tuple[uvicorn.Server, threading.Thread, nemo_gym.server_utils.HeadServer]nemo_gym.server_utils.HeadServer.run_webserver() -> typing.Tuple[uvicorn.Server, threading.Thread, nemo_gym.server_utils.HeadServer]
classmethod
nemo_gym.server_utils.HeadServer.set_server_instances(
    instances: typing.List
) -> None
nemo_gym.server_utils.HeadServer.setup_webserver() -> fastapi.FastAPI
class nemo_gym.server_utils.ProfilingMiddlewareConfig()

Bases: ProfilingMiddlewareInputConfig

profiling_enabled
bool = False
class nemo_gym.server_utils.ProfilingMiddlewareInputConfig()

Bases: BaseModel

profiling_results_dirpath
Optional[str] = None
class nemo_gym.server_utils.ServerClient()

Bases: BaseModel

global_config_dict
DictConfig
head_server_config
BaseServerConfig
model_config
= ConfigDict(arbitrary_types_allowed=True)
nemo_gym.server_utils.ServerClient._build_server_base_url(
    server_config_dict: omegaconf.OmegaConf
) -> str
nemo_gym.server_utils.ServerClient.get(
    server_name: str,
    url_path: str,
    kwargs: typing.Unpack[aiohttp.client._RequestOptions] = {}
) -> aiohttp.ClientResponse
async

Parameters:

server_name
str

str The name of the server you are trying to call.

url_path
str

str The URL path in the server you are trying to call e.g. “/v1/responses”.

nemo_gym.server_utils.ServerClient.load_from_global_config(
    head_server_config: typing.Optional[nemo_gym.config_types.BaseServerConfig] = None
) -> nemo_gym.server_utils.ServerClient
classmethod
nemo_gym.server_utils.ServerClient.load_head_server_config() -> nemo_gym.config_types.BaseServerConfig
classmethod
nemo_gym.server_utils.ServerClient.poll_for_status(
    server_name: str
) -> nemo_gym.server_utils.ServerStatus
nemo_gym.server_utils.ServerClient.post(
    server_name: str,
    url_path: str,
    kwargs: typing.Unpack[aiohttp.client._RequestOptions] = {}
) -> aiohttp.ClientResponse
async

Parameters:

server_name
str

str The name of the server you are trying to call.

url_path
str

str The URL path in the server you are trying to call e.g. “/v1/responses”.

nemo_gym.server_utils.ServerClient.request(
    server_name: str,
    url_path: str,
    method: str,
    kwargs: typing.Unpack[aiohttp.client._RequestOptions] = {}
) -> aiohttp.ClientResponse
async
class nemo_gym.server_utils.ServerInstanceDisplayConfig()

Bases: BaseModel

config_path
Optional[str] = None
dir_path
Optional[Path] = None
entrypoint
Optional[str] = None
host
Optional[str] = None
name
Optional[str] = None
pid
Optional[int] = None
port
Optional[int] = None
process_name
Optional[str] = None
server_type
Optional[str] = None
start_time
Optional[float] = None
status
Optional[ServerStatus] = None
uptime_seconds
Optional[float] = None
url
Optional[str] = None
class nemo_gym.server_utils.SimpleServer()

Bases: BaseServer

server_client
ServerClient
nemo_gym.server_utils.SimpleServer.get_session_middleware_key() -> str
nemo_gym.server_utils.SimpleServer.prefix_server_logs() -> None
nemo_gym.server_utils.SimpleServer.run_webserver() -> typing.Optional[fastapi.FastAPI]
classmethod
nemo_gym.server_utils.SimpleServer.set_ulimit(
    target_soft_limit: int = 65535
)
nemo_gym.server_utils.SimpleServer.setup_exception_middleware(
    app: fastapi.FastAPI
) -> None
nemo_gym.server_utils.SimpleServer.setup_profiling(
    app: fastapi.FastAPI,
    profiling_config: nemo_gym.server_utils.ProfilingMiddlewareConfig
) -> None
nemo_gym.server_utils.SimpleServer.setup_session_middleware(
    app: fastapi.FastAPI
) -> None
nemo_gym.server_utils.SimpleServer.setup_webserver() -> fastapi.FastAPI
abstract
class nemo_gym.server_utils.UvicornLoggingConfig()

Bases: BaseModel

uvicorn_logging_show_200_ok
bool = False
nemo_gym.server_utils.get_global_aiohttp_client(
    global_config_dict_parser_config: typing.Optional[nemo_gym.global_config.GlobalConfigDictParserConfig] = None,
    global_config_dict_parser_cls: typing.Type[nemo_gym.global_config.GlobalConfigDictParser] = GlobalConfigDictParser
) -> aiohttp.ClientSession
 
nemo_gym.server_utils.get_nemo_gym_fastapi_num_workers() -> int
 
nemo_gym.server_utils.get_response_json(
    response: aiohttp.ClientResponse
) -> typing.Any
async
nemo_gym.server_utils.get_server_url(
    server_name: str
) -> str
 
nemo_gym.server_utils.global_aiohttp_client_exit()
 
nemo_gym.server_utils.initialize_ray() -> None

Initialize ray cluster in a process. We store the Ray address in the global config dict so that child processes can connect to it. This avoids the need to start a new Ray cluster in each child process. Note: This function will modify the global config dict - update ray_head_node_address

nemo_gym.server_utils.is_global_aiohttp_client_request_debug_enabled() -> bool
 
nemo_gym.server_utils.is_global_aiohttp_client_setup() -> bool
 
nemo_gym.server_utils.is_nemo_gym_fastapi_entrypoint(
    file: str
) -> bool
 
nemo_gym.server_utils.is_nemo_gym_fastapi_worker() -> bool
 
nemo_gym.server_utils.maybe_ray_cluster_exit()
 
nemo_gym.server_utils.raise_for_status(
    response: aiohttp.ClientResponse
) -> None
async
nemo_gym.server_utils.request(
    method: str,
    url: str,
    _internal: bool = False,
    kwargs: typing.Unpack[aiohttp.client._RequestOptions] = {}
) -> aiohttp.ClientResponse
async
nemo_gym.server_utils.set_global_aiohttp_client(
    cfg: nemo_gym.server_utils.GlobalAIOHTTPAsyncClientConfig
) -> aiohttp.ClientSession
 
nemo_gym.server_utils.set_is_nemo_gym_fastapi_entrypoint(
    file: str
) -> None
 
nemo_gym.server_utils.set_is_nemo_gym_fastapi_worker() -> None
 
nemo_gym.server_utils.set_nemo_gym_fastapi_num_workers(
    num_workers: int
) -> None
 
nemo_gym.server_utils.DEFAULT_HEAD_SERVER_PORT = 11000
 
nemo_gym.server_utils.DISCONNECTED_CLIENT_OS_HELP_TEXT = "We've run into this issue in two different scenarios previously:\n1. Too many o...
 
nemo_gym.server_utils.DISCONNECTED_CLIENT_OS_PRINT_INTERVAL: int = 100
 
nemo_gym.server_utils.IS_NEMO_GYM_FASTAPI_ENTRYPOINT_KEY_NAME = 'IS_NEMO_GYM_FASTAPI_ENTRYPOINT'
 
nemo_gym.server_utils.IS_NEMO_GYM_FASTAPI_WORKER_KEY_NAME = 'IS_NEMO_GYM_FASTAPI_WORKER'
 
nemo_gym.server_utils.MAX_NUM_TRIES = 3
 
nemo_gym.server_utils.NEMO_GYM_FASTAPI_NUM_WORKERS = 'NEMO_GYM_FASTAPI_NUM_WORKERS'
 
nemo_gym.server_utils.SESSION_ID_KEY = 'session_id'
 
nemo_gym.server_utils.ServerStatus = Union[Literal['success'], Literal['connection_error'], Literal['timeout'], Liter...
 
nemo_gym.server_utils._GLOBAL_AIOHTTP_CLIENT: Union[None, ClientSession] = None
 
nemo_gym.server_utils._GLOBAL_AIOHTTP_CLIENT_REQUEST_DEBUG: bool = False
 
nemo_gym.server_utils._NEMO_GYM_STARTED_RAY_CLUSTER: bool = False
 
nemo_gym.server_utils._NUM_CLIENT_OS_ERROR: int = 0
 
nemo_gym.server_utils._NUM_SERVER_DISCONNECTED_ERROR: int = 0