core.inference.text_generation_server.text_generation_server#

Module Contents#

Classes#

MegatronGenerate

Text generation endpoint.

MegatronServer

Megatron text generation server.

API#

class core.inference.text_generation_server.text_generation_server.MegatronGenerate(engine, args)#

Bases: flask_restful.Resource

Text generation endpoint.

Initialization

put()#

Handle generation request.

class core.inference.text_generation_server.text_generation_server.MegatronServer(model, args=None)#

Bases: object

Megatron text generation server.

Initialization

run(url, port)#

Run the server.