modifiers.async_llm_pii_modifier

Module Contents

Classes

AsyncLLMInference
AsyncLLMPiiModifier

API

class modifiers.async_llm_pii_modifier.AsyncLLMInference(

base_url: str,

api_key: str | None,

model: str,

system_prompt: str,

)

Initialization

async infer(text: str) → list[dict[str, str]]

class modifiers.async_llm_pii_modifier.AsyncLLMPiiModifier(

base_url: str,

api_key: str | None = None,

model: str = 'meta/llama-3.1-70b-instruct',

system_prompt: str | None = None,

pii_labels: list[str] | None = None,

language: str = 'en',

max_concurrent_requests: int | None = None,

)

Bases: nemo_curator.modifiers.DocumentModifier

Initialization

batch_redact(

text: pandas.Series,

pii_entities_lists: list[list[dict[str, str]]],

) → pandas.Series

async call_inferer(

text: pandas.Series,

inferer: modifiers.async_llm_pii_modifier.AsyncLLMInference,

) → list[str]

load_inferer() → modifiers.async_llm_pii_modifier.AsyncLLMInference

modify_document(text: pandas.Series) → pandas.Series