`core.tokenizers.vision.libraries.multimodal_tokenizer`#

Multimodal tokenizer.

Module Contents#

Classes#

MegatronMultimodalTokenizer

Multimodal Tokenizer.

Functions#

`_build_nemotron6_moe_native_tool_boundary_mask`	Mark rendered user boundaries that originate from native tool turns.
`_find_nemotron6_moe_assistant_indices`	Find assistant turns selected by the Nemotron 6 MoE loss-mask policy.

Data#

`IMAGE_TAGS`
`_NEMOTRON6_MOE_MESSAGE_START_TOKEN_ID`
`_NEMOTRON6_MOE_MESSAGE_END_TOKEN_ID`
`_NEMOTRON6_MOE_TOOL_RESPONSE_TOKEN_ID`
`_NEMOTRON6_MOE_LINE_BREAK_TOKEN_ID`
`_NEMOTRON6_MOE_ASSISTANT_ROLE_TOKEN_IDS`
`_NEMOTRON6_MOE_USER_ROLE_TOKEN_ID`
`_NEMOTRON6_MOE_TOKENIZER_CONTRACT`
`mistral_custom_template`
`nvlm_yi_34b_template`
`qwen2p0_custom_template`
`llama3p1_chat_template`
`nemotron_custom_template`
`nemotron_aligned_custom_template`
`llama_nemotron_template`
`nemotron_h_reasoning_template`
`llama_nemotron_super_template`
`llama_nemotron_super_1p5_template`

API#

core.tokenizers.vision.libraries.multimodal_tokenizer.IMAGE_TAGS#: None

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_MESSAGE_START_TOKEN_ID#: 10

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_MESSAGE_END_TOKEN_ID#: 11

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_TOOL_RESPONSE_TOKEN_ID#: 16

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_LINE_BREAK_TOKEN_ID#: 1010

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_ASSISTANT_ROLE_TOKEN_IDS#: (1503, 19464)

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_USER_ROLE_TOKEN_ID#: 3263

core.tokenizers.vision.libraries.multimodal_tokenizer._NEMOTRON6_MOE_TOKENIZER_CONTRACT#: None

core.tokenizers.vision.libraries.multimodal_tokenizer._build_nemotron6_moe_native_tool_boundary_mask( conversation: List[Dict], ) → numpy.ndarray#: Mark rendered user boundaries that originate from native tool turns.

core.tokenizers.vision.libraries.multimodal_tokenizer._find_nemotron6_moe_assistant_indices( tokens: numpy.ndarray, prompt_format: str, train_only_on_last_assistant_turn: bool, has_nonempty_thinking_trace: bool, tool_response_as_turn_boundary: bool, native_tool_boundary_mask: numpy.ndarray, ) → numpy.ndarray#: Find assistant turns selected by the Nemotron 6 MoE loss-mask policy.

core.tokenizers.vision.libraries.multimodal_tokenizer.mistral_custom_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.nvlm_yi_34b_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.qwen2p0_custom_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.llama3p1_chat_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.nemotron_custom_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.nemotron_aligned_custom_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.llama_nemotron_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.nemotron_h_reasoning_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.llama_nemotron_super_template = <Multiline-String>#

core.tokenizers.vision.libraries.multimodal_tokenizer.llama_nemotron_super_1p5_template = <Multiline-String>#

class core.tokenizers.vision.libraries.multimodal_tokenizer.MegatronMultimodalTokenizer(

path: str,

prompt_format: str,

special_tokens: List[str],

image_tag_type: str,

force_system_message: bool = False,

use_gigatoken: bool = False,

keep_history_thinking: bool = False,

**kwargs,

)#

Multimodal Tokenizer.

Initialization

Tokenizer with a support for non-text inputs.

Note: Currently, only HuggingFaceTokenizer is supported as the underlying text tokenizer.

Parameters:

path (str) – Path to the underlying Hugging Face tokenizer.
prompt_format (str) – Prompt format for the tokenizer.
special_tokens (List[str]) – Non-text tokens.
image_tag_type (str) – Image tag to apply, if any. For example .
use_gigatoken (bool) – Use GigaToken for encoding, retaining HF chat templates.
keep_history_thinking (bool) – Preserve thinking traces in conversation history.

_validate_nemotron6_moe_tokenizer_contract() → None#: Fail if the tokenizer no longer matches the masking token IDs.

_MM_MARKER#: ‘\ue000’

static _as_parts( content: Union[str, List[Dict[str, Any]]], ) → List[Dict[str, Any]]#: Coerce string content into a single text part.

static _plain_text( content: Union[str, List[Dict[str, Any]]], ) → str#: Concatenate the text portions of a content value, ignoring multimodal parts.

_render_parts( parts: List[Dict[str, Any]], ) → tuple[str, List[List[int]]]#: Render content parts to (text-with-markers, ordered replacement id-spans).

_encode_with_markers( text: str, replacements: List[List[int]], ) → List[int]#: Tokenize text, splicing replacement id-spans at every marker position.

_render_conversation( conversation: List[Dict[str, Any]], ) → tuple[List[Dict[str, Any]], List[List[List[int]]]]#: Lower structured turns to chat-template turns and per-turn replacements.

_apply_image_tag(text: str) → str#: Preserve tag wrapping for legacy string image markers.

_apply_chat_template_to_text(

conversation: List[Dict[str, Any]],

add_generation_prompt: bool,

**kwargs,

) → str#

offsets(ids: list[int], text: str) → list[int]#: Assume that the tokenizer is a HuggingFaceTokenizer. Copied from megatron.training.tokenizer.tokenizer.py:_HuggingFaceTokenizer.offsets

_has_nonempty_thinking_trace( conversation: List[Dict], ) → bool#: Return True if any assistant message has a non-empty trace.

_tokenize_text_with_offsets( text: str, boundary_offsets: List[int], ) → Optional[tuple[List[int], List[int]]]#

Tokenize text once and map character boundaries to token boundaries.

Returns None when the tokenizer cannot provide reliable offsets, or when a requested boundary falls inside a token. In that case callers must use the cumulative-prefix path to preserve legacy behavior.

_tokenize_raw_conversation_slow( rendered_turns: List[Dict[str, Any]], replacements_per_turn: List[List[List[int]]], return_target: bool, )#

_tokenize_raw_conversation( rendered_turns: List[Dict[str, Any]], replacements_per_turn: List[List[List[int]]], return_target: bool, )#: Tokenize a pre-rendered conversation without applying a chat template.

tokenize(

text: Union[str, List[Dict]],

add_special_tokens: bool = True,

**kwargs,

) → List[int]#: Tokenize conversation or string input.

tokenize_conversation(

conversation: List[Dict],

return_target: bool,

add_generation_prompt: bool,

train_only_on_last_assistant_turn: bool = False,

skip_chat_template: bool = False,

tool_response_as_turn_boundary: bool = False,

assistant_turn_loss: Optional[List[bool]] = None,

**kwargs,

)#

Convert a conversation to tokens.

Each turn’s content may be either a legacy string or a list of structured parts of the form [{"type": "text", "text": str}, {"type": "image"}]. Legacy string "<image>" markers retain tag wrapping and their added-vocabulary IDs, as expected by existing task encoders. Structured image parts use dedicated sentinel IDs; "<image>" within a structured text part remains literal text.

convert_tokens_to_ids(tokens: List[str])#: Convert tokens to IDs.

detokenize(tokens: List[int])#: Detokenize tokens, surfacing multimodal sentinels as readable text.

_image_part_text() → str#: Return the textual representation of an image sentinel.

add_special_tokens(special_tokens: List[str])#: Add special tokens.

get_special_tokens()#: Get special tokens.

property pad#: Pad token ID.

property bos#: Beginning of sentence token ID.

property eod#: End of sentence token ID.

property vocab#: Tokenizer vocab.

abstract property inv_vocab#: Inverse vocab.

property vocab_size#: Vocabulary size.

property image_token_index#: Internal sentinel id used for image slots.

core.tokenizers.vision.libraries.multimodal_tokenizer#