Nemotron-Omni

import json
from datasets import load_dataset
dataset = load_dataset("naver-clova-ix/cord-v2")
print(f"Train      : {len(dataset['train'])} samples")
print(f"Validation : {len(dataset['validation'])} samples")
print(f"Test       : {len(dataset['test'])} samples")
# Inspect a sample
ex = dataset["train"][0]
gt = json.loads(ex["ground_truth"])["gt_parse"]
print(f"\nGround-truth keys: {list(gt.keys())}")

import torch
import json
from transformers import AutoModel, AutoProcessor
from datasets import load_dataset
from nemo_automodel.components.datasets.vlm.utils import json2token
CKPT = "vlm_checkpoints/LOWEST_VAL/model/consolidated"
# Load processor
processor = AutoProcessor.from_pretrained(CKPT, trust_remote_code=True)
tokenizer = processor.tokenizer
# `device_map` streams weights directly to GPU; skipping the AutoModel.from_config
# CPU-instantiation step saves ~5 min on the 30B v3 dump.
model = AutoModel.from_pretrained(
    CKPT, trust_remote_code=True, torch_dtype=torch.bfloat16,
    device_map={"": torch.cuda.current_device()},
)
# Reset RADIO's `summary_idxs` (non-persistent buffer; can be a meta tensor after load)
if hasattr(model, "vision_model") and hasattr(model.vision_model, "radio_model"):
    model.vision_model.radio_model.summary_idxs = None
model.eval()
# Load dataset
dataset = load_dataset("naver-clova-ix/cord-v2")
# v3 processor returns extra placeholder-expansion metadata that is NOT a generate() kwarg.
PROCESSOR_METADATA_KEYS = ("num_patches", "num_tokens", "imgs_sizes")
# Run inference on the first 5 validation samples
for i in range(5):
    sample = dataset["validation"][i]
    image = sample["image"].convert("RGB")
    gt = json.loads(sample["ground_truth"])["gt_parse"]
    gt_text = json2token(gt, sort_json_key=True)
    # Build prompt — enable_thinking=False for structured output
    messages = [{"role": "user", "content": "<image>\nDescribe this image."}]
    text = tokenizer.apply_chat_template(
        messages, tokenize=False,
        add_generation_prompt=True, enable_thinking=False,
    )
    inputs = processor(text=text, images=[image], return_tensors="pt")
    for k in PROCESSOR_METADATA_KEYS:
        inputs.pop(k, None)
    inputs = {k: v.to("cuda") if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
    with torch.no_grad():
        output_ids = model.generate(**inputs, max_new_tokens=1024, do_sample=False)
    generated = tokenizer.decode(
        output_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True,
    ).strip()
    print(f"\n=== Sample {i} ===")
    print(f"Ground truth: {gt_text}")
    print(f"Prediction:   {generated}")

import json, re
import torch
from pathlib import Path
from safetensors import safe_open
from transformers import AutoModel, AutoProcessor
BASE    = "<path_to_nemotron_omni_v3>"
ADAPTER = "<ckpt_dir>/LOWEST_VAL/model"
# Load base directly to GPU. Skip AutoModel.from_config — instantiating a 30B
# model on CPU just to read the class type adds 5+ minutes.
processor = AutoProcessor.from_pretrained(BASE, trust_remote_code=True)
model = AutoModel.from_pretrained(
    BASE, trust_remote_code=True, dtype=torch.bfloat16,
    device_map={"": torch.cuda.current_device()},
)
if hasattr(model, "vision_model") and hasattr(model.vision_model, "radio_model"):
    model.vision_model.radio_model.summary_idxs = None
# Wrapper -> HF base FQN translation. vision_projector.* targets are listed in
# adapter_config.json but no tensors are saved for them, so we just skip those.
def translate(fqn):
    if fqn.startswith("language_model.model."):
        return "language_model.backbone." + fqn[len("language_model.model."):]
    return None
cfg   = json.loads((Path(ADAPTER) / "adapter_config.json").read_text())
scale = cfg["lora_alpha"] / cfg["r"]
pairs = {}
with safe_open(str(Path(ADAPTER) / "adapter_model.safetensors"), framework="pt") as f:
    for k in f.keys():
        m = re.match(r"^base_model\.model\.(.+)\.lora_(A|B)\.weight$", k)
        if m:
            pairs.setdefault(m.group(1), {})[m.group(2)] = f.get_tensor(k)
modules = dict(model.named_modules())
for wrapper_fqn, ab in pairs.items():
    hf_fqn = translate(wrapper_fqn)
    if hf_fqn is None or hf_fqn not in modules:
        continue
    W = modules[hf_fqn].weight
    A = ab["A"].to(device=W.device, dtype=torch.float32)
    B = ab["B"].to(device=W.device, dtype=torch.float32)
    with torch.no_grad():
        W.add_(((B @ A) * scale).to(W.dtype))
model.eval()
# ... then run the same generate() loop as in the SFT example above.