ExperimentConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
model_name	string	Name of model if invoking task via model_agnostic
encryption_key	string	Key for encrypting model checkpoints
results_dir	string	Path to where all the assets generated from a task are stored
wandb	collection						False
model	collection	Configurable parameters to construct the model for the NVPanoptix3D experiment					False
dataset	collection	Configurable parameters to construct the dataset for the NVPanoptix3D experiment					False
train	collection	Configurable parameters to construct the trainer for the NVPanoptix3D experiment					False
inference	collection	Configurable parameters to construct the inferencer for the NVPanoptix3D experiment					False
evaluate	collection	Configurable parameters to construct the evaluator for the NVPanoptix3D experiment					False
export	collection	Configurable parameters to construct the exporter for the NVPanoptix3D experiment					False
gen_trt_engine	collection	Configurable parameters to construct the TensorRT engine builder for a NVPanoptix3D experiment					False

WandBConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
enable	bool		True
project	string		TAO Toolkit
entity	string
group	string
tags	list		[‘tao-toolkit’]				False
reinit	bool		False
sync_tensorboard	bool		False
save_code	bool		False
name	string		TAO Toolkit Training
run_id	string

NVPanoptix3DModelConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
backbone	collection	Configuration hyper parameters for the NVPanoptix3D Backbone					False
sem_seg_head	collection	Configuration hyper parameters for the Mask2Former Semantic Segmentation Head					False
mask_former	collection	Configuration hyper parameters for the Mask2Former model					False
frustum3d	collection	Configuration hyper parameters for the Frustum3D model					False
projection	collection	Configuration hyper parameters for the Projection model					False
mode	categorical	Segmentation mode	panoptic			panoptic,instance,semantic
object_mask_threshold	float	The value of the threshold to be used when filtering out the object mask	0.4
overlap_threshold	float	The value of the threshold to be used when evaluating overlap	0.5
test_topk_per_image	int	Keep topk instances per image for instance segmentation	100

NVPanoptix3DDatasetConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
train	collection	Configurable parameters to construct the train dataset					False
val	collection	Configurable parameters to construct the validation dataset					False
test	collection	Configurable parameters to construct the test dataset					False
workers	int	The number of parallel workers processing data	8	1
pin_memory	bool	Flag to allocate pagelocked memory for faster of data between the CPU and GPU	True
augmentation	collection	Configuration parameters for data augmentation					False
contiguous_id	bool	Flag to enable contiguous IDs for labels	False
label_map	string	A path to label map file
name	categorical	Dataset name	front3d			front3d,matterport,synthetic_hospital,synthetic_warehouse
downsample_factor	int	Downsample factor (1: Synthetic & Front3D, 2: Matterport3D)	1
iso_value	float	ISO value to reconstruct mesh from TUDF volume	1.0
ignore_label	int	Ignore label value	255
min_instance_pixels	int	Minimum number of pixels required for an instance to be considered valid	200
img_format	string	Image format	RGB
target_size	list	Input image size to resize	[320, 240]				False
reduced_target_size	list	Image size to process at 3D stage	[160, 120]				False
depth_size	list	Input depth size to resize	[120, 160]				False
depth_bound	bool	Enable depth truncation in bounds	False
depth_min	float	Min depth value	0.4
depth_max	float	Max depth value	6.0
frustum_mask_path	string	Relative frustum mask path	meta/frustum_mask.npz
occ_truncation_lvl	list	Value to create occuppancy volume from TUDF volume	[8.0, 6.0]				False
truncation_range	list	truncation range for TUDF volume	[0.0, 12.0]				False
enable_3d	bool	Enable 3d for training	False
enable_mp_occ	bool	Enable multi-plane occupancy	True
depth_scale	float	Depth scale	25.0
num_thing_classes	int	Number of thing classes	9

NVPanoptix3DTrainExpConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
num_gpus	int	The number of GPUs to run the train job	1	1
gpu_ids	list	List of GPU IDs to run the training on; length of list must equal train.num_gpus	[0]				False
num_nodes	int	Number of nodes to run the training on; if > 1, multi-node is enabled	1	1
seed	int	Seed for the initializer in PyTorch; if < 0, fixed seed is disabled	1234	-1	inf
cudnn	collection						False
num_epochs	int	Number of epochs to run the training	10	1	inf
checkpoint_interval	int	The interval (in epochs) at which a checkpoint is saved	1	1
checkpoint_interval_unit	categorical	The unit of the checkpoint interval	epoch			epoch,step
validation_interval	int	The interval (in epochs) at which an evaluation is triggered on the validation set	1	1
resume_training_checkpoint_path	string	Path to the checkpoint to resume training from
results_dir	string	The folder in which to save the experiment
checkpoint_2d	string	Path to 2D stage checkpoint to initialize the 3D stage training
checkpoint_3d	string	Path to 3D stage checkpoint to initialize the 3D stage training
val_check_interval	int	The number of iterations between validation checks	5
freeze	list	List of layer names to freeze Example: [“backbone”, “transformer.encoder”, “input_proj”]	[]				False
clip_grad_norm	float	Amount to clip the gradient by L2 Norm	0.1
clip_grad_norm_type
clip_grad_type	string	Gradient clip type	full
is_dry_run	bool	Whether to run the trainer in Dry Run mode	False
optim	collection	Hyper parameters to configure the optimizer					False
precision	categorical	Precision to run the training on	fp32			fp16,fp32
distributed_strategy	categorical	The multi-GPU training strategy DDP (Distributed Data Parallel) and Fully Sharded DDP are supported	ddp			ddp,fsdp
activation_checkpoint	bool	A True value instructs train to recompute in backward pass to save GPU memory, rather than storing activations	True
verbose	bool	Flag to enable printing of detailed learning rate scaling from the optimizer	False
iters_per_epoch	int	Number of iterations per epoch

NVPanoptix3DInferenceExpConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
num_gpus	int	The number of GPUs to run the evaluation job	1	1
gpu_ids	list	List of GPU IDs to run the inference on; length of list must equal inference.num_gpus	[0]				False
num_nodes	int	Number of nodes to run the inference on; if > 1, multi-node is enabled	1	1
checkpoint	string	Path to the checkpoint file used for inference
trt_engine	string	Path to the TensorRT engine folder to be used for inference
results_dir	string	Path to where all the assets generated from a task are stored
batch_size	int	The batch size of the input tensor; important if batch_size > 1 for large datasets	-1	-1
mode	categorical	Mode to run inference	panoptic			semantic,instance,panoptic
images_dir	string	Path to the images directory

NVPanoptix3DEvaluateExpConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
num_gpus	int	The number of GPUs to run the evaluation job	1	1
gpu_ids	list	List of GPU IDs to run the evaluation on; length of list must equal evaluate.num_gpus	[0]				False
num_nodes	int	Number of nodes to run the evaluation on; if > 1, multi-node is enabled	1	1
checkpoint	string	Path to the checkpoint file used for evaluation
trt_engine	string	Path to the TensorRT engine to be used for evaluation; only works with tao-deploy
results_dir	string	Path to where all the assets generated from a task are stored
batch_size	int	The batch size of the input tensor; important if batch_size > 1 for large datasets	-1	-1

NVPanoptix3DExportExpConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
results_dir	string	Path to where all the assets generated from a task are stored
gpu_id	int	The index of the GPU used to build the TensorRT engine	0
checkpoint	string	Path to the checkpoint file to run export	???
onnx_file	string	Path to the ONNX model file	???
on_cpu	bool	Flag to export CPU compatible model	False
input_channel	ordered_int	Number of channels in the input tensor	3	1		1,3
input_width	int	Width of the input image tensor	960	32
input_height	int	Height of the input image tensor	544	32
opset_version	int	Operator set version of the ONNX model used to generate the TensorRT engine	17	1
batch_size	int	The batch size of the input tensor for the engine A value of -1 implies dynamic tensor shapes	-1	-1
verbose	bool	Flag to enable verbose TensorRT logging	False
format	categorical	File format to export to	onnx			onnx,xdl
onnx_file_2d	string	Path to the ONNX model 2D file
onnx_file_3d	string	Path to the ONNX model 3D file
max_voxels	int	The maximum number of voxels in the input tensor for the engine	700000	1

NVPanoptix3DGenTRTEngineExpConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
results_dir	string	Path to where all the assets generated from a task are stored
gpu_id	int	The index of the GPU used to build the TensorRT engine	0	0
onnx_file	string	Path to the ONNX model file	???
trt_engine	string	Path to the generated TensorRT engine; only works with tao-deploy	???
timing_cache	string	Path to a TensorRT timing cache that speeds up engine generation; will be created, read, and updated
batch_size	int	The batch size of the input tensor for the engine A value of -1 implies dynamic tensor shapes	-1	-1
verbose	bool	Flag to enable verbose TensorRT logging	False
tensorrt	collection	Hyper parameters to configure the NVPanoptix3D TensorRT Engine builder					False

Backbone Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
backbone_type	categorical	Type of backbone to use. Available backbone: vggt	vggt			vggt
pretrained_model_path	string	Path to a pretrained backbone file

SemanticSegmentationHead Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
common_stride	int	Common stride	4	2
transformer_enc_layers	int	Number of transformer encoder layers	6	1
convs_dim	int	Convolutional layer dimension	256	1
mask_dim	int	Mask head dimension	256	1
depth_dim	int	Depth head dimension	256	1
ignore_value	int	Ignore value	255	0	255
deformable_transformer_encoder_in_features	list	List of feature names for deformable transformer encoder input	[‘res3’, ‘res4’, ‘res5’]				False
num_classes	int	Number of classes	13	1
norm	string	Norm layer type	GN
in_features	list	List of input feature names	[‘res2’, ‘res3’, ‘res4’, ‘res5’]				False

MaskFormer Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
dropout	float	The probability to drop out	0	0.0	1.0
nheads	int	Number of heads	8
num_object_queries	int	The number of queries	100	1	inf
hidden_dim	int	Dimension of the hidden units	256
transformer_dim_feedforward	int	Dimension of the feedforward network in the transformer	1024	1
dim_feedforward	int	Dimension of the feedforward network	2048	1
dec_layers	int	Number of decoder layers in the transformer	10	1
pre_norm	bool	Whether to add layer norm in the encoder; 1=add layer norm, 0=do not add	0
class_weight	float	The relative weight of the classification error in the matching cost	2	0.0	inf
dice_weight	float	The relative weight of the focal loss of the binary mask in the matching cost	5	0.0	inf
mask_weight	float	The relative weight of the dice loss of the binary mask in the matching cost	5	0.0	inf
depth_weight	float	The relative weight of the depth loss in the matching cost	5	0.0	inf
mp_occ_weight	float	The relative weight of the mp occ loss in the matching cost	5	0.0	inf
train_num_points	int	The number of points to sample	12544
oversample_ratio	float	Oversampling parameter	3
importance_sample_ratio	float	Ratio of points that are sampled via important sampling	0.75
deep_supervision	bool	Flag to enable deep supervision	1
no_object_weight	float	The relative classification weight applied to the no-object category	0.1
size_divisibility	int	Size divisibility	32

Frustum3D Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
truncation	float	The truncation value	3.0
iso_recon_value	float	The iso recon value	2.0
panoptic_weight	float	The weight of the panoptic loss	25.0
completion_weights	list	The weights of the completion loss	[50.0, 25.0, 10.0]				False
surface_weight	float	The weight of the surface loss	5.0
unet_output_channels	int	The number of output channels of the UNet	16
unet_features	int	The number of features of the UNet	16
use_multi_scale	bool	Whether to use multi-scale	False
grid_dimensions	int	The number of grid dimensions	256
frustum_dims	int	The number of frustum dimensions	256
signed_channel	int	The number of signed channel	3

Projection Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
voxel_size	float	The size of the voxel	0.03
sign_channel	bool	Whether to use signed channel	1
depth_feature_dim	int	The dimension of the depth feature	256

Dataset Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
base_dir	string	Root directory of the dataset
json_path	string	JSON file for image/mask pair
batch_size	int	Batch size	1	1
num_workers	int	Number of workers in the dataloader	1	0

Dataset Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
base_dir	string	Root directory of the dataset
json_path	string	JSON file for image/mask pair
batch_size	int	Batch size	1	1
num_workers	int	Number of workers in the dataloader	1	0

Dataset Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
base_dir	string	Root directory of the dataset
json_path	string	JSON file for image/mask pair
batch_size	int	Batch size	1	1
num_workers	int	Number of workers in the dataloader	1	0

AugmentationConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
train_min_size	list	A list of sizes to perform random resize	[448]				False
train_max_size	int	The maximum random crop size for training data	768	32	960
train_crop_size	list	The random crop size for training data in [H, W]	[240, 240]				False
test_min_size	int	The minimum resize size for test data	240	32	960
test_max_size	int	The maximum resize size for test	960	32	960
color_aug_ssd	bool	Color augmentation	False
enable_crop	bool	Enable cropping for input image	False
crop_size	list	Size to crop input image	[240, 240]				False
single_category_max_area	float	Maximum ratio of crop area that can be occupied by a single semantic category	1.0	0.0	1.0
random_flip	string	Flip horizontal/vertical
random_flip_prob	float	Flip probability	0.5	0.0	1.0
size_divisibility	float	Size divisibility to pad	-1
gen_aug_weight	float	Weight for generated augmentation, 0.0 will disable generated augmentation	0.0	0.0	1.0

CuDNNConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
benchmark	bool	Whether to enable cuDNN benchmark mode	False
deterministic	bool	Whether to enable cuDNN deterministic mode	True

OptimConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
type	categorical	Type of optimizer used to train the network	AdamW			AdamW
monitor_name	categorical	The metric value to be monitored for the AutoReduce Scheduler	val_loss			val_loss,train_loss
lr	float	The initial learning rate for training the model	0.0002	0.0	1.0		True
backbone_multiplier	float	A multiplier for backbone learning rate	0.1	0.0	1.0		True
momentum	float	The momentum for the AdamW optimizer	0.9	0.0	1.0		True
weight_decay	float	The weight decay coefficient	0.05	0.0	1.0		True
lr_scheduler	categorical	The learning scheduler: MultiStep: Decrease the lr by lr_decay from lr_steps Warmuppoly: Poly learning rate schedule	MultiStep			MultiStep,Warmuppoly
milestones	list	Learning rate decay epochs	[88, 96]				False
gamma	float	Multiplicative factor of learning rate decay	0.1
max_steps	int	The maximum number of steps to train the model	160000
warmup_factor	float	The warmup factor for the learning rate scheduler	1.0
warmup_iters	int	The number of warmup iterations	0

NVPanoptix3DTrtConfig Fields

Field	value_type	description	default_value	valid_min	valid_max	valid_options	automl_enabled
workspace_size	int	The size (in megabytes) of the workspace TensorRT has to run its optimization tactics and generate the TensorRT engine	1024	0
min_batch_size	int	The minimum batch size in the optimization profile for the input tensor of the TensorRT engine	1	1
opt_batch_size	int	The optimum batch size in the optimization profile for the input tensor of the TensorRT engine	1	1
max_batch_size	int	The maximum batch size in the optimization profile for the input tensor of the TensorRT engine	1	1
layers_precision	list	The list to specify layer precision	[]				False
data_type	categorical	The precision to be set for building the TensorRT engine	FP32			FP32,FP16