This page is for version 3.3.0 (default).
For other versions, use one of these documentation indexes:
- 3.3.0 (default): https://docs.nvidia.com/vss/3.3.0/llms.txt
- Dev: https://docs.nvidia.com/vss/dev/llms.txt
- Archives: https://docs.nvidia.com/vss/archives/llms.txt
The protobuf schemas are defined using the two files as described below.
schema.protocontains the Frames definition which is used by perception to send metadataext.protocontains the Behavior definition which is used the by behavior analytics
schema.proto
An example of json frame object which follows the above protobuf schema can be found below
{"version": "4.0","timestamp": "2023-10-30T22:30:30.000Z","id": "1","sensorId": "Sensor_1","objects": [{"dir": [],"lipActivity": null,"info": {"visibility": "0.974372","footLocation2D": "899.152,316.812","footLocation3D": "9.85312,-12.53092","convexHull": "10,-4,23,-19,7,20,-18,26"},"coordinate": null,"speed": 0,"bbox": {"topY": 107.74882507324219,"rightX": 1116.9134521484375,"bottomY": 311.5032653808594,"leftX": 1063.572021484375},"embedding": {"info": {},"vector": [-0.003528113476932049,-0.002565900795161724,-0.020954856649041176,0.023293567821383476]},"location": null,"gaze": null,"id": "1","confidence": 0.9556514024734497,"pose": null,"type": "Person"},{"dir": [],"lipActivity": null,"info": {"visibility": "0.991698","footLocation2D": "297.655,179.069","footLocation3D": "17.56687,20.29478","convexHull": "-19,-33,-18,-32,3,35"},"coordinate": null,"speed": 0,"bbox": {"topY": 115.80743408203125,"rightX": 861.8057861328125,"bottomY": 328.8199768066406,"leftX": 814.3302001953125},"embedding": {"info": {},"vector": [0.0002149203501176089,-0.023472599685192108,-0.02617083303630352,0.022965412586927414]},"location": null,"gaze": null,"id": "2","confidence": 0.9136765003204346,"pose": null,"type": "Person"}]}
ext.proto
syntax = "proto3";package nv;option java_package = "nv.schema";import "google/protobuf/timestamp.proto";import "schema.proto";//import "struct.proto";/** represents locations array, for geo coodinates Array[Array[lon, lat]], for cartesian coordinates Array[Array[x,y]]*/message GeoLocation{string type = 1;/*Array(x,y)*/message Point{repeated double point = 1;}/*Array of points*/repeated Point coordinates = 2;//repeated google.protobuf.ListValue coordinates = 2;}/** represent object Behavior, and comprise of attributes from object movement + [[Object]] appearance* with highest confidence + [[Sensor]] + [[Place]]* + [[Pose]] + [[LipActivity]] + [[Gaze]]***** the JSON representation of this object Behavior message is stored in persistent store like Elasticsearch for search and indexing***/message Behavior{/** object id, usually a combination of sensorId + objectId*/string id = 1;/** start timestamp*/google.protobuf.Timestamp timestamp = 2;/** end timestamp*/google.protobuf.Timestamp end = 3;// start timestamp//google.protobuf.Timestamp timestamp = 4;/** representing array of [lon,lat] or [x,y]*/GeoLocation locations = 5;/** representing smoothened array of [lon,lat] or [x,y]*/GeoLocation smoothLocations = 6;/** road network edges, only applicable for geo coordinates*/repeated string edges = 7;/** distance traveled in meters*/double distance = 8;/** avg speed in mph*/double speed = 9;/** speed over a period of the trajectory*/repeated double speedOverTime = 10;/** timeInterval, end time - start time*/double timeInterval = 11;/** bearing angle*/double bearing = 12;/** N, S, E or W based on bearing*/string direction = 13;/** number of raw coordinates*/int32 length = 14;/** where object was seen by the sensor*/Place place = 15;/** sensor details*/Sensor sensor = 16;/** AnalyticsModule*/AnalyticsModule analyticsModule = 17;/** object details, associated with this trajectory*/Object object = 18;/** event details*/Event event = 19;/** video url if available*/string videoPath = 20;/** Array of pose*/repeated Pose poses = 21;/** Array of LipActivity*/repeated LipActivity lipActivities = 22;/** Array of Gaze*/repeated Gaze gazes = 23;/** Array of embeddings*/repeated Embedding embeddings = 24;/** LLM query, response, embeddings for behavior analysis and anomaly detection*/LLM llm = 26;/** addition information, that may be needed by an application*/map<string, string> info = 25;}message Incident{/** sensor id*/string sensorId = 1;/** start timestamp*/google.protobuf.Timestamp timestamp = 2;/** end timestamp*/google.protobuf.Timestamp end = 3;/** Array of object IDs involved in the incident. If the array is not empty, then the first ID represents the primary object.* May be empty for incidents that don't involve any specific objects.*/repeated string objectIds = 4;/** Array of frameIds when the incident took place (Optional)*/repeated string frameIds = 5;/** where object was seen by the sensor*/Place place = 6;/** AnalyticsModule: Module responsible for generating the Incident*/AnalyticsModule analyticsModule = 7;/** Category of the incident*/string category = 8;/** Embeddings can be related to object appearance or it could be representative of a scene description provided by VLM*/repeated Embedding embeddings = 9;/** Is the incident an anomaly*/bool isAnomaly = 10;/** LLM query, response, embeddings for incident analysis and anomaly detection*/LLM llm = 12;/** optional attributes*/map<string, string> info = 11;}message SpaceUtilizationMetrics{/** area size of the occupied space (in square meters)*/double spaceOccupied = 1;/** area size of the free space (in square meters)*/double freeSpace = 2;/** area size of the total space (in square meters)*/double totalSpace = 3;/** the ratio/percentage that the total space is utilized/occupied*/double spaceUtilization = 4;/** the max number of extra pallets that can fit into the free space*/int32 numExtraPallets = 5;/** area size of the utilizable free space*/double utilizableFreeSpace = 6;/** the ratio/percentage that the free space is utilizable*/double freeSpaceQuality = 7;/** flag to indicate if the space is safe*/bool isUnsafe = 8;}message SpaceUtilizationLayouts{/** layout of the free space*/repeated Polygon freeSpace = 1;/** layout of the utilizable free space*/repeated Polygon utilizableFreeSpace = 2;}message SpaceUtilization{/** ROI id*/string id = 1;/** timestamp in UTC format*/google.protobuf.Timestamp timestamp = 2;/** Metrics regarding utilization of space*/SpaceUtilizationMetrics metrics = 3;/** Sensors associated with the ROI*/repeated string sensors = 4;/** Layouts of free space and utilizable free space*/SpaceUtilizationLayouts layouts = 5;}
An example of json behavior object which follows the above protobuf schema can be found below
{"analyticsModule": {"description": "","info": {},"version": "","id": "","source": ""},"end": "2023-11-11T09:50:31.252Z","object": {"bbox": {"bottomY": 144,"leftX": 1625,"topY": 72,"rightX": 1664},"location": {"lat": -0.0002441081085205078,"lon": -0.00026202797699034064,"alt": 0},"info": {"visibility": "0.991698","footLocation2D": "297.655,179.069","footLocation3D": "17.56687,20.29478","convexHull": "-19,-33,-18,-32,3,35"},"coordinate": {"z": 0,"y": 27.123123168945312,"x": 29.114219665527344},"id": "1052","confidence": 0.9485750198364258,"pose": null,"gaze": null,"type": "Person","lipActivity": null,"speed": 0,"dir": [],"embedding": null},"timestamp": "2023-11-11T09:50:31.218Z","id": "Sensor_1 #-# 1052","edges": [],"timeInterval": 0.034,"bearing": 239.8590251375903,"poses": [],"place": {"location": null,"name": "building=Building_1","type": "","coordinate": null,"info": {},"id": ""},"locations": {"type": "linestring","coordinates": [[29.199949264526367,27.270771026611328],[29.114219665527344,27.123123168945312]]},"info": {},"distance": 0.1707321118533282,"videoPath": "frameId-10696","direction": "Down","gazes": [],"speed": 11.232845266702702,"event": null,"smoothLocations": {"type": "linestring","coordinates": [[29.199949264526367,27.270771026611328],[29.114219665527344,27.123123168945312]]},"length": 2,"sensor": {"description": "Sensor_1","location": null,"type": "","coordinate": null,"info": {},"id": "Sensor_1"},"lipActivities": [],"speedOverTime": [11.232845266702702]}