Skip to navigation

Protobuf Schema

View as Markdown

The protobuf schemas are defined using the two files as described below.

  • schema.proto contains the Frames definition which is used by perception to send metadata
  • ext.proto contains the Behavior definition which is used the by behavior analytics

schema.proto

An example of json frame object which follows the above protobuf schema can be found below

{
"version": "4.0",
"timestamp": "2023-10-30T22:30:30.000Z",
"id": "1",
"sensorId": "Sensor_1",
"objects": [
{
"dir": [],
"lipActivity": null,
"info": {
"visibility": "0.974372",
"footLocation2D": "899.152,316.812",
"footLocation3D": "9.85312,-12.53092",
"convexHull": "10,-4,23,-19,7,20,-18,26"
},
"coordinate": null,
"speed": 0,
"bbox": {
"topY": 107.74882507324219,
"rightX": 1116.9134521484375,
"bottomY": 311.5032653808594,
"leftX": 1063.572021484375
},
"embedding": {
"info": {},
"vector": [
-0.003528113476932049,
-0.002565900795161724,
-0.020954856649041176,
0.023293567821383476
]
},
"location": null,
"gaze": null,
"id": "1",
"confidence": 0.9556514024734497,
"pose": null,
"type": "Person"
},
{
"dir": [],
"lipActivity": null,
"info": {
"visibility": "0.991698",
"footLocation2D": "297.655,179.069",
"footLocation3D": "17.56687,20.29478",
"convexHull": "-19,-33,-18,-32,3,35"
},
"coordinate": null,
"speed": 0,
"bbox": {
"topY": 115.80743408203125,
"rightX": 861.8057861328125,
"bottomY": 328.8199768066406,
"leftX": 814.3302001953125
},
"embedding": {
"info": {},
"vector": [
0.0002149203501176089,
-0.023472599685192108,
-0.02617083303630352,
0.022965412586927414
]
},
"location": null,
"gaze": null,
"id": "2",
"confidence": 0.9136765003204346,
"pose": null,
"type": "Person"
}
]
}

ext.proto

syntax = "proto3";
package nv;
option java_package = "nv.schema";
import "google/protobuf/timestamp.proto";
import "schema.proto";
//import "struct.proto";
/*
* represents locations array, for geo coodinates Array[Array[lon, lat]], for cartesian coordinates Array[Array[x,y]]
*/
message GeoLocation{
string type = 1;
/*
Array(x,y)
*/
message Point{
repeated double point = 1;
}
/*
Array of points
*/
repeated Point coordinates = 2;
//repeated google.protobuf.ListValue coordinates = 2;
}
/*
* represent object Behavior, and comprise of attributes from object movement + [[Object]] appearance
* with highest confidence + [[Sensor]] + [[Place]]
* + [[Pose]] + [[LipActivity]] + [[Gaze]]
*
*
*
*
* the JSON representation of this object Behavior message is stored in persistent store like Elasticsearch for search and indexing
*
*
*/
message Behavior{
/*
* object id, usually a combination of sensorId + objectId
*/
string id = 1;
/*
* start timestamp
*/
google.protobuf.Timestamp timestamp = 2;
/*
* end timestamp
*/
google.protobuf.Timestamp end = 3;
// start timestamp
//google.protobuf.Timestamp timestamp = 4;
/*
* representing array of [lon,lat] or [x,y]
*/
GeoLocation locations = 5;
/*
* representing smoothened array of [lon,lat] or [x,y]
*/
GeoLocation smoothLocations = 6;
/*
* road network edges, only applicable for geo coordinates
*/
repeated string edges = 7;
/*
* distance traveled in meters
*/
double distance = 8;
/*
* avg speed in mph
*/
double speed = 9;
/*
* speed over a period of the trajectory
*/
repeated double speedOverTime = 10;
/*
* timeInterval, end time - start time
*/
double timeInterval = 11;
/*
* bearing angle
*/
double bearing = 12;
/*
* N, S, E or W based on bearing
*/
string direction = 13;
/*
* number of raw coordinates
*/
int32 length = 14;
/*
* where object was seen by the sensor
*/
Place place = 15;
/*
* sensor details
*/
Sensor sensor = 16;
/*
* AnalyticsModule
*/
AnalyticsModule analyticsModule = 17;
/*
* object details, associated with this trajectory
*/
Object object = 18;
/*
* event details
*/
Event event = 19;
/*
* video url if available
*/
string videoPath = 20;
/*
* Array of pose
*/
repeated Pose poses = 21;
/*
* Array of LipActivity
*/
repeated LipActivity lipActivities = 22;
/*
* Array of Gaze
*/
repeated Gaze gazes = 23;
/*
* Array of embeddings
*/
repeated Embedding embeddings = 24;
/*
* LLM query, response, embeddings for behavior analysis and anomaly detection
*/
LLM llm = 26;
/*
* addition information, that may be needed by an application
*/
map<string, string> info = 25;
}
message Incident{
/*
* sensor id
*/
string sensorId = 1;
/*
* start timestamp
*/
google.protobuf.Timestamp timestamp = 2;
/*
* end timestamp
*/
google.protobuf.Timestamp end = 3;
/*
* Array of object IDs involved in the incident. If the array is not empty, then the first ID represents the primary object.
* May be empty for incidents that don't involve any specific objects.
*/
repeated string objectIds = 4;
/*
* Array of frameIds when the incident took place (Optional)
*/
repeated string frameIds = 5;
/*
* where object was seen by the sensor
*/
Place place = 6;
/*
* AnalyticsModule: Module responsible for generating the Incident
*/
AnalyticsModule analyticsModule = 7;
/*
* Category of the incident
*/
string category = 8;
/*
* Embeddings can be related to object appearance or it could be representative of a scene description provided by VLM
*/
repeated Embedding embeddings = 9;
/*
* Is the incident an anomaly
*/
bool isAnomaly = 10;
/*
* LLM query, response, embeddings for incident analysis and anomaly detection
*/
LLM llm = 12;
/*
* optional attributes
*/
map<string, string> info = 11;
}
message SpaceUtilizationMetrics{
/*
* area size of the occupied space (in square meters)
*/
double spaceOccupied = 1;
/*
* area size of the free space (in square meters)
*/
double freeSpace = 2;
/*
* area size of the total space (in square meters)
*/
double totalSpace = 3;
/*
* the ratio/percentage that the total space is utilized/occupied
*/
double spaceUtilization = 4;
/*
* the max number of extra pallets that can fit into the free space
*/
int32 numExtraPallets = 5;
/*
* area size of the utilizable free space
*/
double utilizableFreeSpace = 6;
/*
* the ratio/percentage that the free space is utilizable
*/
double freeSpaceQuality = 7;
/*
* flag to indicate if the space is safe
*/
bool isUnsafe = 8;
}
message SpaceUtilizationLayouts{
/*
* layout of the free space
*/
repeated Polygon freeSpace = 1;
/*
* layout of the utilizable free space
*/
repeated Polygon utilizableFreeSpace = 2;
}
message SpaceUtilization{
/*
* ROI id
*/
string id = 1;
/*
* timestamp in UTC format
*/
google.protobuf.Timestamp timestamp = 2;
/*
* Metrics regarding utilization of space
*/
SpaceUtilizationMetrics metrics = 3;
/*
* Sensors associated with the ROI
*/
repeated string sensors = 4;
/*
* Layouts of free space and utilizable free space
*/
SpaceUtilizationLayouts layouts = 5;
}

An example of json behavior object which follows the above protobuf schema can be found below

{
"analyticsModule": {
"description": "",
"info": {},
"version": "",
"id": "",
"source": ""
},
"end": "2023-11-11T09:50:31.252Z",
"object": {
"bbox": {
"bottomY": 144,
"leftX": 1625,
"topY": 72,
"rightX": 1664
},
"location": {
"lat": -0.0002441081085205078,
"lon": -0.00026202797699034064,
"alt": 0
},
"info": {
"visibility": "0.991698",
"footLocation2D": "297.655,179.069",
"footLocation3D": "17.56687,20.29478",
"convexHull": "-19,-33,-18,-32,3,35"
},
"coordinate": {
"z": 0,
"y": 27.123123168945312,
"x": 29.114219665527344
},
"id": "1052",
"confidence": 0.9485750198364258,
"pose": null,
"gaze": null,
"type": "Person",
"lipActivity": null,
"speed": 0,
"dir": [],
"embedding": null
},
"timestamp": "2023-11-11T09:50:31.218Z",
"id": "Sensor_1 #-# 1052",
"edges": [],
"timeInterval": 0.034,
"bearing": 239.8590251375903,
"poses": [],
"place": {
"location": null,
"name": "building=Building_1",
"type": "",
"coordinate": null,
"info": {},
"id": ""
},
"locations": {
"type": "linestring",
"coordinates": [
[
29.199949264526367,
27.270771026611328
],
[
29.114219665527344,
27.123123168945312
]
]
},
"info": {},
"distance": 0.1707321118533282,
"videoPath": "frameId-10696",
"direction": "Down",
"gazes": [],
"speed": 11.232845266702702,
"event": null,
"smoothLocations": {
"type": "linestring",
"coordinates": [
[
29.199949264526367,
27.270771026611328
],
[
29.114219665527344,
27.123123168945312
]
]
},
"length": 2,
"sensor": {
"description": "Sensor_1",
"location": null,
"type": "",
"coordinate": null,
"info": {},
"id": "Sensor_1"
},
"lipActivities": [],
"speedOverTime": [
11.232845266702702
]
}