File Batcher Module
This module loads the input files, removes files that are older than the chosen window of time, and then groups the remaining files by period that fall inside the window.
Parameter |
Type |
Description |
Example Value |
Default Value |
---|---|---|---|---|
|
dictionary |
Options for batching |
See below |
|
|
string |
Cache directory |
“./file_batcher_cache” |
|
|
string |
File type |
“JSON” |
|
|
boolean |
Whether to filter null values |
false |
|
|
dictionary |
Data schema |
See below |
|
|
string |
Name of the timestamp column |
“timestamp” |
|
Key |
Type |
Description |
Example Value |
Default Value |
---|---|---|---|---|
|
datetime/string |
Endtime of the time window |
“2023-03-14T23:59:59” |
|
|
string |
Regex pattern for ISO date matching |
“\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}” |
|
|
dictionary |
Additional arguments for the parser |
{} |
|
|
string |
Time period for grouping files |
“1d” |
|
|
integer |
Sampling rate in seconds |
60 |
|
|
datetime/string |
Start time of the time window |
“2023-03-01T00:00:00” |
|
Key |
Type |
Description |
Example Value |
Default Value |
---|---|---|---|---|
|
string |
Encoding |
“latin1” |
|
|
string |
Schema string |
“string” |
|
{
"batching_options": {
"end_time": "2023-03-14T23:59:59",
"iso_date_regex_pattern": "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}",
"parser_kwargs": {},
"period": "1d",
"sampling_rate_s": 60,
"start_time": "2023-03-01T00:00:00"
},
"cache_dir": "./file_batcher_cache",
"file_type": "JSON",
"filter_nulls": false,
"schema": {
"schema_str": "string",
"encoding": "latin1"
},
"timestamp_column_name": "timestamp"
}