Seeding with External Datasets

1
import data_designer.config as dd
2
3
# Public dataset
4
dd.HuggingFaceSeedSource(path="datasets/username/dataset/data/*.parquet")
5
6
# Private dataset (requires token)
7
dd.HuggingFaceSeedSource(
8
    path="datasets/username/dataset/data/*.parquet",
9
    token="default/hf-token",  # Reference to a Secrets API secret
10
)

1
from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource
2
3
FilesetFileSeedSource(
4
    path="default/my-fileset#data.parquet"  # Format: workspace/fileset#file-path
5
)

1
import os
2
import tempfile
3
import urllib.request
4
from nemo_platform import NeMoPlatform
5
6
WORKSPACE = "default"
7
FILESET_NAME = "seed-data"
8
FILE_PATH = "symptom_to_diagnosis.csv"
9
10
base_url = os.environ.get("NMP_BASE_URL", "http://localhost:8080")
11
client = NeMoPlatform(base_url=base_url, workspace=WORKSPACE)
12
13
# Create fileset
14
client.files.filesets.create(name=FILESET_NAME)
15
16
# Download and upload seed data
17
with tempfile.NamedTemporaryFile(suffix=".csv") as tmpfile:
18
    url = "https://raw.githubusercontent.com/NVIDIA/GenerativeAIExamples/refs/heads/main/nemo/NeMo-Data-Designer/data/gretelai_symptom_to_diagnosis.csv"
19
    urllib.request.urlretrieve(url, tmpfile.name)
20
21
    client.files.upload(
22
        fileset=FILESET_NAME,
23
        local_path=tmpfile.name,
24
        remote_path=FILE_PATH,
25
    )

1
import data_designer.config as dd
2
3
MODEL_ALIAS = "text"
4
5
model_configs = [
6
    dd.ModelConfig(
7
        provider="default/nvidia-build",
8
        model="nvidia/nemotron-3-nano-30b-a3b",  # Use the `served_model_name` from the provider
9
        alias=MODEL_ALIAS,
10
        inference_parameters=dd.ChatCompletionInferenceParams(
11
            temperature=1.0,
12
            top_p=1.0,
13
        ),
14
    )
15
]
16
17
config_builder = dd.DataDesignerConfigBuilder(model_configs)

1
from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource
2
3
config_builder.with_seed_dataset(
4
    FilesetFileSeedSource(path=f"{WORKSPACE}/{FILESET_NAME}#{FILE_PATH}")
5
)

1
# Patient details
2
config_builder.add_column(
3
    dd.SamplerColumnConfig(
4
        name="patient_sampler",
5
        sampler_type=dd.SamplerType.PERSON_FROM_FAKER,
6
        params=dd.PersonFromFakerSamplerParams(),
7
    )
8
)
9
10
# Doctor details
11
config_builder.add_column(
12
    dd.SamplerColumnConfig(
13
        name="doctor_sampler",
14
        sampler_type=dd.SamplerType.PERSON_FROM_FAKER,
15
        params=dd.PersonFromFakerSamplerParams(),
16
    )
17
)
18
19
# Patient ID
20
config_builder.add_column(
21
    dd.SamplerColumnConfig(
22
        name="patient_id",
23
        sampler_type=dd.SamplerType.UUID,
24
        params=dd.UUIDSamplerParams(
25
            prefix="PT-",
26
            short_form=True,
27
            uppercase=True,
28
        ),
29
    )
30
)
31
32
# Extract patient name from sampler
33
config_builder.add_column(
34
    dd.ExpressionColumnConfig(
35
        name="first_name",
36
        expr="{{ patient_sampler.first_name }}",
37
    )
38
)
39
40
config_builder.add_column(
41
    dd.ExpressionColumnConfig(
42
        name="last_name",
43
        expr="{{ patient_sampler.last_name }}",
44
    )
45
)
46
47
config_builder.add_column(
48
    dd.ExpressionColumnConfig(
49
        name="dob",
50
        expr="{{ patient_sampler.birth_date }}",
51
    )
52
)
53
54
# Symptom onset date
55
config_builder.add_column(
56
    dd.SamplerColumnConfig(
57
        name="symptom_onset_date",
58
        sampler_type=dd.SamplerType.DATETIME,
59
        params=dd.DatetimeSamplerParams(start="2024-01-01", end="2024-12-31"),
60
    )
61
)
62
63
# Visit date (1-30 days after symptom onset)
64
config_builder.add_column(
65
    dd.SamplerColumnConfig(
66
        name="date_of_visit",
67
        sampler_type=dd.SamplerType.TIMEDELTA,
68
        params=dd.TimeDeltaSamplerParams(
69
            dt_min=1,
70
            dt_max=30,
71
            reference_column_name="symptom_onset_date",
72
        ),
73
    )
74
)
75
76
# Physician name
77
config_builder.add_column(
78
    dd.ExpressionColumnConfig(
79
        name="physician",
80
        expr="Dr. {{ doctor_sampler.last_name }}",
81
    )
82
)
83
84
# LLM-generated physician notes
85
config_builder.add_column(
86
    dd.LLMTextColumnConfig(
87
        name="physician_notes",
88
        prompt="""\
89
You are a primary-care physician who just had an appointment with {{ first_name }} {{ last_name }},
90
who has been struggling with symptoms from {{ diagnosis }} since {{ symptom_onset_date }}.
91
The date of today's visit is {{ date_of_visit }}.
92
93
{{ patient_summary }}
94
95
Write careful notes about your visit with {{ first_name }},
96
as Dr. {{ doctor_sampler.first_name }} {{ doctor_sampler.last_name }}.
97
98
Format the notes as a busy doctor might.
99
Respond with only the notes, no other text.
100
""",
101
        model_alias=MODEL_ALIAS,
102
    )
103
)

1
def load_config_builder() -> dd.DataDesignerConfigBuilder:
2
    return config_builder

1
# Using the client instance from Step 1
2
data_designer = client.data_designer

1
preview = data_designer.preview(config_builder)
2
3
# Display a random sample record
4
preview.display_sample_record()
5
6
# Access the full preview dataset as a pandas DataFrame
7
df = preview.dataset
8
print(df.head())
9
10
# View statistical analysis
11
preview.analysis.to_report()

1
# Defaulting to 30 for demo speed purposes. Happy with the output? Scale it up!
2
job = data_designer.create(config_builder, num_records=30)
3
4
# Block until the job completes
5
job.wait_until_done()
6
7
# Download the generated artifacts
8
results = job.download_artifacts()
9
10
# Load the dataset as a pandas DataFrame
11
dataset = results.load_dataset()
12
print(dataset.head())
13
14
# Load the full analysis report
15
analysis = results.load_analysis()
16
analysis.to_report()