Import and Fine-Tune Private HuggingFace Models

1
import json
2
# Note: This assumes MODEL_NAME, NAMESPACE, and hf_api are already defined from previous code blocks
3
4
# Example training data for Gemma 2 (instruction format)
5
# Gemma 2 uses standard chat message format
6
training_data = [
7
    {
8
        "messages": [
9
            {"role": "user", "content": "Hello, how are you?"},
10
            {
11
                "role": "assistant",
12
                "content": "I'm doing well, thank you for asking! How can I assist you today?",
13
            },
14
        ]
15
    },
16
    {
17
        "messages": [
18
            {"role": "user", "content": "What's the weather like?"},
19
            {
20
                "role": "assistant",
21
                "content": "I don't have access to current weather data, but I can help you find weather information or discuss weather patterns.",
22
            },
23
        ]
24
    },
25
    {
26
        "messages": [
27
            {"role": "user", "content": "Tell me a joke"},
28
            {
29
                "role": "assistant",
30
                "content": "Why don't scientists trust atoms? Because they make up everything!",
31
            },
32
        ]
33
    },
34
    {
35
        "messages": [
36
            {"role": "user", "content": "Can you help me with coding?"},
37
            {
38
                "role": "assistant",
39
                "content": "Absolutely! I'd be happy to help you with coding. What programming language or specific problem are you working on?",
40
            },
41
        ]
42
    },
43
    {
44
        "messages": [
45
            {"role": "user", "content": "What is machine learning?"},
46
            {
47
                "role": "assistant",
48
                "content": "Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed.",
49
            },
50
        ]
51
    },
52
    {
53
        "messages": [
54
            {"role": "user", "content": "How do I write a function in Python?"},
55
            {
56
                "role": "assistant",
57
                "content": "To write a function in Python, use the 'def' keyword followed by the function name and parameters: def my_function(param): return result",
58
            },
59
        ]
60
    },
61
    {
62
        "messages": [
63
            {"role": "user", "content": "What are the benefits of exercise?"},
64
            {
65
                "role": "assistant",
66
                "content": "Exercise offers numerous benefits including improved cardiovascular health, stronger muscles, better mental health, and increased energy levels.",
67
            },
68
        ]
69
    },
70
    {
71
        "messages": [
72
            {"role": "user", "content": "Explain the solar system"},
73
            {
74
                "role": "assistant",
75
                "content": "The solar system consists of the Sun and all celestial objects that orbit it, including eight planets, moons, asteroids, and comets.",
76
            },
77
        ]
78
    },
79
]
80
81
# Expand training data by repeating examples to create a larger dataset
82
training_data = training_data * 10  # 80 total examples (8 unique * 10 repetitions)
83
84
# Save training data to JSONL file
85
with open("training_data.jsonl", "w") as f:
86
    for item in training_data:
87
        f.write(json.dumps(item) + "\n")
88
89
# Create dataset in datastore (similar to model creation)
90
dataset_name = f"{MODEL_NAME}-training-data"
91
dataset_repo_id = f"{NAMESPACE}/{dataset_name}"
92
93
# Create dataset repository
94
hf_api.create_repo(dataset_repo_id, repo_type="dataset", exist_ok=True)
95
96
# Upload training data
97
hf_api.upload_file(
98
    path_or_fileobj="training_data.jsonl",
99
    path_in_repo="training.jsonl",
100
    repo_id=dataset_repo_id,
101
    repo_type="dataset",
102
    commit_message="Upload training data",
103
)
104
105
validation_data = [
106
    {
107
        "messages": [
108
            {"role": "user", "content": "How do you stay motivated?"},
109
            {
110
                "role": "assistant",
111
                "content": "Staying motivated involves setting clear goals, celebrating small wins, and maintaining a positive mindset even when facing challenges.",
112
            },
113
        ]
114
    },
115
    {
116
        "messages": [
117
            {"role": "user", "content": "What's the capital of France?"},
118
            {
119
                "role": "assistant",
120
                "content": "The capital of France is Paris, a beautiful city known for its art, culture, and iconic landmarks like the Eiffel Tower.",
121
            },
122
        ]
123
    },
124
    {
125
        "messages": [
126
            {"role": "user", "content": "Explain quantum physics"},
127
            {
128
                "role": "assistant",
129
                "content": "Quantum physics is the branch of physics that studies matter and energy at the smallest scales, where particles behave in ways that seem strange compared to our everyday experience.",
130
            },
131
        ]
132
    },
133
    {
134
        "messages": [
135
            {"role": "user", "content": "What is photosynthesis?"},
136
            {
137
                "role": "assistant",
138
                "content": "Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen, providing energy for the plant.",
139
            },
140
        ]
141
    },
142
    {
143
        "messages": [
144
            {"role": "user", "content": "How does the internet work?"},
145
            {
146
                "role": "assistant",
147
                "content": "The internet works through a global network of interconnected computers that communicate using standardized protocols to share information and resources.",
148
            },
149
        ]
150
    },
151
    {
152
        "messages": [
153
            {"role": "user", "content": "What is artificial intelligence?"},
154
            {
155
                "role": "assistant",
156
                "content": "Artificial intelligence is the simulation of human intelligence in machines that are programmed to think, learn, and solve problems like humans.",
157
            },
158
        ]
159
    },
160
    {
161
        "messages": [
162
            {"role": "user", "content": "Describe the water cycle"},
163
            {
164
                "role": "assistant",
165
                "content": "The water cycle is the continuous movement of water through evaporation, condensation, precipitation, and collection that sustains life on Earth.",
166
            },
167
        ]
168
    },
169
    {
170
        "messages": [
171
            {"role": "user", "content": "What are renewable energy sources?"},
172
            {
173
                "role": "assistant",
174
                "content": "Renewable energy sources include solar, wind, hydroelectric, geothermal, and biomass energy that can be naturally replenished and don't deplete over time.",
175
            },
176
        ]
177
    },
178
]
179
180
# Expand validation data by repeating examples to create a larger dataset
181
validation_data = validation_data * 10  # 80 total examples (8 unique * 10 repetitions)
182
183
# Save validation data to JSONL file
184
with open("validation_data.jsonl", "w") as f:
185
    for item in validation_data:
186
        f.write(json.dumps(item) + "\n")
187
188
# Upload validation data
189
hf_api.upload_file(
190
    path_or_fileobj="validation_data.jsonl",
191
    path_in_repo="validation.jsonl",
192
    repo_id=dataset_repo_id,
193
    repo_type="dataset",
194
    commit_message="Upload validation data",
195
)
196
197
# Create dataset entity in Entity Store
198
dataset = client.datasets.create(
199
    name=dataset_name, namespace=NAMESPACE, files_url=f"hf://datasets/{dataset_repo_id}"
200
)
201
print(f"Created dataset entity: {dataset.namespace}/{dataset.name}")