Automation and CI/CD Integration

1
# GitHub Actions
2
name: Configuration Drift Detection
3
on:
4
  schedule:
5
    - cron: '0 */6 * * *'  # Every 6 hours
6
7
jobs:
8
  snapshot:
9
    runs-on: ubuntu-latest
10
    steps:
11
      - name: Configure kubectl
12
        uses: azure/k8s-set-context@v4
13
        with:
14
          kubeconfig: ${{ secrets.KUBECONFIG }}
15
      
16
      - name: Deploy AICR Agent
17
        run: |
18
          aicr snapshot --output cm://gpu-operator/aicr-snapshot --timeout 300s
19
      
20
      - name: Wait for completion
21
        run: |
22
          kubectl wait --for=condition=complete --timeout=300s job/aicr -n gpu-operator
23
      
24
      - name: Capture snapshot from ConfigMap
25
        run: |
26
          kubectl get configmap aicr-snapshot -n gpu-operator -o jsonpath='{.data.snapshot\.yaml}' > snapshot-$(date +%Y%m%d-%H%M%S).yaml
27
      
28
      - name: Compare with baseline
29
        run: |
30
          # Download baseline
31
          curl -O https://your-artifacts/baseline.yaml
32
          
33
          # Compare
34
          if ! diff -q baseline.yaml snapshot-*.yaml; then
35
            echo "::error::Configuration drift detected"
36
            diff baseline.yaml snapshot-*.yaml
37
            exit 1
38
          fi
39
      
40
      - name: Upload artifact
41
        uses: actions/upload-artifact@v4
42
        with:
43
          name: cluster-snapshots
44
          path: snapshot-*.yaml

1
# GitHub Actions
2
name: GPU Stack Deploy
3
on:
4
  workflow_dispatch:
5
6
jobs:
7
  deploy:
8
    runs-on: ubuntu-latest
9
    steps:
10
      - name: Configure kubectl
11
        uses: azure/k8s-set-context@v4
12
        with:
13
          kubeconfig: ${{ secrets.KUBECONFIG }}
14
15
      # 1. Snapshot: agent Job writes cluster state to a ConfigMap.
16
      - name: Capture snapshot
17
        run: |
18
          aicr snapshot --output cm://gpu-operator/aicr-snapshot --timeout 300s
19
          kubectl wait --for=condition=complete --timeout=300s \
20
            job/aicr -n gpu-operator
21
22
      # 2. Recipe: read the snapshot ConfigMap, emit an optimized recipe.
23
      #    Use --service/--accelerator/--intent flags for query mode instead.
24
      - name: Generate recipe
25
        run: |
26
          aicr recipe \
27
            --snapshot cm://gpu-operator/aicr-snapshot \
28
            --intent training \
29
            --platform kubeflow \
30
            --output recipe.yaml
31
32
      # 3. Bundle: render deployment artifacts. Add --set to override values,
33
      #    or --deployer argocd for GitOps output (see Pattern 3).
34
      - name: Create bundle
35
        run: aicr bundle --recipe recipe.yaml --output ./bundles
36
37
      # 4. Deploy: verify checksums, then run the generated script.
38
      - name: Deploy
39
        run: |
40
          cd bundles
41
          sha256sum -c checksums.txt
42
          chmod +x deploy.sh
43
          ./deploy.sh

1
# GitHub Actions
2
name: GitOps Deploy with Argo CD
3
on:
4
  push:
5
    branches: [main]
6
7
jobs:
8
  generate-and-deploy:
9
    runs-on: ubuntu-latest
10
    steps:
11
      - name: Checkout
12
        uses: actions/checkout@v4
13
      
14
      - name: Setup aicr
15
        run: |
16
          curl -sLO https://github.com/nvidia/aicr/releases/latest/download/aicr_linux_amd64.tar.gz
17
          tar -xzf aicr_linux_amd64.tar.gz
18
          sudo mv aicr /usr/local/bin/
19
      
20
      - name: Generate recipe
21
        run: |
22
          aicr recipe \
23
            --service eks \
24
            --accelerator h100 \
25
            --intent training \
26
            --os ubuntu \
27
            --output recipe.yaml
28
      
29
      - name: Generate Argo CD bundles
30
        run: |
31
          aicr bundle \
32
            --recipe recipe.yaml \
33
            --deployer argocd \
34
            --repo https://github.com/${{ github.repository }}.git \
35
            --output ./bundles
36
      
37
      - name: Commit to GitOps repo
38
        run: |
39
          # Copy entire bundle to GitOps repository
40
          # Argo CD apps are in <component>/argocd/ directories
41
          # app-of-apps.yaml is at bundle root
42
          cp -r bundles/* gitops-repo/
43
          
44
          cd gitops-repo
45
          git add .
46
          git commit -m "Update GPU stack components"
47
          git push

1
# bundles/gpu-operator/argocd/application.yaml
2
apiVersion: argoproj.io/v1alpha1
3
kind: Application
4
metadata:
5
  name: gpu-operator
6
  namespace: argocd
7
  annotations:
8
    argocd.argoproj.io/sync-wave: "1"  # Deployed after cert-manager (wave 0)
9
spec:
10
  project: default
11
  sources:
12
    # Helm chart from upstream
13
    - repoURL: https://helm.ngc.nvidia.com/nvidia
14
      chart: gpu-operator
15
      targetRevision: v26.3.2
16
      helm:
17
        valueFiles:
18
          - $values/gpu-operator/values.yaml
19
    # Values from GitOps repo
20
    - repoURL: <YOUR_GIT_REPO>
21
      targetRevision: main
22
      ref: values
23
    # Additional manifests (ClusterPolicy, etc.)
24
    - repoURL: <YOUR_GIT_REPO>
25
      targetRevision: main
26
      path: gpu-operator/manifests
27
  destination:
28
    server: https://kubernetes.default.svc
29
    namespace: gpu-operator
30
  syncPolicy:
31
    automated:
32
      prune: true
33
      selfHeal: true
34
    syncOptions:
35
      - CreateNamespace=true

1
# prometheus-config.yaml
2
scrape_configs:
3
  - job_name: 'aicrd'
4
    static_configs:
5
      - targets: ['aicrd.default.svc.cluster.local:8080']
6
    metrics_path: /metrics

1
# prometheus-rules.yaml
2
groups:
3
  - name: aicr_alerts
4
    interval: 30s
5
    rules:
6
      - alert: AICRHighErrorRate
7
        expr: |
8
          rate(aicr_http_requests_total{status=~"5.."}[5m]) > 0.05
9
        for: 5m
10
        labels:
11
          severity: warning
12
        annotations:
13
          summary: "AICR API high error rate"
14
          description: "Error rate is {{ $value | humanizePercentage }}"
15
      
16
      - alert: AICRHighLatency
17
        expr: |
18
          histogram_quantile(0.95,
19
            rate(aicr_http_request_duration_seconds_bucket[5m])
20
          ) > 1
21
        for: 5m
22
        labels:
23
          severity: warning
24
        annotations:
25
          summary: "AICR API high latency"
26
          description: "P95 latency is {{ $value }}s"
27
      
28
      - alert: AICRRateLimitHit
29
        expr: |
30
          rate(aicr_rate_limit_rejects_total[5m]) > 1
31
        for: 5m
32
        labels:
33
          severity: info
34
        annotations:
35
          summary: "AICR API rate limit reached"
36
          description: "Rate limit rejections: {{ $value }}/s"

1
apiVersion: networking.k8s.io/v1
2
kind: NetworkPolicy
3
metadata:
4
  name: aicr-agent
5
  namespace: gpu-operator
6
spec:
7
  podSelector:
8
    matchLabels:
9
      job-name: aicr
10
  policyTypes:
11
    - Egress
12
  egress:
13
    - to:
14
        - namespaceSelector: {}
15
      ports:
16
        - protocol: TCP
17
          port: 443  # Kubernetes API