Automation and CI/CD Integration

1
# GitHub Actions
2
name: Configuration Drift Detection
3
on:
4
  schedule:
5
    - cron: '0 */6 * * *'  # Every 6 hours
6
7
jobs:
8
  snapshot:
9
    runs-on: ubuntu-latest
10
    steps:
11
      - name: Configure kubectl
12
        uses: azure/k8s-set-context@v4
13
        with:
14
          kubeconfig: ${{ secrets.KUBECONFIG }}
15
      
16
      # On AKS, the gpuStack profile qualifies against the
17
      # K8s.aks-gpu-pools.gpu-driver reading — capture it BEFORE the
18
      # snapshot step or later snapshot-qualified resolution / validate
19
      # readiness fails closed. Export the path once; snapshot/validate
20
      # pick it up via AICR_AKS_GPU_POOLS_PATH. (Profile selection happens
21
      # at recipe generation — see Pattern 2.)
22
      # - name: Dump AKS GPU pool modes (AKS clusters only)
23
      #   run: |
24
      #     az aks nodepool list -g "$RG" --cluster-name "$CLUSTER" -o json \
25
      #       > "$GITHUB_WORKSPACE/aks-gpu-pools.json"
26
      #     echo "AICR_AKS_GPU_POOLS_PATH=$GITHUB_WORKSPACE/aks-gpu-pools.json" >> "$GITHUB_ENV"
27
28
      - name: Deploy AICR Agent
29
        run: |
30
          # aicr snapshot deploys the in-cluster Job, waits synchronously for
31
          # it to complete, writes the result, then cleans the Job up — no
32
          # separate `kubectl wait` step is needed.
33
          aicr snapshot --namespace gpu-operator \
34
            --output cm://gpu-operator/aicr-snapshot --timeout 300s
35
36
      - name: Capture snapshot from ConfigMap
37
        run: |
38
          kubectl get configmap aicr-snapshot -n gpu-operator -o jsonpath='{.data.snapshot\.yaml}' > snapshot.yaml
39
      
40
      - name: Compare with baseline
41
        run: |
42
          # Download baseline
43
          curl -O https://your-artifacts/baseline.yaml
44
45
          # Compare with the semantic differ. A raw `diff` would flag every
46
          # new snapshot timestamp as drift; `aicr diff --fail-on-drift`
47
          # exits non-zero only on a meaningful configuration change.
48
          aicr diff --baseline baseline.yaml --target snapshot.yaml \
49
            --fail-on-drift
50
      
51
      - name: Upload artifact
52
        uses: actions/upload-artifact@v4
53
        with:
54
          name: cluster-snapshots
55
          path: snapshot.yaml

1
# GitHub Actions
2
name: GPU Stack Deploy
3
on:
4
  workflow_dispatch:
5
6
jobs:
7
  deploy:
8
    runs-on: ubuntu-latest
9
    steps:
10
      - name: Configure kubectl
11
        uses: azure/k8s-set-context@v4
12
        with:
13
          kubeconfig: ${{ secrets.KUBECONFIG }}
14
15
      # 1. Snapshot: agent Job writes cluster state to a ConfigMap.
16
      #    aicr snapshot waits synchronously for the Job and cleans it up,
17
      #    so no separate `kubectl wait` step is needed.
18
      #    On AKS, dump the GPU pool modes and export AICR_AKS_GPU_POOLS_PATH
19
      #    BEFORE this step (see Pattern 1's "Dump AKS GPU pool modes").
20
      - name: Capture snapshot
21
        run: |
22
          aicr snapshot --namespace gpu-operator \
23
            --output cm://gpu-operator/aicr-snapshot --timeout 300s
24
25
      # 2. Recipe: read the snapshot ConfigMap, emit an optimized recipe.
26
      #    Use --service/--accelerator/--intent flags for query mode instead.
27
      #    AKS: add `--profile gpuStack=operator-managed` when the GPU pools were
28
      #    created with --gpu-driver none (the default azure-managed value
29
      #    requires pools reading Install and fails closed otherwise).
30
      - name: Generate recipe
31
        run: |
32
          aicr recipe \
33
            --snapshot cm://gpu-operator/aicr-snapshot \
34
            --intent training \
35
            --platform kubeflow \
36
            --output recipe.yaml
37
38
      # 3. Bundle: render deployment artifacts. Add --set to override values,
39
      #    or --deployer argocd for GitOps output (see Pattern 3).
40
      - name: Create bundle
41
        run: aicr bundle --recipe recipe.yaml --output ./bundles
42
43
      # 4. Deploy: verify the closed-world inventory, then run the generated script.
44
      - name: Deploy
45
        run: |
46
          cd bundles
47
          aicr verify .
48
          chmod +x deploy.sh
49
          ./deploy.sh

1
# GitHub Actions
2
name: GitOps Deploy with Argo CD
3
on:
4
  push:
5
    branches: [main]
6
7
jobs:
8
  generate-and-deploy:
9
    runs-on: ubuntu-latest
10
    steps:
11
      - name: Checkout
12
        uses: actions/checkout@v4
13
      
14
      - name: Setup aicr
15
        run: |
16
          # GoReleaser archives are versioned (aicr_<version>_<os>_<arch>.tar.gz),
17
          # so resolve the latest tag first rather than a fixed filename.
18
          VERSION=$(curl -s https://api.github.com/repos/NVIDIA/aicr/releases/latest | jq -r '.tag_name')
19
          curl -sLO "https://github.com/NVIDIA/aicr/releases/download/${VERSION}/aicr_${VERSION#v}_linux_amd64.tar.gz"
20
          tar -xzf "aicr_${VERSION#v}_linux_amd64.tar.gz"
21
          sudo mv aicr /usr/local/bin/
22
      
23
      - name: Generate recipe
24
        run: |
25
          aicr recipe \
26
            --service eks \
27
            --accelerator h100 \
28
            --intent training \
29
            --os ubuntu \
30
            --output recipe.yaml
31
      
32
      - name: Generate Argo CD bundles
33
        run: |
34
          aicr bundle \
35
            --recipe recipe.yaml \
36
            --deployer argocd \
37
            --repo https://github.com/${{ github.repository }}.git \
38
            --output ./bundles
39
      
40
      - name: Commit to GitOps repo
41
        run: |
42
          # Copy entire bundle to GitOps repository
43
          # Argo CD apps are in NNN-<component>/application.yaml files
44
          # app-of-apps.yaml is at bundle root
45
          cp -r bundles/* gitops-repo/
46
          
47
          cd gitops-repo
48
          git add .
49
          git commit -m "Update GPU stack components"
50
          git push

1
# bundles/NNN-gpu-operator/application.yaml
2
apiVersion: argoproj.io/v1alpha1
3
kind: Application
4
metadata:
5
  name: gpu-operator
6
  namespace: argocd
7
  annotations:
8
    argocd.argoproj.io/sync-wave: "1"  # Deployed after cert-manager (wave 0)
9
spec:
10
  project: default
11
  sources:
12
    # Helm chart from upstream
13
    - repoURL: https://helm.ngc.nvidia.com/nvidia
14
      chart: gpu-operator
15
      targetRevision: v26.3.3
16
      helm:
17
        valueFiles:
18
          # Values live under the numbered bundle dir (NNN-<component>/)
19
          - $values/002-gpu-operator/values.yaml
20
    # Values from GitOps repo
21
    - repoURL: <YOUR_GIT_REPO>
22
      targetRevision: main
23
      ref: values
24
  # Post-install manifests (ClusterPolicy, etc.) are NOT a source on this
25
  # Application — the bundler emits them as a separate numbered Application
26
  # (e.g. NNN-gpu-operator-post/application.yaml) ordered by sync-wave.
27
  destination:
28
    server: https://kubernetes.default.svc
29
    namespace: gpu-operator
30
  syncPolicy:
31
    automated:
32
      prune: true
33
      selfHeal: true
34
    syncOptions:
35
      - CreateNamespace=true

1
# prometheus-config.yaml
2
scrape_configs:
3
  - job_name: 'aicrd'
4
    static_configs:
5
      - targets: ['aicrd.default.svc.cluster.local:8080']
6
    metrics_path: /metrics

1
# prometheus-rules.yaml
2
groups:
3
  - name: aicr_alerts
4
    interval: 30s
5
    rules:
6
      - alert: AICRHighErrorRate
7
        expr: |
8
          rate(aicr_http_requests_total{status=~"5.."}[5m]) > 0.05
9
        for: 5m
10
        labels:
11
          severity: warning
12
        annotations:
13
          summary: "AICR API high error rate"
14
          description: "Error rate is {{ $value | humanizePercentage }}"
15
      
16
      - alert: AICRHighLatency
17
        expr: |
18
          histogram_quantile(0.95,
19
            rate(aicr_http_request_duration_seconds_bucket[5m])
20
          ) > 1
21
        for: 5m
22
        labels:
23
          severity: warning
24
        annotations:
25
          summary: "AICR API high latency"
26
          description: "P95 latency is {{ $value }}s"
27
      
28
      - alert: AICRRateLimitHit
29
        expr: |
30
          rate(aicr_rate_limit_rejects_total[5m]) > 1
31
        for: 5m
32
        labels:
33
          severity: info
34
        annotations:
35
          summary: "AICR API rate limit reached"
36
          description: "Rate limit rejections: {{ $value }}/s"

1
apiVersion: networking.k8s.io/v1
2
kind: NetworkPolicy
3
metadata:
4
  name: aicr-agent
5
  namespace: gpu-operator
6
spec:
7
  podSelector:
8
    matchLabels:
9
      job-name: aicr
10
  policyTypes:
11
    - Egress
12
  egress:
13
    - to:
14
        - namespaceSelector: {}
15
      ports:
16
        - protocol: TCP
17
          port: 443  # Kubernetes API