diff --git a/hugo/content/en/gpu_monitoring/tracing.md b/hugo/content/en/gpu_monitoring/tracing/_index.md similarity index 90% rename from hugo/content/en/gpu_monitoring/tracing.md rename to hugo/content/en/gpu_monitoring/tracing/_index.md index b287e0a8350..18411dcf193 100644 --- a/hugo/content/en/gpu_monitoring/tracing.md +++ b/hugo/content/en/gpu_monitoring/tracing/_index.md @@ -4,6 +4,9 @@ is_beta: true private: true description: Enable GPU activity tracing for selected Kubernetes workloads. further_reading: +- link: "/gpu_monitoring/tracing/compatibility" + tag: "Documentation" + text: "GPU tracing compatibility" - link: "/gpu_monitoring/setup" tag: "Documentation" text: "Set up GPU Monitoring" @@ -12,9 +15,9 @@ further_reading: text: "Learn more about what GPU Monitoring offers" --- -{{< beta-callout url="#" btn_hidden="true" >}} +{{< callout url="#" btn_hidden="true" >}} Continuous tracing with GPU Monitoring is in Early Access Preview. -{{< /beta-callout >}} +{{< /callout >}} ## Overview @@ -28,7 +31,7 @@ Continuous tracing with GPU Monitoring enables lightweight GPU activity tracing To begin continuously tracing your workloads, you must first meet the following criteria: - You are running the Datadog Cluster Agent version 7.80 or later with [GPU Monitoring enabled][1]. -- Minimum required CUDA and CUPTI version: 13. +- Minimum required CUDA and CUPTI version: 13. For supported tracer and framework versions, see [GPU tracing compatibility][3]. ### 1. Configure GPU tracing @@ -99,3 +102,4 @@ pod_name: kube_namespace: [1]: /gpu_monitoring/setup [2]: /tracing/trace_explorer/ +[3]: /gpu_monitoring/tracing/compatibility/ diff --git a/hugo/content/en/gpu_monitoring/tracing/compatibility.md b/hugo/content/en/gpu_monitoring/tracing/compatibility.md new file mode 100644 index 00000000000..90740761a69 --- /dev/null +++ b/hugo/content/en/gpu_monitoring/tracing/compatibility.md @@ -0,0 +1,43 @@ +--- +title: GPU Tracing Compatibility +is_beta: true +private: true +description: Review supported CUDA, CUPTI, tracer, and framework versions for GPU tracing. +further_reading: +- link: "/gpu_monitoring/tracing" + tag: "Documentation" + text: "Set up continuous tracing with GPU Monitoring" +- link: "/gpu_monitoring/setup" + tag: "Documentation" + text: "Set up GPU Monitoring" +--- + +{{< callout url="#" btn_hidden="true" >}} +Continuous tracing with GPU Monitoring is in Early Access Preview. +{{< /callout >}} + +GPU tracing supports CUDA 13.x with CUPTI 13.x. CUDA 12.x and earlier are not supported. + +## Tracer versions + +| C tracer version | CUDA | CUPTI | Framework coverage | Important changes | +|---|---|---|---|---| +| 0.18.0 | 13.x | 13.x | PyTorch, vLLM, and TensorRT-LLM | Adds vLLM request correlation and improves profiling efficiency for short workloads. | +| 0.19.0 | 13.x | 13.x | PyTorch, vLLM, and TensorRT-LLM | Improves startup reliability across PyTorch and CUDA environments. | +| 0.19.1 | 13.x | 13.x | PyTorch, vLLM, and TensorRT-LLM | Improves completeness and accuracy of NCCL activity metrics. | +| 0.20.0 | 13.x | 13.x | PyTorch, vLLM, TensorRT-LLM, and SGLang | Adds SGLang inference-step GPU tracing. | +| 0.21.0 | 13.x | 13.x | PyTorch, vLLM, TensorRT-LLM, and SGLang | Adds CUDA Graph identity to kernel spans and reduces sampling overhead. | +| 0.21.1 | 13.x | 13.x | PyTorch, vLLM, TensorRT-LLM, and SGLang | Reduces workload overhead and improves GPU tracing reliability across x86_64 and ARM64. **Recommended.** | + +## Minimum framework versions + +| Framework | Minimum version | C tracer version | Coverage | +|---|---|---|---| +| PyTorch | 2.8.0 | 0.18.0+ | Training, inference, CUDA, and NCCL activity | +| vLLM | 0.11.0 | 0.18.0+ | Prefill and decode boundaries with request correlation | +| TensorRT-LLM | 1.2.0 | 0.18.0+ | PyTorch executor prefill and decode tracing | +| SGLang | 0.5.10.post1 | 0.20.0+ | Prefill, decode, and mixed inference tracing | + +## Further reading + +{{< partial name="whats-next/whats-next.html" >}}