Modern AI and HPC systems use 2–8 GPUs connected via PCIe or NVLink. CUDA exposes each GPU as a separate device; you must explicitly manage which device is active and how data moves between them.
int deviceCount;
cudaGetDeviceCount(&deviceCount);
for (int i = 0; i < deviceCount; i++) {
cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, i);
printf("Device %d: %s, %.0f GB, %d SMs, PCIe gen%d x%d\n",
i, prop.name,
prop.totalGlobalMem / 1e9,
prop.multiProcessorCount,
prop.pciBusID, // useful for identifying physical slots
prop.pciDeviceID);
}
Each host thread has a current device — all CUDA calls apply to it:
cudaSetDevice(0); // GPU 0 is active
float *d0; cudaMalloc(&d0, bytes);
cudaSetDevice(1); // Switch to GPU 1
float *d1; cudaMalloc(&d1, bytes);
For multi-threaded host code, each thread should call cudaSetDevice independently. Consider using one CPU thread per GPU.
When GPUs are connected via NVLink or PCIe peer mapping, they can read/write each other's memory directly — no host staging copy needed:
int canAccess;
cudaDeviceCanAccessPeer(&canAccess, 0, 1); // can GPU 0 access GPU 1?
if (canAccess) {
cudaSetDevice(0);
cudaDeviceEnablePeerAccess(1, 0); // GPU 0 can now read d1
// Direct GPU0 → GPU1 copy (goes over NVLink, not PCIe+host)
cudaMemcpyPeer(d1, 1, d0, 0, bytes);
}
NVLink bandwidth (A100): 600 GB/s bidirectional (vs ~32 GB/s PCIe Gen4 x16).
On systems with UVA, all device and pinned host pointers occupy a single virtual address space. cudaMemcpy infers direction from the pointer value — no need to specify cudaMemcpyPeer:
// With UVA active (Pascal+ on 64-bit Linux), direction is automatic
cudaMemcpy(d1, d0, bytes, cudaMemcpyDefault); // cudaMemcpyDefault infers direction
For distributed reductions, broadcasts, and all-reduce (the core of data-parallel training), use NCCL:
#include <nccl.h>
ncclComm_t comms[NUM_GPUS];
int devs[NUM_GPUS] = {0, 1, 2, 3};
ncclCommInitAll(comms, NUM_GPUS, devs);
// All-reduce: sum d_buf across all GPUs, result on every GPU
ncclGroupStart();
for (int i = 0; i < NUM_GPUS; i++) {
cudaSetDevice(devs[i]);
ncclAllReduce(d_buf[i], d_buf[i], N, ncclFloat, ncclSum, comms[i], streams[i]);
}
ncclGroupEnd();
| Strategy | Use when | |---|---| | Data parallelism — split input across GPUs | Same model, different batches (DL training) | | Model parallelism — split layers across GPUs | Model too large for one GPU (LLMs) | | Domain decomposition — split spatial domain | Simulations with boundary exchange | | Pipeline parallelism — stage GPUs in sequence | Long sequential stages with different memory requirements |
// CODE EXAMPLE
// Multi-GPU vector add: split N/2 to GPU 0, N/2 to GPU 1
#include <cuda_runtime.h>
#include <cstdio>
__global__ void vecAdd(const float *a, const float *b, float *c, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) c[i] = a[i] + b[i];
}
int main() {
const int N = 1 << 20; // 1M elements total
const int HALF = N / 2;
const size_t HALF_BYTES = HALF * sizeof(float);
// Host data (pinned for async transfers)
float *h_a, *h_b, *h_c;
cudaMallocHost(&h_a, N * sizeof(float));
cudaMallocHost(&h_b, N * sizeof(float));
cudaMallocHost(&h_c, N * sizeof(float));
for (int i = 0; i < N; i++) { h_a[i] = 1.0f; h_b[i] = 2.0f; }
float *d_a[2], *d_b[2], *d_c[2];
cudaStream_t s[2];
for (int d = 0; d < 2; d++) {
cudaSetDevice(d);
cudaMalloc(&d_a[d], HALF_BYTES);
cudaMalloc(&d_b[d], HALF_BYTES);
cudaMalloc(&d_c[d], HALF_BYTES);
cudaStreamCreate(&s[d]);
}
// Upload each half to its GPU and launch
for (int d = 0; d < 2; d++) {
cudaSetDevice(d);
int offset = d * HALF;
cudaMemcpyAsync(d_a[d], h_a + offset, HALF_BYTES, cudaMemcpyHostToDevice, s[d]);
cudaMemcpyAsync(d_b[d], h_b + offset, HALF_BYTES, cudaMemcpyHostToDevice, s[d]);
int tpb = 256, blocks = (HALF + tpb - 1) / tpb;
vecAdd<<<blocks, tpb, 0, s[d]>>>(d_a[d], d_b[d], d_c[d], HALF);
cudaMemcpyAsync(h_c + offset, d_c[d], HALF_BYTES, cudaMemcpyDeviceToHost, s[d]);
}
// Wait for both GPUs
for (int d = 0; d < 2; d++) {
cudaSetDevice(d);
cudaStreamSynchronize(s[d]);
}
printf("c[0]=%.0f c[N-1]=%.0f (expected 3)\n", h_c[0], h_c[N-1]);
// cleanup omitted for brevity
}// TRY IT
GPU execution// PRACTICE PROJECTS
// ALL CONCEPTS
Browse reference