A CUDA stream is an ordered queue of GPU operations (kernel launches, memory copies, events) that execute in order. Operations in different streams may overlap.
All CUDA calls without an explicit stream use the null stream (stream 0). It is synchronizing: it waits for all preceding operations in all other streams, and all other streams wait for it. Safe for simple programs; prevents all overlap.
cudaStream_t s;
cudaStreamCreate(&s);
kernel<<<grid, block, 0, s>>>(args); // kernel in stream s
cudaMemcpyAsync(dst, src, bytes, cudaMemcpyHostToDevice, s); // async copy in s
cudaStreamSynchronize(s); // host waits for s to finish
cudaStreamDestroy(s);
Modern GPUs have independent copy engines (DMA units) separate from the compute engines (SMs):
| Overlap Type | Requirement | |---|---| | Concurrent kernel execution | Multiple non-dependent kernels in different streams | | Kernel + H2D transfer | Pinned host memory + separate stream from kernel | | H2D + D2H transfer simultaneously | GPU with separate H2D and D2H DMA engines |
Critical: async memory transfers require pinned (page-locked) host memory allocated with cudaMallocHost. Regular malloc memory cannot be DMA'd directly.
Classic pattern for processing a large dataset in chunks — approaches 2× throughput when compute ≈ transfer time:
Time →
Stream A: [H2D chunk 0]──[compute 0]──[D2H chunk 0]
Stream B: [H2D chunk 1]──[compute 1]──[D2H chunk 1]
[H2D chunk 2]──...
Events let you express fine-grained dependencies without stalling the entire device:
cudaEvent_t e;
cudaEventCreate(&e);
kernelA<<<grid, block, 0, streamA>>>(d_buf);
cudaEventRecord(e, streamA); // record when A finishes
cudaStreamWaitEvent(streamB, e, 0); // B waits for A's event
kernelB<<<grid, block, 0, streamB>>>(d_buf); // uses A's output
cudaEventDestroy(e);
| Call | Blocks until… |
|---|---|
| cudaDeviceSynchronize() | All device operations complete |
| cudaStreamSynchronize(s) | All operations in stream s complete |
| cudaEventSynchronize(e) | Event e has been recorded |
| cudaStreamWaitEvent(s, e) | GPU: stream s waits for event e (no CPU block) |
Prefer stream/event sync over cudaDeviceSynchronize to avoid stalling unrelated work.
For workloads that repeat the same kernel sequence many times, CUDA Graphs eliminate per-iteration CPU launch overhead:
cudaGraph_t graph;
cudaGraphExec_t instance;
// Capture the operation sequence once
cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal);
kernelA<<<grid, block, 0, stream>>>(args);
kernelB<<<grid, block, 0, stream>>>(args);
cudaStreamEndCapture(stream, &graph);
// Instantiate once
cudaGraphInstantiate(&instance, graph, 0);
// Launch 1000 times — ~10× less CPU overhead than re-launching kernels
for (int i = 0; i < 1000; i++)
cudaGraphLaunch(instance, stream);
cudaStreamSynchronize(stream);
cudaGraphExecDestroy(instance);
cudaGraphDestroy(graph);
// CODE EXAMPLE
// Two-stream pipeline: overlap H2D copies with kernel computation
#include <cuda_runtime.h>
__global__ void processChunk(float *d, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) d[i] = d[i] * 2.0f + 1.0f;
}
int main() {
const int N = 1 << 22;
const int CHUNKS = 4;
const int CHUNK_N = N / CHUNKS;
const size_t BYTES = CHUNK_N * sizeof(float);
// Pinned memory is required for async DMA transfers
float *h_data;
cudaMallocHost(&h_data, N * sizeof(float));
for (int i = 0; i < N; i++) h_data[i] = (float)i;
float *d_buf[2];
cudaMalloc(&d_buf[0], BYTES);
cudaMalloc(&d_buf[1], BYTES);
cudaStream_t streams[2];
cudaStreamCreate(&streams[0]);
cudaStreamCreate(&streams[1]);
// Ping-pong between two streams: stream 0 processes while stream 1 copies
for (int c = 0; c < CHUNKS; c++) {
int s = c % 2;
float *h = h_data + c * CHUNK_N;
// H2D: copy next chunk to device
cudaMemcpyAsync(d_buf[s], h, BYTES, cudaMemcpyHostToDevice, streams[s]);
// Compute: overlaps with H2D of other stream
int tpb = 256, blocks = (CHUNK_N + tpb - 1) / tpb;
processChunk<<<blocks, tpb, 0, streams[s]>>>(d_buf[s], CHUNK_N);
// D2H: copy result back
cudaMemcpyAsync(h, d_buf[s], BYTES, cudaMemcpyDeviceToHost, streams[s]);
}
cudaStreamSynchronize(streams[0]);
cudaStreamSynchronize(streams[1]);
cudaFreeHost(h_data);
cudaFree(d_buf[0]); cudaFree(d_buf[1]);
cudaStreamDestroy(streams[0]); cudaStreamDestroy(streams[1]);
}// TRY IT
GPU execution