Repository navigation
Expand file tree
/
Copy pathgelu_forward.cu
More file actions
174 lines (141 loc) · 5.49 KB
/
Copy pathgelu_forward.cu
File metadata and controls
174 lines (141 loc) · 5.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
/*
Kernels for gelu forward pass.
Compile example:
nvcc -O3 --use_fast_math -lcublas -lcublasLt gelu_forward.cu -o gelu_forward
If encountering "error: identifier "M_PI" is undefined", add the following lines to the top of the file:
#define _USE_MATH_DEFINES
#include <math.h> OR #include <cmath>
version 1 is naive CPU port
./gelu_forward 1
version 2 is bfloat16 with the Packed128 data structure
./gelu_forward 2
*/
#include <stdio.h>
#include <stdlib.h>
#include <cuda_runtime.h>
#define ENABLE_BF16
#include "common.h"
// ----------------------------------------------------------------------------
// CPU code reference
#define GELU_SCALING_FACTOR sqrtf(2.0f / M_PI)
void gelu_forward_cpu(float* out, const float* inp, int N) {
for (int i = 0; i < N; i++) {
float x = inp[i];
float cube = 0.044715f * x * x * x;
out[i] = 0.5f * x * (1.0f + tanhf(GELU_SCALING_FACTOR * (x + cube)));
}
}
// ----------------------------------------------------------------------------
// GPU kernels
// elementwise ops are nice and ez
__global__ void gelu_forward_kernel1(floatX* out, const floatX* inp, int N) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < N) {
float xi = inp[i];
float cube = 0.044715f * xi * xi * xi;
out[i] = 0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube)));
}
}
// elementwise ops are nice and ez
__global__ void gelu_forward_kernel2(floatX* out, const floatX* inp, int N) {
int i = (blockIdx.x * blockDim.x + threadIdx.x) * x128::size;
if (i < N) {
x128 packed_out;
x128 packed_inp = load128cs(inp + i
A31B
); // load and do not keep in cache
for(int k = 0; k < packed_inp.size; ++k) {
float xi = (float)packed_inp[k];
float cube = 0.044715f * xi * xi * xi;
packed_out[k] = (floatX)(0.5f * xi * (1.0f + tanhf(GELU_SCALING_FACTOR * (xi + cube))));
}
// store instead of storecs (without cache streaming) in case it is useful for the
// data to be in the cache for the next operation after this GeLU
store128(out + i, packed_out);
}
}
// ----------------------------------------------------------------------------
// kernel launcher
void gelu_forward1(floatX* out, const floatX* inp, int N, const int block_size) {
const int grid_size = ceil_div(N, block_size);
gelu_forward_kernel1<<<grid_size, block_size>>>(out, inp, N);
cudaCheck(cudaGetLastError());
}
void gelu_forward2(floatX* out, const floatX* inp, int N, const int block_size) {
const int grid_size = ceil_div(N, block_size * x128::size);
gelu_forward_kernel2<<<grid_size, block_size>>>(out, inp, N);
cudaCheck(cudaGetLastError());
}
// kernel version dispatch
void gelu_forward(int kernel_num,
floatX* out,
const floatX* inp,
int B, int T, int C,
int block_size) {
switch (kernel_num) {
case 1:
gelu_forward1(out, inp, B * T * C, block_size);
break;
case 2:
gelu_forward2(out, inp, B * T * C, block_size);
break;
default:
printf("Invalid kernel number\n");
exit(1);
}
}
// ----------------------------------------------------------------------------
int main(int argc, const char **argv) {
setup_main();
int B = 8;
int T = 1024;
int C = 768;
// create host memory of random numbers
float* out = (float*)malloc(B * T * C * sizeof(float));
float* inp = make_random_float(B * T * C);
// read kernel_num from command line
int kernel_num = 1;
if (argc > 1) {
kernel_num = atoi(argv[1]);
}
printf("Using kernel %d\n", kernel_num);
// first check the correctness of the kernel
gelu_forward_cpu(out, inp, B * T * C);
// move to GPU
floatX* d_out;
floatX* d_inp;
cudaCheck(cudaMalloc(&d_out, B * T * C * sizeof(floatX)));
cudaCheck(cudaMalloc(&d_inp, B * T * C * sizeof(floatX)));
cudaCheck(memcpy_convert(d_inp, inp, B * T * C));
// time the kernel at different block sizes
int block_sizes[] = {32, 64, 128, 256, 512, 1024};
for (int j = 0; j < sizeof(block_sizes) / sizeof(int); j++) {
int block_size = block_sizes[j];
printf("Checking block size %d.\n", block_size);
gelu_forward(kernel_num, d_out, d_inp, B, T, C, block_size);
#if !defined(ENABLE_BF16) && !defined(ENABLE_FP16)
float tol = 1e-5;
#else
float tol = 1e-2f;
#endif
validate_result(d_out, out, "out", B * T * C, tol);
}
printf("All results match. Starting benchmarks.\n\n");
for (int j = 0; j < sizeof(block_sizes) / sizeof(int); j++) {
int block_size = block_sizes[j];
int repeat_times = 1000;
float elapsed_time = benchmark_kernel(repeat_times, gelu_forward,
kernel_num, d_out, d_inp,
B, T, C, block_size);
// napkin math: estimate the memory bandwidth achieved
// for each (B,T,C) output element, we do 1 read and 1 write, 4 bytes each
// and e.g. A100 40GB PCIe is advertised at 1,555GB/s
long memory_ops = B * T * C * 2 * (int)sizeof(floatX);
float memory_bandwidth = memory_ops / elapsed_time / 1e6;
printf("block_size %4d | time %.4f ms | bandwidth %.2f GB/s\n", block_size, elapsed_time, memory_bandwidth);
}
// free memory
free(out);
free(inp);
cudaCheck(cudaFree(d_out));
cudaCheck(cudaFree(d_inp));
return 0;
}