Repository navigation
Expand file tree
/
Copy pathcpy.metal
More file actions
323 lines (296 loc) · 11.9 KB
/
Copy pathcpy.metal
File metadata and controls
323 lines (296 loc) · 11.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
struct ds4_metal_args_cpy {
int64_t nk0;
int64_t ne00;
int64_t ne01;
int64_t ne02;
int64_t ne03;
uint64_t nb00;
uint64_t nb01;
uint64_t nb02;
uint64_t nb03;
int64_t ne0;
int64_t ne1;
int64_t ne2;
int64_t ne3;
uint64_t nb0;
uint64_t nb1;
uint64_t nb2;
uint64_t nb3;
};
// Typed copy/conversion between graph tensors. DS4 uses this for layout
// materialization and F32/F16 conversions at graph boundaries such as KV/cache
// packing and compressor pooling.
template<typename T0, typename T1>
kernel void kernel_cpy_t_t(
constant ds4_metal_args_cpy & args,
device const char * src0,
device char * dst,
uint3 tgpig[[threadgroup_position_in_grid]],
ushort tiitg[[thread_index_in_threadgroup]],
ushort3 ntg[[threads_per_threadgroup]]) {
const int i03 = tgpig[2];
const int i02 = tgpig[1];
const int i01 = ntg[1] == 1 ? tgpig[0]%args.ne01 : tgpig[0]*ntg[1] + tiitg/ntg[0];
const int iw0 = ntg[1] == 1 ? tgpig[0]/args.ne01 : 0;
const int64_t n = i03*args.ne02*args.ne01*args.ne00 + i02*args.ne01*args.ne00 + i01*args.ne00;
const int64_t i3 = n/(args.ne2*args.ne1*args.ne0);
const int64_t i2 = (n - i3*args.ne2*args.ne1*args.ne0)/(args.ne1*args.ne0);
const int64_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0;
const int64_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0);
device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0);
for (int64_t i00 = iw0*ntg[0] + tiitg%ntg[0]; i00 < args.ne00; ) {
device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00);
dst_data[i00] = (T1) src[0];
break;
}
}
typedef decltype(kernel_cpy_t_t<float, float>) kernel_cpy_t;
// Host-visible copy/conversion variants used by the DS4 graph.
template [[host_name("kernel_cpy_f32_f32")]] kernel kernel_cpy_t kernel_cpy_t_t<float, float>;
template [[host_name("kernel_cpy_f32_f16")]] kernel kernel_cpy_t kernel_cpy_t_t<float, half>;
template [[host_name("kernel_cpy_f16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t<half, float>;
template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy_t_t<half, half>;
// Contiguous 1D conversions avoid the generic tensor-index reconstruction
// above. Packed vector types retain scalar alignment, so tensor views whose
// offsets are float/half aligned do not need additional 16/8-byte alignment.
// Bind scalar pointers so Metal also accepts buffers containing fewer than
// four elements. Vectorize only complete groups; the tail stays scalar.
kernel void kernel_cpy_contig_f32_f16_4(
constant uint & n,
device const float * src,
device half * dst,
uint gid [[thread_position_in_grid]]) {
const uint i = gid * 4u;
if (i >= n) {
return;
}
const uint remaining = n - i;
if (remaining >= 4u) {
const float4 value = float4(((device const packed_float4 *)src)[gid]);
((device packed_half4 *)dst)[gid] = packed_half4(half4(value));
return;
}
10BC8
div> for (uint lane = 0; lane < remaining; ++lane) {
dst[i + lane] = half(src[i + lane]);
}
}
kernel void kernel_cpy_contig_f16_f32_4(
constant uint & n,
device const half * src,
device float * dst,
uint gid [[thread_position_in_grid]]) {
const uint i = gid * 4u;
if (i >= n) {
return;
}
const uint remaining = n - i;
if (remaining >= 4u) {
const half4 value = half4(((device const packed_half4 *)src)[gid]);
((device packed_float4 *)dst)[gid] = packed_float4(float4(value));
return;
}
for (uint lane = 0; lane < remaining; ++lane) {
dst[i + lane] = float(src[i + lane]);
}
}
// Bitwise F16 transport for cache staging. Use ushort rather than half so NaN
// payloads and every other binary16 encoding pass through unchanged.
kernel void kernel_cpy_contig_f16_f16_bits_4(
constant uint & n,
device const ushort * src,
device ushort * dst,
uint gid [[thread_position_in_grid]]) {
const uint i = gid * 4u;
if (i >= n) {
return;
}
const uint remaining = n - i;
if (remaining >= 4u) {
((device packed_ushort4 *)dst)[gid] = ((device const packed_ushort4 *)src)[gid];
return;
}
for (uint lane = 0; lane < remaining; ++lane) {
dst[i + lane] = src[i + lane];
}
}
struct ds4_metal_args_flash_kv_stage_f16 {
uint raw_cap;
uint raw_start;
uint n_raw;
uint n_comp;
uint pad_rows;
uint shared_pad;
};
// Decode-time gathered attention consumes a logical raw-cache ring followed
// by an already-F16 compressed cache. Pack both regions into the contiguous
// F16 FlashAttention scratch in one dispatch. The raw conversion expression
// and compressed ushort4 transport exactly match the standalone copy kernels.
// GATHER reads the compressed rows as f32 through ids (DeepSeek V4.1's
// cache) instead of contiguous f16 rows.
template<bool GATHER>
static inline void dsv4_flash_kv_stage_body(
constant ds4_metal_args_flash_kv_stage_f16 & args,
device const char * raw_src,
device const char * comp_src,
device char * dst,
device const char * mask_src,
device char * pad_dst,
uint gid,
device const int * ids) {
constexpr uint row_vecs = 128;
const uint raw_vecs = args.n_raw * row_vecs;
const uint n_keys = args.n_raw + args.n_comp;
const uint total_vecs = n_keys * row_vecs;
if (gid < raw_vecs) {
const uint logical_row = gid >> 7;
const uint col = gid & 127u;
uint physical_row = args.raw_start + logical_row;
if (physical_row >= args.raw_cap) {
physical_row -= args.raw_cap;
}
device const packed_float4 *raw =
(device const packed_float4 *)raw_src;
device packed_half4 *dst_half = (device packed_half4 *)dst;
const float4 value =
float4(raw[physical_row * row_vecs + col]);
dst_half[gid] = packed_half4(half4(value));
return;
}
if (gid < total_vecs) {
if (GATHER) {
const uint comp_gid = gid - raw_vecs;
device const packed_float4 *comp =
(device const packed_float4 *)comp_src;
device packed_half4 *dst_half = (device packed_half4 *)dst;
const float4 value =
float4(comp[(uint)ids[comp_gid >> 7] * row_vecs + (comp_gid & 127u)]);
dst_half[gid] = packed_half4(half4(value));
return;
}
device const packed_ushort4 *comp =
(device const packed_ushort4 *)comp_src;
device packed_ushort4 *dst_bits = (device packed_ushort4 *)dst;
dst_bits[gid] = comp[gid - raw_vecs];
return;
}
// The vector FlashAttention kernel redirects its final partial block to
// a compact K/V/mask buffer. When requested, append those writes to this
// dispatch so gathered decode does not need a standalone pad dispatch.
const uint pad_rows = args.pad_rows;
const uint pad_vecs = pad_rows * row_vecs;
uint pad_gid = gid - total_vecs;
if (pad_gid < pad_vecs) {
const uint row = pad_gid >> 7;
const uint col = pad_gid & 127u;
const uint valid_rows = n_keys % pad_rows;
device packed_half4 *pad_half = (device packed_half4 *)pad_dst;
device packed_ushort4 *pad_bits = (device packed_ushort4 *)pad_dst;
if (row >= valid_rows) {
const packed_half4 zero = packed_half4(half4(0.0h));
pad_half[pad_gid] = zero;
if (!args.shared_pad) {
pad_half[pad_vecs + pad_gid] = zero;
}
return;
}
const uint logical_row = n_keys - valid_rows + row;
if (logical_row < args.n_raw) {
uint physical_row = args.raw_start + logical_row;
if (physical_row >= args.raw_cap) {
physical_row -= args.raw_cap;
}
device const packed_float4 *raw =
(device const packed_float4 *)raw_src;
const float4 value =
float4(raw[physical_row * row_vecs + col]);
const packed_half4 value_half = packed_half4(half4(value));
pad_half[pad_gid] = value_half;
if (!args.shared_pad) {
pad_half[pad_vecs + pad_gid] = value_half;
}
} else if (GATHER) {
device const packed_float4 *comp =
(device const packed_float4 *)comp_src;
const float4 value =
float4(comp[(uint)ids[logical_row - args.n_raw] * row_vecs + col]);
const packed_half4 value_half = packed_half4(half4(value));
pad_half[pad_gid] = value_half;
if (!args.shared_pad) {
pad_half[pad_vecs + pad_gid] = value_half;
}
} else {
device const packed_ushort4 *comp =
(device const packed_ushort4 *)comp_src;
const packed_ushort4 value_bits =
comp[(logical_row - args.n_raw) * row_vecs + col];
pad_bits[pad_gid] = value_bits;
if (!args.shared_pad) {
pad_bits[pad_vecs + pad_gid] = value_bits;
}
}
return;
}
pad_gid -= pad_vecs;
if (pad_gid < pad_rows) {
const uint valid_rows = n_keys % pad_rows;
device const ushort *mask_bits = (device const ushort *)mask_src;
device ushort *pad_mask_bits =
(device ushort *)pad_dst + 2u * pad_vecs * 4u;
pad_mask_bits[pad_gid] = pad_gid < valid_rows
? mask_bits[n_keys - valid_rows + pad_gid]
: 0xfbffu;
}
}
kernel void kernel_dsv4_flash_kv_stage_f16(
constant ds4_metal_args_flash_kv_stage_f16 & args,
device const char * raw_src,
device const char * comp_src,
device char * dst,
device const char * mask_src,
device char * pad_dst,
uint gid [[thread_position_in_grid]]) {
dsv4_flash_kv_stage_body<false>(args, raw_src, comp_src, dst, mask_src, pad_dst, gid, (device const int *)mask_src);
}
kernel void kernel_dsv41_flash_kv_stage(
constant ds4_metal_args_flash_kv_stage_f16 & args,
device const char * raw_src,
device const char * comp_src,
device char * dst,
device const char * mask_src,
device char * pad_dst,
device const int * ids,
uint gid [[thread_position_in_grid]]) {
dsv4_flash_kv_stage_body<true>(args, raw_src, comp_src, dst, mask_src, pad_dst, gid, ids);
}
// Tiled-row expansion of a small table: dst row t = src row (pos0 + t) % ratio.
// Replaces the per-segment copies the compressor store used to encode (one
// dispatch per prefill layer instead of n_tokens/ratio single-threadgroup
// copies). Values are identical to the contiguous copies.
struct ds4_cpy_tile_rows_args {
uint n_total; // n_tokens * width
uint width;
uint ratio;
uint pos0;
};
kernel void kernel_cpy_tile_rows_f16_f32(
constant ds4_cpy_tile_rows_args & args,
device const half * src,
device float * dst,
uint gid [[thread_position_in_grid]]) {
if (gid >= args.n_total) return;
const uint t = gid / args.width;
const uint e = gid - t * args.width;
const uint r = (args.pos0 + t) % args.ratio;
dst[gid] = (float)src[r * args.width + e];
}
kernel void kernel_cpy_tile_rows_f32_f32(
constant ds4_cpy_tile_rows_args & args,
device const float * src,
device float * dst,
uint gid [[thread_position_in_grid]]) {
if (gid >= args.n_total) return;
const uint t = gid / args.width;
const uint e = gid - t * args.width;
const uint r = (args.pos0 + t) % args.ratio;
dst[gid] = src[r * args.width + e];
}