Repository navigation
Expand file tree
/
Copy pathkda.py
More file actions
153 lines (134 loc) · 6.78 KB
/
Copy pathkda.py
File metadata and controls
153 lines (134 loc) · 6.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
"""KDA (Kimi Delta Attention) forward.
S_t = (I - beta_t k_t k_t^T) Diag(alpha_t) S_{t-1} + beta_t k_t v_t^T
o_t = S_t^T q_t (k/q rows l2-normalized, beta = sigmoid(x_beta),
alpha = sigmoid(x_decay) per-channel, log-space)
This module owns the eager preprocessing (K0): fused
l2norm (one-pass backend row kernel), sigmoid, stable log-sigmoid, the
per-chunk decay cumsum ``g`` (one broadcast-batch bmm against a tril-ones
matrix — the autotuned matmul_v2 path, replacing two in-kernel cumsum
dots), the chunk totals ``gt`` (last row of g) and the scalar write-key
shift ``c``/``emc = exp(-c)``. Both backends see identical preprocessed
inputs. The chunked recurrence itself lives in the backends: the default
f32 route is the v2 fused pipeline (fewer launches, same math); the
five-kernel v1 (K1 wy-build, K1s ``tl.solve_tril`` inversion, K1b
pseudo-value factors, K2 state scan, K3 out) is kept for tests; tf32/
bf16 tiers run the mma kernels. CPU is the fp64 numpy closed form (the
numerical oracle).
Forward-only (no tape): the whole op runs under ``no_grad`` — backward is
not implemented.
"""
from __future__ import annotations
import numpy as np
from ..autograd import no_grad
from ..tensor import zeros
from . import _be, _tensor, f32
from . import (cat, exp, log_sigmoid, matmul, min as _omin, relu, sigmoid)
KDA_CHUNK = 64
# write-key materialization clip bound: the scalar
# shift c = max(min_d gT[d], -CL) keeps every exp() argument inside +-CL;
# exact for within-chunk decay excursions <= 2*CL, bounded beyond.
_KDA_CL = 80.0
# tril-ones (1, C, C) for the decay-cumsum bmm, cached per device
_TRIL_ONES = {}
def _tril_ones(device):
t = _TRIL_ONES.get(device)
if t is None:
il = np.tril(np.ones((1, KDA_CHUNK, KDA_CHUNK), dtype=np.float32))
t = _tensor(np.ascontiguousarray(il), device=device)
_TRIL_ONES[device] = t
return t
def _l2norm(be, x):
R, d = x.numel() // x.shape[-1], x.shape[-1]
xc = x if x.is_contiguous() else x.contiguous()
return be.l2norm_rows(xc.reshape((R, d))).reshape(x.shape)
def kda_preprocess(q, k, v, x_beta, x_decay):
"""The K0 eager preprocessing: returns
(kn, qn, beta, loga, g, gt, c, emc) — l2-normalized q/k, sigmoid beta,
stable-log-sigmoid decay gates loga (B,H,T,dk), the per-chunk inclusive
decay cumsum g (B*H*NC, C, dk), chunk totals gt (B,H,NC,dk), and the
scalar write-key shift c (B,H,NC) with emc = exp(-c). Exposed for the
stage-wise kernel tests; kda_forward consumes it."""
B, H, T, dk = k.shape
be = _be(k.device)
kn, qn = _l2norm(be, k), _l2norm(be, q)
beta = sigmoid(x_beta) # (B, H, T)
loga = log_sigmoid(x_decay) # (B, H, T, dk) <= 0
NC = (T + KDA_CHUNK - 1) // KDA_CHUNK
pad = NC * KDA_CHUNK - T
# pad rows are zero-decay by construction; kernels mask on T elsewhere
loga_p = cat([loga, zeros((B, H, pad, dk), f32, loga.device)],
dim=2) if pad else loga
slabs = loga_p.reshape((B * H * NC, KDA_CHUNK, dk))
il = _tril_ones(k.device).broadcast_to(
(B * H * NC, KDA_CHUNK, KDA_CHUNK))
g = matmul(il, slabs) # per-chunk inclusive cumsum
gt = g.reshape((B, H, NC, KDA_CHUNK, dk))[:, :, :, KDA_CHUNK - 1:, :]
gt = gt.contiguous().reshape((B, H, NC, dk))
# scalar write-key shift c = max(min_d gT[d], -CL); emc = exp(-c)
c = relu(_omin(gt, dim=-1) + _KDA_CL) - _KDA_CL
emc = exp(-c)
return kn, qn, beta, loga, g, gt, c, emc
def kda_forward(q, k, v, x_beta, x_decay, initial_state=None,
output_final_state=False, precision=None):
"""KDA forward (f32, CUDA/CPU).
q, k: (B, H, T, dk); v: (B, H, T, dv); x_beta: (B, H, T);
x_decay: (B, H, T, dk) — raw gates, sigmoid/log-sigmoid applied inside.
initial_state: (B, H, dk, dv) or None (zero state). dk/dv in {64, 128}.
Returns o (B, H, T, dv); with output_final_state=True returns
(o, S_end (B, H, dk, dv)).
precision: None/"fp32" = the fp32 CUDA-core pipeline (default);
"tf32"/"bf16" = the tensor-core mma tiers (CUDA only,
f32 in/out, f32 accumulators; mma.sync inner products).
"""
q, k, v = _tensor(q), _tensor(k), _tensor(v)
x_beta, x_decay = _tensor(x_beta), _tensor(x_decay)
if precision in (None, "fp32"):
precision = None
elif precision not in ("tf32", "bf16"):
raise RuntimeError(
f"kda_forward: precision '{precision}' not in "
"(None, 'fp32', 'tf32', 'bf16')")
if q.device != k.device or q.device != v.device:
raise RuntimeError("kda_forward: q/k/v on different devices")
for t, nm in ((q, "q"), (k, "k"), (v, "v"), (x_beta, "x_beta"),
(x_decay, "x_decay")):
if t.dtype != f32:
raise RuntimeError(f"kda_forward: {nm} must be f32, got {t.dtype}")
B, H, T, dk = k.shape
if q.shape != (B, H, T, dk) or x_decay.shape != (B, H, T, dk):
raise RuntimeError(f"kda_forward: q/x_decay shape mismatch vs k "
f"{k.shape}: {q.shape} {x_decay.shape}")
if v.shape[:3] != (B, H, T):
raise RuntimeError(f"kda_forward: v shape {v.shape} vs k {k.shape}")
dv = v.shape[-1]
if x_beta.shape != (B, H, T):
raise RuntimeError(f"kda_forward: x_beta shape {x_beta.shape} "
f"vs {(B, H, T)}")
if dk not in (64, 128) or dv not in (64, 128):
raise NotImplementedError(
f"kda_forward currently supports dk/dv in (64, 128), got {dk}/{dv}")
if initial_state is not None:
initial_state = _tensor(initial_state)
if initial_state.shape != (B, H, dk, dv):
raise RuntimeError(f"kda_forward: initial_state shape "
f"{initial_state.shape} vs {(B, H, dk, dv)}")
if initial_state.dtype != f32:
raise RuntimeError("kda_forward: initial_state must be f32")
with no_grad():
if precision is None and q.device == "cuda":
# v2 fused pipeline: the eager K0 preprocessing is
# folded into the kernels, which take the raw gates directly.
return _be(q.device).kda_forward_fused(
q, k, v, x_beta, x_decay, initial_state, output_final_state)
kn, qn, beta, loga, g, gt, c, emc = kda_preprocess(q, k, v, x_beta,
x_decay)
if precision is not None:
if q.device != "cuda":
raise RuntimeError(
"kda_forward: precision tiers are CUDA-only")
return _be(q.device).kda_forward_mma(
kn, qn, v, beta, loga, g, gt, c, emc,
initial_state, output_final_state, precision)
return _be(q.device).kda_forward(
kn, qn, v, beta, loga, g, gt, c, emc,
initial_state, output_final_state)