AzeezIsh commited on
Commit
f970465
·
verified ·
1 Parent(s): 6e3bbf1

Uploaded using `kernel-builder`.

Browse files
README.md DELETED
@@ -1,46 +0,0 @@
1
- # atlas-gdn
2
-
3
- Hand-tuned Gated DeltaNet kernels for the linear-attention path of
4
- Qwen3.6 hybrid models on NVIDIA GB10 (DGX Spark, SM121).
5
-
6
- ## What's inside
7
-
8
- | Op | Use |
9
- |-----------------------------|--------------------------------------------------------|
10
- | `gdn_decode` | Single-token recurrent decode (FP32 Q/K/V, BF16 out) |
11
- | `gdn_prefill` | Multi-token prefill (BF16 throughout) |
12
- | `gdn_chunk2` / `gdn_chunk3` | MTP K=2/3 chunkwise verify (Qwen3.6 NVFP4 specialized) |
13
- | `gdn_wy2` / `wy3` / `wy4` | 2-pass WY-chunkwise verify (general K=2/3/4) |
14
- | `causal_conv1d_fwd` | Depthwise causal Conv1d (SSM input projection) |
15
- | `causal_conv1d_update` | Single-step Conv1d update (decode) |
16
-
17
- ## Hardware
18
-
19
- These kernels target **only** NVIDIA GB10 (compute capability 12.1,
20
- `sm_121f`). They will not load on any other GPU. GB10 has:
21
-
22
- - Unified LPDDR5X memory (~273 GB/s) — bandwidth-bound, not occupancy-bound
23
- - No multi-CTA clusters (ClusterShape forced to 1×1×1)
24
- - No `cvt.rn.satfinite.e2m1x2.f32` PTX (software E2M1 conversion path)
25
- - Cooperative-only scheduling (no Pingpong)
26
-
27
- `build.toml` pins `cuda-capabilities = ["12.1"]` so the build matrix
28
- yields a single SM121 binary; no fallback binaries are produced.
29
-
30
- ## Models tested
31
-
32
- | Model | Layers using these kernels |
33
- |-----------------------------------------|----------------------------|
34
- | Qwen/Qwen3.6-27B (dense, hybrid) | 48 GDN layers |
35
- | Qwen/Qwen3.6-35B-A3B (sparse MoE, hybrid) | 30 GDN layers |
36
-
37
- ## Provenance
38
-
39
- Sources are extracted from the Atlas inference engine
40
- (<https://github.com/Avarok-Cybersecurity/atlas>, AGPL-3.0). The GDN
41
- NVFP4 variant ships with `__launch_bounds__` annotations specific to
42
- Qwen3.6 hidden dimensions (k_dim=128, v_dim=128, 16/32 K/V heads).
43
-
44
- ## License
45
-
46
- AGPL-3.0-only.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch214-cxx11-rocm72-x86_64-linux/{_gdn_rocm_7813975.abi3.so → _gdn_rocm_29ce3de.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7194e74efe71b5b1a8dbe82abe647c4a8604febf694514196af6490e93635601
3
- size 530584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af0817a9d18767a9436ae91f5cdd58ee5d92987c02a1e17ff2dac67c5b6df3d5
3
+ size 570872
build/torch214-cxx11-rocm72-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _gdn_rocm_7813975
3
- ops = torch.ops._gdn_rocm_7813975
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_gdn_rocm_7813975::{op_name}"
 
1
  import torch
2
+ from . import _gdn_rocm_29ce3de
3
+ ops = torch.ops._gdn_rocm_29ce3de
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_gdn_rocm_29ce3de::{op_name}"
build/torch214-cxx11-rocm72-x86_64-linux/layers.py CHANGED
@@ -246,3 +246,157 @@ class Qwen3_5GatedDeltaNet(nn.Module):
246
  # Qwen3.6-35B-A3B (MoE) shares the dense layer's GDN core verbatim. Expose the
247
  # host class name so a single LayerRepository entry resolves for either model.
248
  Qwen3_5MoeGatedDeltaNet = Qwen3_5GatedDeltaNet
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  # Qwen3.6-35B-A3B (MoE) shares the dense layer's GDN core verbatim. Expose the
247
  # host class name so a single LayerRepository entry resolves for either model.
248
  Qwen3_5MoeGatedDeltaNet = Qwen3_5GatedDeltaNet
249
+
250
+
251
+ class causal_conv1d_fn(nn.Module):
252
+ """Drop-in for ``causal_conv1d_fn`` on gfx1151 (Qwen3.5/3.6/3.8, Nemotron-H, Mamba-2)."""
253
+
254
+ has_backward: bool = False
255
+ can_torch_compile: bool = True
256
+
257
+ def forward(self, hidden_states, weight, bias=None, activation=None, **kwargs):
258
+ x = hidden_states.contiguous()
259
+ w = weight.contiguous()
260
+ b = bias.float().contiguous() if bias is not None else None
261
+ out = torch.empty_like(x, memory_format=torch.contiguous_format)
262
+ ops.causal_conv1d_fwd(x, w, b, out)
263
+ return out
264
+
265
+
266
+ class causal_conv1d_update(nn.Module):
267
+ """Drop-in for ``causal_conv1d_update`` on gfx1151 (Qwen3.5/3.6/3.8, Nemotron-H, Mamba-2)."""
268
+
269
+ has_backward: bool = False
270
+ can_torch_compile: bool = True
271
+
272
+ def forward(self, hidden_states, conv_state, weight, bias=None, activation=None):
273
+ squeeze_last = hidden_states.dim() == 3
274
+ x = (hidden_states[:, :, 0] if squeeze_last else hidden_states).contiguous()
275
+ cs = conv_state.float().contiguous()
276
+ w = weight.contiguous()
277
+ b = bias.float().contiguous() if bias is not None else None
278
+ out = torch.empty_like(x, memory_format=torch.contiguous_format)
279
+ ops.causal_conv1d_update(cs, x, w, b, out)
280
+ conv_state.copy_(cs.to(conv_state.dtype))
281
+ return out.unsqueeze(-1) if squeeze_last else out
282
+
283
+
284
+ class mamba_chunk_scan_combined(nn.Module):
285
+ """Drop-in for ``mamba_chunk_scan_combined`` on gfx1151 (Nemotron-H / Super-VL / Mamba-2)."""
286
+
287
+ has_backward: bool = False
288
+ can_torch_compile: bool = True
289
+
290
+ def forward(
291
+ self,
292
+ hidden_states: torch.Tensor,
293
+ dt: torch.Tensor,
294
+ A: torch.Tensor,
295
+ B: torch.Tensor,
296
+ C: torch.Tensor,
297
+ chunk_size: int,
298
+ D: torch.Tensor | None = None,
299
+ dt_bias: torch.Tensor | None = None,
300
+ initial_states: torch.Tensor | None = None,
301
+ dt_softplus: bool = False,
302
+ dt_limit: tuple[float, float] = (0.0, float("inf")),
303
+ return_final_states: bool = False,
304
+ **kwargs,
305
+ ):
306
+ bsz, _, nheads, headdim = hidden_states.shape
307
+ dstate = B.shape[-1]
308
+ if initial_states is not None:
309
+ h = initial_states.float().clone().contiguous()
310
+ else:
311
+ h = torch.zeros(bsz, nheads, headdim, dstate, device=hidden_states.device, dtype=torch.float32)
312
+ out = torch.empty_like(hidden_states, memory_format=torch.contiguous_format)
313
+ d_f32 = D.float().contiguous() if D is not None else None
314
+ dtb_f32 = dt_bias.float().contiguous() if dt_bias is not None else None
315
+ ops.mamba2_scan_reg(
316
+ h,
317
+ hidden_states.contiguous(),
318
+ B.contiguous(),
319
+ C.contiguous(),
320
+ dt.to(torch.bfloat16).contiguous(),
321
+ A.float().contiguous(),
322
+ d_f32,
323
+ dtb_f32,
324
+ None,
325
+ out,
326
+ bool(dt_softplus),
327
+ float(dt_limit[0]),
328
+ float(dt_limit[1]),
329
+ )
330
+ if return_final_states:
331
+ return out, h
332
+ return out
333
+
334
+
335
+ class selective_state_update(nn.Module):
336
+ """Drop-in for ``selective_state_update`` on gfx1151 (Nemotron-H / Super-VL / Mamba-2 decode)."""
337
+
338
+ has_backward: bool = False
339
+ can_torch_compile: bool = True
340
+
341
+ def forward(
342
+ self,
343
+ state: torch.Tensor,
344
+ hidden_states: torch.Tensor,
345
+ dt: torch.Tensor,
346
+ A: torch.Tensor,
347
+ B: torch.Tensor,
348
+ C: torch.Tensor,
349
+ D: torch.Tensor | None = None,
350
+ dt_bias: torch.Tensor | None = None,
351
+ dt_softplus: bool = False,
352
+ z: torch.Tensor | None = None,
353
+ **kwargs,
354
+ ):
355
+ is_f32_contig = state.dtype == torch.float32 and state.is_contiguous()
356
+ h = state if is_f32_contig else state.float().contiguous()
357
+ x = hidden_states.contiguous()
358
+ Bm = B.contiguous()
359
+ Cm = C.contiguous()
360
+ dt_bf16 = dt if dt.dtype == torch.bfloat16 else dt.to(torch.bfloat16)
361
+ A_f32 = A if A.dtype == torch.float32 else A.float()
362
+ D_f32 = (D if D.dtype == torch.float32 else D.float()) if D is not None else None
363
+ dtb_f32 = (dt_bias if dt_bias.dtype == torch.float32 else dt_bias.float()) if dt_bias is not None else None
364
+ z_bf16 = z.contiguous() if z is not None else None
365
+ out = torch.empty_like(x, memory_format=torch.contiguous_format)
366
+ ops.mamba2_decode_reg(
367
+ h, x, Bm, Cm, dt_bf16, A_f32, D_f32, dtb_f32, z_bf16, out, bool(dt_softplus)
368
+ )
369
+ if not is_f32_contig:
370
+ state.copy_(h.to(state.dtype))
371
+ return out
372
+
373
+
374
+ class mamba_split_conv1d_scan_combined(nn.Module):
375
+ """Stub for ``mamba_split_conv1d_scan_combined`` (returns None to route inference through conv1d + chunk scan)."""
376
+
377
+ has_backward: bool = False
378
+ can_torch_compile: bool = True
379
+
380
+ def forward(
381
+ self,
382
+ zxbcdt: torch.Tensor,
383
+ conv1d_weight: torch.Tensor,
384
+ conv1d_bias: torch.Tensor | None,
385
+ dt_bias: torch.Tensor,
386
+ A: torch.Tensor,
387
+ D: torch.Tensor,
388
+ chunk_size: int,
389
+ initial_states: torch.Tensor | None = None,
390
+ dt_limit: tuple[float, float] = (0.0, float("inf")),
391
+ return_final_states: bool = False,
392
+ activation: str = "silu",
393
+ rmsnorm_weight: torch.Tensor | None = None,
394
+ rmsnorm_eps: float = 1e-6,
395
+ outproj_weight: torch.Tensor | None = None,
396
+ outproj_bias: torch.Tensor | None = None,
397
+ headdim: int | None = None,
398
+ ngroups: int = 1,
399
+ norm_before_gate: bool = True,
400
+ **kwargs,
401
+ ):
402
+ return None
build/torch214-cxx11-rocm72-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "gdn",
3
- "id": "_gdn_rocm_7813975",
4
  "version": 1,
5
  "kernels-minver": "0.14.0",
6
  "license": "AGPL-3.0-only",
@@ -16,9 +16,9 @@
16
  "algorithm": "sha256",
17
  "files": {
18
  "__init__.py": "LiP481kYPHgVyWn/hX7QlwghehLrO6jOKeUkE8w0HNU=",
19
- "_gdn_rocm_7813975.abi3.so": "cZTnTv5xtbGo2+gqvmR8SoYE/r9pRRQZavZJDpNjVgE=",
20
- "_ops.py": "tjUEuMG2XDblchEvb5ZxNzJkKLWoAuprzpo/5sET07k=",
21
- "layers.py": "zk+WPfwG8fQsGGECA5/e/Vp7tleXh24qa/td8olPHmo="
22
  }
23
  },
24
  "provenance": {
@@ -28,7 +28,7 @@
28
  "dirty": false
29
  },
30
  "kernel": {
31
- "commit": "78139759223bc2cac161e82d02e17b65739fca34",
32
  "dirty": false
33
  }
34
  }
 
1
  {
2
  "name": "gdn",
3
+ "id": "_gdn_rocm_29ce3de",
4
  "version": 1,
5
  "kernels-minver": "0.14.0",
6
  "license": "AGPL-3.0-only",
 
16
  "algorithm": "sha256",
17
  "files": {
18
  "__init__.py": "LiP481kYPHgVyWn/hX7QlwghehLrO6jOKeUkE8w0HNU=",
19
+ "_gdn_rocm_29ce3de.abi3.so": "rwgXqdGHZ6lDaukfXN1Y7l2SmHwCoeF/8trGfFtt89U=",
20
+ "_ops.py": "Ox3uWpzaLionFZP8zF9TVqV7K6GeVK7UwkoYtdap1So=",
21
+ "layers.py": "WKNl84v/UIdFIi+80We54lwJsrAD04lcaCcN3rVx90c="
22
  }
23
  },
24
  "provenance": {
 
28
  "dirty": false
29
  },
30
  "kernel": {
31
+ "commit": "29ce3def897af8a824c70532b1121e108adc283b",
32
  "dirty": false
33
  }
34
  }