rootonchair commited on
Commit
0e572c9
·
verified ·
1 Parent(s): 7d810a1

Uploaded using `kernel-builder`.

Browse files
Files changed (37) hide show
  1. benchmarks/benchmark.py +0 -1
  2. build/torch212-cxx11-cu126-x86_64-linux/__init__.py +1 -1
  3. build/torch212-cxx11-cu126-x86_64-linux/{_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → _nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} +2 -2
  4. build/torch212-cxx11-cu126-x86_64-linux/_ops.py +3 -3
  5. build/torch212-cxx11-cu126-x86_64-linux/interface.py +3 -3
  6. build/torch212-cxx11-cu126-x86_64-linux/metadata.json +16 -6
  7. build/torch212-cxx11-cu126-x86_64-linux/nunchaku_lite_kernels/__init__.py +0 -26
  8. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +1 -1
  9. build/{torch212-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → torch212-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} +2 -2
  10. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  11. build/torch212-cxx11-cu130-x86_64-linux/interface.py +3 -3
  12. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +16 -6
  13. build/torch212-cxx11-cu130-x86_64-linux/nunchaku_lite_kernels/__init__.py +0 -26
  14. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +1 -1
  15. build/{torch212-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → torch212-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} +2 -2
  16. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  17. build/torch212-cxx11-cu132-x86_64-linux/interface.py +3 -3
  18. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +16 -6
  19. build/torch212-cxx11-cu132-x86_64-linux/nunchaku_lite_kernels/__init__.py +0 -26
  20. build/torch213-cxx11-cu126-x86_64-linux/__init__.py +2 -0
  21. build/torch213-cxx11-cu126-x86_64-linux/__pycache__/__init__.cpython-313.pyc +0 -0
  22. build/torch213-cxx11-cu126-x86_64-linux/__pycache__/_ops.cpython-313.pyc +0 -0
  23. build/torch213-cxx11-cu126-x86_64-linux/__pycache__/interface.cpython-313.pyc +0 -0
  24. build/torch213-cxx11-cu126-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so +3 -0
  25. build/torch213-cxx11-cu126-x86_64-linux/_ops.py +9 -0
  26. build/torch213-cxx11-cu126-x86_64-linux/interface.py +295 -0
  27. build/torch213-cxx11-cu126-x86_64-linux/metadata.json +36 -0
  28. build/torch213-cxx11-cu130-x86_64-linux/__init__.py +2 -0
  29. build/torch213-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so +3 -0
  30. build/torch213-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  31. build/torch213-cxx11-cu130-x86_64-linux/interface.py +295 -0
  32. build/torch213-cxx11-cu130-x86_64-linux/metadata.json +38 -0
  33. build/torch213-cxx11-cu132-x86_64-linux/__init__.py +2 -0
  34. build/torch213-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so +3 -0
  35. build/torch213-cxx11-cu132-x86_64-linux/_ops.py +9 -0
  36. build/torch213-cxx11-cu132-x86_64-linux/interface.py +295 -0
  37. build/torch213-cxx11-cu132-x86_64-linux/metadata.json +38 -0
benchmarks/benchmark.py CHANGED
@@ -1,5 +1,4 @@
1
  import torch
2
-
3
  from kernels.benchmark import Benchmark
4
 
5
 
 
1
  import torch
 
2
  from kernels.benchmark import Benchmark
3
 
4
 
build/torch212-cxx11-cu126-x86_64-linux/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
- from .interface import * # noqa: F401,F403
2
  from .interface import __all__ # noqa: F401
 
1
+ from .interface import *
2
  from .interface import __all__ # noqa: F401
build/torch212-cxx11-cu126-x86_64-linux/{_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → _nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c38e1fb13c7064060be208f3bddfe33a792514b7f4b466b21c2d3bc10fc43d15
3
- size 226233984
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4df7d3d51555596d2499aaa2ec46cda1e61d31e96596af95ce77cbaf94d72eac
3
+ size 226233976
build/torch212-cxx11-cu126-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _nunchaku_lite_kernels_cuda_473b39a_dirty
3
- ops = torch.ops._nunchaku_lite_kernels_cuda_473b39a_dirty
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_nunchaku_lite_kernels_cuda_473b39a_dirty::{op_name}"
 
1
  import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch212-cxx11-cu126-x86_64-linux/interface.py CHANGED
@@ -110,7 +110,7 @@ def _fake_fused_cross_head_qk_norm_rope(
110
 
111
  def _get_ops():
112
  try:
113
- ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
@@ -278,10 +278,10 @@ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: to
278
 
279
 
280
  __all__ = [
281
- "attention_fp16_cuda",
282
  "attention_fp16",
283
- "awq_gemm_w4a16_g128_int16",
284
  "awq_gemm_w4a16_g64_int32",
 
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
 
110
 
111
  def _get_ops():
112
  try:
113
+ _ = ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
 
278
 
279
 
280
  __all__ = [
 
281
  "attention_fp16",
282
+ "attention_fp16_cuda",
283
  "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
build/torch212-cxx11-cu126-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "nunchaku-lite-kernels",
3
- "id": "_nunchaku_lite_kernels_cuda_473b39a_dirty",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -16,11 +16,21 @@
16
  "digest": {
17
  "algorithm": "sha256",
18
  "files": {
19
- "__init__.py": "87pNOdi+WC4T4fU+sP2XuEooTUR+2xxcTe6kCAOHdYo=",
20
- "_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so": "w44fsTxwZAYL4gjzvd/jOnklFLf0tGayHC07wQ/EPRU=",
21
- "_ops.py": "qfLrpvroK0kiufObpMs/guo3+BKrjakeOpBI12NNKbs=",
22
- "interface.py": "5BhmgdJFtczitG1n3byCAvCuTmTQmlaiVSpyz2ZlOYs=",
23
- "nunchaku_lite_kernels/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
 
 
 
 
 
 
 
 
 
 
24
  }
25
  }
26
  }
 
1
  {
2
  "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
16
  "digest": {
17
  "algorithm": "sha256",
18
  "files": {
19
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
20
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "TffT1RVVWW0kmaqi7EbNoeYdMelllq+VznfLr5TXLqw=",
21
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
22
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
23
+ }
24
+ },
25
+ "provenance": {
26
+ "kernel-builder": {
27
+ "version": "0.17.0-dev0",
28
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
29
+ "dirty": false
30
+ },
31
+ "kernel": {
32
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
33
+ "dirty": false
34
  }
35
  }
36
  }
build/torch212-cxx11-cu126-x86_64-linux/nunchaku_lite_kernels/__init__.py DELETED
@@ -1,26 +0,0 @@
1
- import ctypes
2
- import importlib.util
3
- import sys
4
- from pathlib import Path
5
- from types import ModuleType
6
-
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
- module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
- return module
24
-
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
- from .interface import * # noqa: F401,F403
2
  from .interface import __all__ # noqa: F401
 
1
+ from .interface import *
2
  from .interface import __all__ # noqa: F401
build/{torch212-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → torch212-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e5b882033d141a028bdba9871b950f4594cf0b5d4f9808fa5f70e0c25c479a0c
3
- size 202603200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4c54c9a3c3b70dc0a57b8aed916eebf2fa73f00ca4538ea7ca93d5b0fab95eb
3
+ size 196122688
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _nunchaku_lite_kernels_cuda_473b39a_dirty
3
- ops = torch.ops._nunchaku_lite_kernels_cuda_473b39a_dirty
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_nunchaku_lite_kernels_cuda_473b39a_dirty::{op_name}"
 
1
  import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/interface.py CHANGED
@@ -110,7 +110,7 @@ def _fake_fused_cross_head_qk_norm_rope(
110
 
111
  def _get_ops():
112
  try:
113
- ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
@@ -278,10 +278,10 @@ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: to
278
 
279
 
280
  __all__ = [
281
- "attention_fp16_cuda",
282
  "attention_fp16",
283
- "awq_gemm_w4a16_g128_int16",
284
  "awq_gemm_w4a16_g64_int32",
 
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
 
110
 
111
  def _get_ops():
112
  try:
113
+ _ = ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
 
278
 
279
 
280
  __all__ = [
 
281
  "attention_fp16",
282
+ "attention_fp16_cuda",
283
  "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "nunchaku-lite-kernels",
3
- "id": "_nunchaku_lite_kernels_cuda_473b39a_dirty",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -18,11 +18,21 @@
18
  "digest": {
19
  "algorithm": "sha256",
20
  "files": {
21
- "__init__.py": "87pNOdi+WC4T4fU+sP2XuEooTUR+2xxcTe6kCAOHdYo=",
22
- "_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so": "MCdzG2xyBRvSK5miLs5zewuPunVc6jLrRCYI2ZssLgE=",
23
- "_ops.py": "qfLrpvroK0kiufObpMs/guo3+BKrjakeOpBI12NNKbs=",
24
- "interface.py": "5BhmgdJFtczitG1n3byCAvCuTmTQmlaiVSpyz2ZlOYs=",
25
- "nunchaku_lite_kernels/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
 
 
 
 
 
 
 
 
 
 
26
  }
27
  }
28
  }
 
1
  {
2
  "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
18
  "digest": {
19
  "algorithm": "sha256",
20
  "files": {
21
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
22
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "9MVMmjw7cNwKV7iu2Rbuvy+nPwDKRTjqfKk9Ww+rles=",
23
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
24
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
25
+ }
26
+ },
27
+ "provenance": {
28
+ "kernel-builder": {
29
+ "version": "0.17.0-dev0",
30
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
31
+ "dirty": false
32
+ },
33
+ "kernel": {
34
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
35
+ "dirty": false
36
  }
37
  }
38
  }
build/torch212-cxx11-cu130-x86_64-linux/nunchaku_lite_kernels/__init__.py DELETED
@@ -1,26 +0,0 @@
1
- import ctypes
2
- import importlib.util
3
- import sys
4
- from pathlib import Path
5
- from types import ModuleType
6
-
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
- module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
- return module
24
-
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
- from .interface import * # noqa: F401,F403
2
  from .interface import __all__ # noqa: F401
 
1
+ from .interface import *
2
  from .interface import __all__ # noqa: F401
build/{torch212-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so → torch212-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3027731b6c72051bd22b99a22ece737b0b8fba755cea32eb442608d99b2c2e01
3
- size 196122704
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6de134574c89b2019848001ba0986e74411de14b47dfe736693bdb0e3bfa4639
3
+ size 202603184
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _nunchaku_lite_kernels_cuda_473b39a_dirty
3
- ops = torch.ops._nunchaku_lite_kernels_cuda_473b39a_dirty
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_nunchaku_lite_kernels_cuda_473b39a_dirty::{op_name}"
 
1
  import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/interface.py CHANGED
@@ -110,7 +110,7 @@ def _fake_fused_cross_head_qk_norm_rope(
110
 
111
  def _get_ops():
112
  try:
113
- ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
@@ -278,10 +278,10 @@ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: to
278
 
279
 
280
  __all__ = [
281
- "attention_fp16_cuda",
282
  "attention_fp16",
283
- "awq_gemm_w4a16_g128_int16",
284
  "awq_gemm_w4a16_g64_int32",
 
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
 
110
 
111
  def _get_ops():
112
  try:
113
+ _ = ops.gemm_w4a4
114
  except AttributeError as exc:
115
  raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
  return ops
 
278
 
279
 
280
  __all__ = [
 
281
  "attention_fp16",
282
+ "attention_fp16_cuda",
283
  "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
  "awq_gemv_w4a16_cuda",
286
  "fused_affine_modulate",
287
  "fused_cross_head_qk_norm_rope",
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "nunchaku-lite-kernels",
3
- "id": "_nunchaku_lite_kernels_cuda_473b39a_dirty",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -18,11 +18,21 @@
18
  "digest": {
19
  "algorithm": "sha256",
20
  "files": {
21
- "__init__.py": "87pNOdi+WC4T4fU+sP2XuEooTUR+2xxcTe6kCAOHdYo=",
22
- "_nunchaku_lite_kernels_cuda_473b39a_dirty.abi3.so": "5biCAz0UGgKL26mHG5UPRZTPC11PmAj6X3DgwlxHmgw=",
23
- "_ops.py": "qfLrpvroK0kiufObpMs/guo3+BKrjakeOpBI12NNKbs=",
24
- "interface.py": "5BhmgdJFtczitG1n3byCAvCuTmTQmlaiVSpyz2ZlOYs=",
25
- "nunchaku_lite_kernels/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
 
 
 
 
 
 
 
 
 
 
26
  }
27
  }
28
  }
 
1
  {
2
  "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
  "version": 2,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
18
  "digest": {
19
  "algorithm": "sha256",
20
  "files": {
21
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
22
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "beE0V0yJsgGYSAAboJhudEEd4UtH3+c2aTvbDjv6Rjk=",
23
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
24
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
25
+ }
26
+ },
27
+ "provenance": {
28
+ "kernel-builder": {
29
+ "version": "0.17.0-dev0",
30
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
31
+ "dirty": false
32
+ },
33
+ "kernel": {
34
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
35
+ "dirty": false
36
  }
37
  }
38
  }
build/torch212-cxx11-cu132-x86_64-linux/nunchaku_lite_kernels/__init__.py DELETED
@@ -1,26 +0,0 @@
1
- import ctypes
2
- import importlib.util
3
- import sys
4
- from pathlib import Path
5
- from types import ModuleType
6
-
7
-
8
- def _import_from_path(file_path: Path) -> ModuleType:
9
- # We cannot use the module name as-is, after adding it to `sys.modules`,
10
- # it would also be used for other imports. So, we make a module name that
11
- # depends on the path for it to be unique using the hex-encoded hash of
12
- # the path.
13
- path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
- module_name = path_hash
15
- spec = importlib.util.spec_from_file_location(module_name, file_path)
16
- if spec is None:
17
- raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
- module = importlib.util.module_from_spec(spec)
19
- if module is None:
20
- raise ImportError(f"Cannot load module {module_name} from spec")
21
- sys.modules[module_name] = module
22
- spec.loader.exec_module(module) # type: ignore
23
- return module
24
-
25
-
26
- globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
build/torch213-cxx11-cu126-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .interface import *
2
+ from .interface import __all__ # noqa: F401
build/torch213-cxx11-cu126-x86_64-linux/__pycache__/__init__.cpython-313.pyc ADDED
Binary file (285 Bytes). View file
 
build/torch213-cxx11-cu126-x86_64-linux/__pycache__/_ops.cpython-313.pyc ADDED
Binary file (604 Bytes). View file
 
build/torch213-cxx11-cu126-x86_64-linux/__pycache__/interface.cpython-313.pyc ADDED
Binary file (10.5 kB). View file
 
build/torch213-cxx11-cu126-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb6df62d560ea123f9f750b7e9a9cedb3f4be666e9b8902dff6320acd97fb19d
3
+ size 226224088
build/torch213-cxx11-cu126-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch213-cxx11-cu126-x86_64-linux/interface.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import math
4
+
5
+ import torch
6
+
7
+ try:
8
+ from ._ops import add_op_namespace_prefix, ops
9
+ except ImportError:
10
+ from .._ops import add_op_namespace_prefix, ops
11
+
12
+
13
+ @torch.library.register_fake(add_op_namespace_prefix("gemm_w4a4"))
14
+ def _fake_gemm_w4a4(
15
+ act,
16
+ wgt,
17
+ out,
18
+ qout,
19
+ ascales,
20
+ wscales,
21
+ oscales,
22
+ poolout,
23
+ lora_act_in,
24
+ lora_up,
25
+ lora_down,
26
+ lora_act_out,
27
+ norm_q,
28
+ norm_k,
29
+ rotary_emb,
30
+ bias,
31
+ smooth_factor,
32
+ out_vk,
33
+ out_linearattn,
34
+ act_unsigned: bool,
35
+ lora_scales: list[float],
36
+ fuse_silu: bool,
37
+ fp4: bool,
38
+ alpha,
39
+ wcscales,
40
+ out_q,
41
+ out_k,
42
+ out_v,
43
+ attn_tokens: int,
44
+ ) -> None:
45
+ return None
46
+
47
+
48
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_w4a4_act_fuse_lora"))
49
+ def _fake_quantize_w4a4_act_fuse_lora(
50
+ input,
51
+ output,
52
+ oscales,
53
+ lora_down,
54
+ lora_act_out,
55
+ smooth,
56
+ fuse_glu: bool,
57
+ fp4: bool,
58
+ ) -> None:
59
+ return None
60
+
61
+
62
+ @torch.library.register_fake(add_op_namespace_prefix("fused_rms_norm_modulate"))
63
+ def _fake_fused_rms_norm_modulate(x, norm_weight, scale, shift, eps: float):
64
+ return torch.empty_like(x)
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("fused_affine_modulate"))
68
+ def _fake_fused_affine_modulate(x, scale, shift):
69
+ return torch.empty_like(x)
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g128_int16"))
73
+ def _fake_awq_gemm_w4a16_g128_int16(in_feats, qweight, scaling_factors, zeros):
74
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
75
+
76
+
77
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g64_int32"))
78
+ def _fake_awq_gemm_w4a16_g64_int32(in_feats, qweight, scaling_factors, zeros):
79
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
80
+
81
+
82
+ @torch.library.register_fake(add_op_namespace_prefix("gemv_awq"))
83
+ def _fake_gemv_awq(in_feats, qweight, scaling_factors, zeros, m: int, n: int, k: int, group_size: int):
84
+ return in_feats.new_empty((m, n))
85
+
86
+
87
+ @torch.library.register_fake(add_op_namespace_prefix("attention_fp16"))
88
+ def _fake_attention_fp16(q, k, v, o, scale: float) -> None:
89
+ return None
90
+
91
+
92
+ @torch.library.register_fake(add_op_namespace_prefix("fused_cross_head_qk_norm_rope"))
93
+ def _fake_fused_cross_head_qk_norm_rope(
94
+ q,
95
+ k,
96
+ q_weight,
97
+ k_weight,
98
+ q_cos,
99
+ q_sin,
100
+ k_cos,
101
+ k_sin,
102
+ q_heads: int,
103
+ k_heads: int,
104
+ head_dim: int,
105
+ eps: float,
106
+ interleaved: bool,
107
+ ):
108
+ return [torch.empty_like(q), torch.empty_like(k)]
109
+
110
+
111
+ def _get_ops():
112
+ try:
113
+ _ = ops.gemm_w4a4
114
+ except AttributeError as exc:
115
+ raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
+ return ops
117
+
118
+
119
+ ops = _get_ops()
120
+
121
+ gemm_w4a4 = ops.gemm_w4a4
122
+ quantize_w4a4_act_fuse_lora = ops.quantize_w4a4_act_fuse_lora
123
+ fused_rms_norm_modulate = ops.fused_rms_norm_modulate
124
+ fused_affine_modulate = ops.fused_affine_modulate
125
+ fused_cross_head_qk_norm_rope = ops.fused_cross_head_qk_norm_rope
126
+ gemv_awq = ops.gemv_awq
127
+ attention_fp16 = ops.attention_fp16
128
+
129
+
130
+ def _ceil_divide(a: int, b: int) -> int:
131
+ return -(-a // b)
132
+
133
+
134
+ def svdq_gemm_w4a4_cuda(
135
+ act: torch.Tensor,
136
+ wgt: torch.Tensor,
137
+ out: torch.Tensor | None = None,
138
+ qout: torch.Tensor | None = None,
139
+ ascales: torch.Tensor | None = None,
140
+ wscales: torch.Tensor | None = None,
141
+ oscales: torch.Tensor | None = None,
142
+ poolout: torch.Tensor | None = None,
143
+ lora_act_in: torch.Tensor | None = None,
144
+ lora_up: torch.Tensor | None = None,
145
+ lora_down: torch.Tensor | None = None,
146
+ lora_act_out: torch.Tensor | None = None,
147
+ norm_q: torch.Tensor | None = None,
148
+ norm_k: torch.Tensor | None = None,
149
+ rotary_emb: torch.Tensor | None = None,
150
+ bias: torch.Tensor | None = None,
151
+ smooth_factor: torch.Tensor | None = None,
152
+ out_vk: torch.Tensor | None = None,
153
+ out_linearattn: torch.Tensor | None = None,
154
+ act_unsigned: bool = False,
155
+ lora_scales: list[float] | None = None,
156
+ fuse_silu: bool = False,
157
+ fp4: bool = False,
158
+ alpha: float | None = 1.0,
159
+ wcscales: torch.Tensor | None = None,
160
+ out_q: torch.Tensor | None = None,
161
+ out_k: torch.Tensor | None = None,
162
+ out_v: torch.Tensor | None = None,
163
+ attn_tokens: int = 0,
164
+ ) -> None:
165
+ if lora_scales is None:
166
+ if lora_up is None:
167
+ lora_scales = []
168
+ else:
169
+ rank = lora_up.shape[1]
170
+ lora_scales = [1.0] * math.ceil(rank / 16)
171
+ if isinstance(alpha, torch.Tensor):
172
+ alpha_arg = alpha
173
+ elif alpha is None or float(alpha) == 1.0:
174
+ alpha_arg = None
175
+ else:
176
+ alpha_arg = torch.tensor(float(alpha), device=act.device)
177
+ ops.gemm_w4a4(
178
+ act,
179
+ wgt,
180
+ out,
181
+ qout,
182
+ ascales,
183
+ wscales,
184
+ oscales,
185
+ poolout,
186
+ lora_act_in,
187
+ lora_up,
188
+ lora_down,
189
+ lora_act_out,
190
+ norm_q,
191
+ norm_k,
192
+ rotary_emb,
193
+ bias,
194
+ smooth_factor,
195
+ out_vk,
196
+ out_linearattn,
197
+ act_unsigned,
198
+ lora_scales,
199
+ fuse_silu,
200
+ fp4,
201
+ alpha_arg,
202
+ wcscales,
203
+ out_q,
204
+ out_k,
205
+ out_v,
206
+ attn_tokens,
207
+ )
208
+
209
+
210
+ def svdq_quantize_w4a4_act_fuse_lora_cuda(
211
+ input: torch.Tensor,
212
+ output: torch.Tensor | None = None,
213
+ oscales: torch.Tensor | None = None,
214
+ lora_down: torch.Tensor | None = None,
215
+ lora_act_out: torch.Tensor | None = None,
216
+ smooth: torch.Tensor | None = None,
217
+ fuse_glu: bool = False,
218
+ fp4: bool = False,
219
+ pad_size: int = 256,
220
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
221
+ if lora_down is None:
222
+ raise ValueError("lora_down is required")
223
+
224
+ batch_size, channels = input.shape
225
+ rank = lora_down.shape[1]
226
+ batch_size_pad = _ceil_divide(batch_size, pad_size) * pad_size
227
+ if output is None:
228
+ output = torch.empty(batch_size_pad, channels // 2, dtype=torch.uint8, device=input.device)
229
+ if oscales is None:
230
+ if fp4:
231
+ if channels % 16 != 0:
232
+ raise ValueError("NVFP4 activation channels must be divisible by 16")
233
+ oscales = torch.empty(channels // 16, batch_size_pad, dtype=torch.float8_e4m3fn, device=input.device)
234
+ else:
235
+ if channels % 64 != 0:
236
+ raise ValueError("INT4 activation channels must be divisible by 64")
237
+ oscales = torch.empty(channels // 64, batch_size_pad, dtype=input.dtype, device=input.device)
238
+ if lora_act_out is None:
239
+ lora_act_out = torch.empty(batch_size_pad, rank, dtype=torch.float32, device=input.device)
240
+
241
+ ops.quantize_w4a4_act_fuse_lora(input, output, oscales, lora_down, lora_act_out, smooth, fuse_glu, fp4)
242
+ return output, oscales, lora_act_out
243
+
244
+
245
+ def awq_gemm_w4a16_g128_int16(
246
+ in_feats: torch.Tensor,
247
+ kernel: torch.Tensor,
248
+ scaling_factors: torch.Tensor,
249
+ zeros: torch.Tensor,
250
+ ) -> torch.Tensor:
251
+ return ops.awq_gemm_w4a16_g128_int16(in_feats, kernel, scaling_factors, zeros)
252
+
253
+
254
+ def awq_gemm_w4a16_g64_int32(
255
+ in_feats: torch.Tensor,
256
+ kernel: torch.Tensor,
257
+ scaling_factors: torch.Tensor,
258
+ zeros: torch.Tensor,
259
+ ) -> torch.Tensor:
260
+ return ops.awq_gemm_w4a16_g64_int32(in_feats, kernel, scaling_factors, zeros)
261
+
262
+
263
+ def awq_gemv_w4a16_cuda(
264
+ in_feats: torch.Tensor,
265
+ kernel: torch.Tensor,
266
+ scaling_factors: torch.Tensor,
267
+ zeros: torch.Tensor,
268
+ m: int,
269
+ n: int,
270
+ k: int,
271
+ group_size: int = 64,
272
+ ) -> torch.Tensor:
273
+ return ops.gemv_awq(in_feats, kernel, scaling_factors, zeros, m, n, k, group_size)
274
+
275
+
276
+ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: torch.Tensor, scale: float) -> None:
277
+ ops.attention_fp16(q, k, v, o, scale)
278
+
279
+
280
+ __all__ = [
281
+ "attention_fp16",
282
+ "attention_fp16_cuda",
283
+ "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
+ "awq_gemv_w4a16_cuda",
286
+ "fused_affine_modulate",
287
+ "fused_cross_head_qk_norm_rope",
288
+ "fused_rms_norm_modulate",
289
+ "gemm_w4a4",
290
+ "gemv_awq",
291
+ "ops",
292
+ "quantize_w4a4_act_fuse_lora",
293
+ "svdq_gemm_w4a4_cuda",
294
+ "svdq_quantize_w4a4_act_fuse_lora_cuda",
295
+ ]
build/torch213-cxx11-cu126-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
+ "version": 2,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "7.5",
11
+ "8.0",
12
+ "8.6",
13
+ "8.9"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
20
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "y232LVYOoSP591C36anO2z9L5mbpuJAt/2MgrNl/sZ0=",
21
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
22
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
23
+ }
24
+ },
25
+ "provenance": {
26
+ "kernel-builder": {
27
+ "version": "0.17.0-dev0",
28
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
29
+ "dirty": false
30
+ },
31
+ "kernel": {
32
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
33
+ "dirty": false
34
+ }
35
+ }
36
+ }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .interface import *
2
+ from .interface import __all__ # noqa: F401
build/torch213-cxx11-cu130-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d120280ecae32d40a2265103eac692d9be45b4d5493d39133f5bc14beab83f9c
3
+ size 196137920
build/torch213-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/interface.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import math
4
+
5
+ import torch
6
+
7
+ try:
8
+ from ._ops import add_op_namespace_prefix, ops
9
+ except ImportError:
10
+ from .._ops import add_op_namespace_prefix, ops
11
+
12
+
13
+ @torch.library.register_fake(add_op_namespace_prefix("gemm_w4a4"))
14
+ def _fake_gemm_w4a4(
15
+ act,
16
+ wgt,
17
+ out,
18
+ qout,
19
+ ascales,
20
+ wscales,
21
+ oscales,
22
+ poolout,
23
+ lora_act_in,
24
+ lora_up,
25
+ lora_down,
26
+ lora_act_out,
27
+ norm_q,
28
+ norm_k,
29
+ rotary_emb,
30
+ bias,
31
+ smooth_factor,
32
+ out_vk,
33
+ out_linearattn,
34
+ act_unsigned: bool,
35
+ lora_scales: list[float],
36
+ fuse_silu: bool,
37
+ fp4: bool,
38
+ alpha,
39
+ wcscales,
40
+ out_q,
41
+ out_k,
42
+ out_v,
43
+ attn_tokens: int,
44
+ ) -> None:
45
+ return None
46
+
47
+
48
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_w4a4_act_fuse_lora"))
49
+ def _fake_quantize_w4a4_act_fuse_lora(
50
+ input,
51
+ output,
52
+ oscales,
53
+ lora_down,
54
+ lora_act_out,
55
+ smooth,
56
+ fuse_glu: bool,
57
+ fp4: bool,
58
+ ) -> None:
59
+ return None
60
+
61
+
62
+ @torch.library.register_fake(add_op_namespace_prefix("fused_rms_norm_modulate"))
63
+ def _fake_fused_rms_norm_modulate(x, norm_weight, scale, shift, eps: float):
64
+ return torch.empty_like(x)
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("fused_affine_modulate"))
68
+ def _fake_fused_affine_modulate(x, scale, shift):
69
+ return torch.empty_like(x)
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g128_int16"))
73
+ def _fake_awq_gemm_w4a16_g128_int16(in_feats, qweight, scaling_factors, zeros):
74
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
75
+
76
+
77
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g64_int32"))
78
+ def _fake_awq_gemm_w4a16_g64_int32(in_feats, qweight, scaling_factors, zeros):
79
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
80
+
81
+
82
+ @torch.library.register_fake(add_op_namespace_prefix("gemv_awq"))
83
+ def _fake_gemv_awq(in_feats, qweight, scaling_factors, zeros, m: int, n: int, k: int, group_size: int):
84
+ return in_feats.new_empty((m, n))
85
+
86
+
87
+ @torch.library.register_fake(add_op_namespace_prefix("attention_fp16"))
88
+ def _fake_attention_fp16(q, k, v, o, scale: float) -> None:
89
+ return None
90
+
91
+
92
+ @torch.library.register_fake(add_op_namespace_prefix("fused_cross_head_qk_norm_rope"))
93
+ def _fake_fused_cross_head_qk_norm_rope(
94
+ q,
95
+ k,
96
+ q_weight,
97
+ k_weight,
98
+ q_cos,
99
+ q_sin,
100
+ k_cos,
101
+ k_sin,
102
+ q_heads: int,
103
+ k_heads: int,
104
+ head_dim: int,
105
+ eps: float,
106
+ interleaved: bool,
107
+ ):
108
+ return [torch.empty_like(q), torch.empty_like(k)]
109
+
110
+
111
+ def _get_ops():
112
+ try:
113
+ _ = ops.gemm_w4a4
114
+ except AttributeError as exc:
115
+ raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
+ return ops
117
+
118
+
119
+ ops = _get_ops()
120
+
121
+ gemm_w4a4 = ops.gemm_w4a4
122
+ quantize_w4a4_act_fuse_lora = ops.quantize_w4a4_act_fuse_lora
123
+ fused_rms_norm_modulate = ops.fused_rms_norm_modulate
124
+ fused_affine_modulate = ops.fused_affine_modulate
125
+ fused_cross_head_qk_norm_rope = ops.fused_cross_head_qk_norm_rope
126
+ gemv_awq = ops.gemv_awq
127
+ attention_fp16 = ops.attention_fp16
128
+
129
+
130
+ def _ceil_divide(a: int, b: int) -> int:
131
+ return -(-a // b)
132
+
133
+
134
+ def svdq_gemm_w4a4_cuda(
135
+ act: torch.Tensor,
136
+ wgt: torch.Tensor,
137
+ out: torch.Tensor | None = None,
138
+ qout: torch.Tensor | None = None,
139
+ ascales: torch.Tensor | None = None,
140
+ wscales: torch.Tensor | None = None,
141
+ oscales: torch.Tensor | None = None,
142
+ poolout: torch.Tensor | None = None,
143
+ lora_act_in: torch.Tensor | None = None,
144
+ lora_up: torch.Tensor | None = None,
145
+ lora_down: torch.Tensor | None = None,
146
+ lora_act_out: torch.Tensor | None = None,
147
+ norm_q: torch.Tensor | None = None,
148
+ norm_k: torch.Tensor | None = None,
149
+ rotary_emb: torch.Tensor | None = None,
150
+ bias: torch.Tensor | None = None,
151
+ smooth_factor: torch.Tensor | None = None,
152
+ out_vk: torch.Tensor | None = None,
153
+ out_linearattn: torch.Tensor | None = None,
154
+ act_unsigned: bool = False,
155
+ lora_scales: list[float] | None = None,
156
+ fuse_silu: bool = False,
157
+ fp4: bool = False,
158
+ alpha: float | None = 1.0,
159
+ wcscales: torch.Tensor | None = None,
160
+ out_q: torch.Tensor | None = None,
161
+ out_k: torch.Tensor | None = None,
162
+ out_v: torch.Tensor | None = None,
163
+ attn_tokens: int = 0,
164
+ ) -> None:
165
+ if lora_scales is None:
166
+ if lora_up is None:
167
+ lora_scales = []
168
+ else:
169
+ rank = lora_up.shape[1]
170
+ lora_scales = [1.0] * math.ceil(rank / 16)
171
+ if isinstance(alpha, torch.Tensor):
172
+ alpha_arg = alpha
173
+ elif alpha is None or float(alpha) == 1.0:
174
+ alpha_arg = None
175
+ else:
176
+ alpha_arg = torch.tensor(float(alpha), device=act.device)
177
+ ops.gemm_w4a4(
178
+ act,
179
+ wgt,
180
+ out,
181
+ qout,
182
+ ascales,
183
+ wscales,
184
+ oscales,
185
+ poolout,
186
+ lora_act_in,
187
+ lora_up,
188
+ lora_down,
189
+ lora_act_out,
190
+ norm_q,
191
+ norm_k,
192
+ rotary_emb,
193
+ bias,
194
+ smooth_factor,
195
+ out_vk,
196
+ out_linearattn,
197
+ act_unsigned,
198
+ lora_scales,
199
+ fuse_silu,
200
+ fp4,
201
+ alpha_arg,
202
+ wcscales,
203
+ out_q,
204
+ out_k,
205
+ out_v,
206
+ attn_tokens,
207
+ )
208
+
209
+
210
+ def svdq_quantize_w4a4_act_fuse_lora_cuda(
211
+ input: torch.Tensor,
212
+ output: torch.Tensor | None = None,
213
+ oscales: torch.Tensor | None = None,
214
+ lora_down: torch.Tensor | None = None,
215
+ lora_act_out: torch.Tensor | None = None,
216
+ smooth: torch.Tensor | None = None,
217
+ fuse_glu: bool = False,
218
+ fp4: bool = False,
219
+ pad_size: int = 256,
220
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
221
+ if lora_down is None:
222
+ raise ValueError("lora_down is required")
223
+
224
+ batch_size, channels = input.shape
225
+ rank = lora_down.shape[1]
226
+ batch_size_pad = _ceil_divide(batch_size, pad_size) * pad_size
227
+ if output is None:
228
+ output = torch.empty(batch_size_pad, channels // 2, dtype=torch.uint8, device=input.device)
229
+ if oscales is None:
230
+ if fp4:
231
+ if channels % 16 != 0:
232
+ raise ValueError("NVFP4 activation channels must be divisible by 16")
233
+ oscales = torch.empty(channels // 16, batch_size_pad, dtype=torch.float8_e4m3fn, device=input.device)
234
+ else:
235
+ if channels % 64 != 0:
236
+ raise ValueError("INT4 activation channels must be divisible by 64")
237
+ oscales = torch.empty(channels // 64, batch_size_pad, dtype=input.dtype, device=input.device)
238
+ if lora_act_out is None:
239
+ lora_act_out = torch.empty(batch_size_pad, rank, dtype=torch.float32, device=input.device)
240
+
241
+ ops.quantize_w4a4_act_fuse_lora(input, output, oscales, lora_down, lora_act_out, smooth, fuse_glu, fp4)
242
+ return output, oscales, lora_act_out
243
+
244
+
245
+ def awq_gemm_w4a16_g128_int16(
246
+ in_feats: torch.Tensor,
247
+ kernel: torch.Tensor,
248
+ scaling_factors: torch.Tensor,
249
+ zeros: torch.Tensor,
250
+ ) -> torch.Tensor:
251
+ return ops.awq_gemm_w4a16_g128_int16(in_feats, kernel, scaling_factors, zeros)
252
+
253
+
254
+ def awq_gemm_w4a16_g64_int32(
255
+ in_feats: torch.Tensor,
256
+ kernel: torch.Tensor,
257
+ scaling_factors: torch.Tensor,
258
+ zeros: torch.Tensor,
259
+ ) -> torch.Tensor:
260
+ return ops.awq_gemm_w4a16_g64_int32(in_feats, kernel, scaling_factors, zeros)
261
+
262
+
263
+ def awq_gemv_w4a16_cuda(
264
+ in_feats: torch.Tensor,
265
+ kernel: torch.Tensor,
266
+ scaling_factors: torch.Tensor,
267
+ zeros: torch.Tensor,
268
+ m: int,
269
+ n: int,
270
+ k: int,
271
+ group_size: int = 64,
272
+ ) -> torch.Tensor:
273
+ return ops.gemv_awq(in_feats, kernel, scaling_factors, zeros, m, n, k, group_size)
274
+
275
+
276
+ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: torch.Tensor, scale: float) -> None:
277
+ ops.attention_fp16(q, k, v, o, scale)
278
+
279
+
280
+ __all__ = [
281
+ "attention_fp16",
282
+ "attention_fp16_cuda",
283
+ "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
+ "awq_gemv_w4a16_cuda",
286
+ "fused_affine_modulate",
287
+ "fused_cross_head_qk_norm_rope",
288
+ "fused_rms_norm_modulate",
289
+ "gemm_w4a4",
290
+ "gemv_awq",
291
+ "ops",
292
+ "quantize_w4a4_act_fuse_lora",
293
+ "svdq_gemm_w4a4_cuda",
294
+ "svdq_quantize_w4a4_act_fuse_lora_cuda",
295
+ ]
build/torch213-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
+ "version": 2,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "12.1a",
12
+ "7.5",
13
+ "8.0",
14
+ "8.6",
15
+ "8.9"
16
+ ]
17
+ },
18
+ "digest": {
19
+ "algorithm": "sha256",
20
+ "files": {
21
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
22
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "0SAoDsrjLUCiJlED6saS2b5FtNVJPTkTP1vBS+q4P5w=",
23
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
24
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
25
+ }
26
+ },
27
+ "provenance": {
28
+ "kernel-builder": {
29
+ "version": "0.17.0-dev0",
30
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
31
+ "dirty": false
32
+ },
33
+ "kernel": {
34
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
35
+ "dirty": false
36
+ }
37
+ }
38
+ }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .interface import *
2
+ from .interface import __all__ # noqa: F401
build/torch213-cxx11-cu132-x86_64-linux/_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c69509cc9da2e38fea1f4622f8138f522212f25823f353110b1d434a1b0c063
3
+ size 202624464
build/torch213-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _nunchaku_lite_kernels_cuda_c1b1dfd
3
+ ops = torch.ops._nunchaku_lite_kernels_cuda_c1b1dfd
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_nunchaku_lite_kernels_cuda_c1b1dfd::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/interface.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import math
4
+
5
+ import torch
6
+
7
+ try:
8
+ from ._ops import add_op_namespace_prefix, ops
9
+ except ImportError:
10
+ from .._ops import add_op_namespace_prefix, ops
11
+
12
+
13
+ @torch.library.register_fake(add_op_namespace_prefix("gemm_w4a4"))
14
+ def _fake_gemm_w4a4(
15
+ act,
16
+ wgt,
17
+ out,
18
+ qout,
19
+ ascales,
20
+ wscales,
21
+ oscales,
22
+ poolout,
23
+ lora_act_in,
24
+ lora_up,
25
+ lora_down,
26
+ lora_act_out,
27
+ norm_q,
28
+ norm_k,
29
+ rotary_emb,
30
+ bias,
31
+ smooth_factor,
32
+ out_vk,
33
+ out_linearattn,
34
+ act_unsigned: bool,
35
+ lora_scales: list[float],
36
+ fuse_silu: bool,
37
+ fp4: bool,
38
+ alpha,
39
+ wcscales,
40
+ out_q,
41
+ out_k,
42
+ out_v,
43
+ attn_tokens: int,
44
+ ) -> None:
45
+ return None
46
+
47
+
48
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_w4a4_act_fuse_lora"))
49
+ def _fake_quantize_w4a4_act_fuse_lora(
50
+ input,
51
+ output,
52
+ oscales,
53
+ lora_down,
54
+ lora_act_out,
55
+ smooth,
56
+ fuse_glu: bool,
57
+ fp4: bool,
58
+ ) -> None:
59
+ return None
60
+
61
+
62
+ @torch.library.register_fake(add_op_namespace_prefix("fused_rms_norm_modulate"))
63
+ def _fake_fused_rms_norm_modulate(x, norm_weight, scale, shift, eps: float):
64
+ return torch.empty_like(x)
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("fused_affine_modulate"))
68
+ def _fake_fused_affine_modulate(x, scale, shift):
69
+ return torch.empty_like(x)
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g128_int16"))
73
+ def _fake_awq_gemm_w4a16_g128_int16(in_feats, qweight, scaling_factors, zeros):
74
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
75
+
76
+
77
+ @torch.library.register_fake(add_op_namespace_prefix("awq_gemm_w4a16_g64_int32"))
78
+ def _fake_awq_gemm_w4a16_g64_int32(in_feats, qweight, scaling_factors, zeros):
79
+ return in_feats.new_empty((*in_feats.shape[:-1], qweight.shape[0] * 4))
80
+
81
+
82
+ @torch.library.register_fake(add_op_namespace_prefix("gemv_awq"))
83
+ def _fake_gemv_awq(in_feats, qweight, scaling_factors, zeros, m: int, n: int, k: int, group_size: int):
84
+ return in_feats.new_empty((m, n))
85
+
86
+
87
+ @torch.library.register_fake(add_op_namespace_prefix("attention_fp16"))
88
+ def _fake_attention_fp16(q, k, v, o, scale: float) -> None:
89
+ return None
90
+
91
+
92
+ @torch.library.register_fake(add_op_namespace_prefix("fused_cross_head_qk_norm_rope"))
93
+ def _fake_fused_cross_head_qk_norm_rope(
94
+ q,
95
+ k,
96
+ q_weight,
97
+ k_weight,
98
+ q_cos,
99
+ q_sin,
100
+ k_cos,
101
+ k_sin,
102
+ q_heads: int,
103
+ k_heads: int,
104
+ head_dim: int,
105
+ eps: float,
106
+ interleaved: bool,
107
+ ):
108
+ return [torch.empty_like(q), torch.empty_like(k)]
109
+
110
+
111
+ def _get_ops():
112
+ try:
113
+ _ = ops.gemm_w4a4
114
+ except AttributeError as exc:
115
+ raise ImportError("nunchaku_lite_kernels extension did not register dispatcher ops") from exc
116
+ return ops
117
+
118
+
119
+ ops = _get_ops()
120
+
121
+ gemm_w4a4 = ops.gemm_w4a4
122
+ quantize_w4a4_act_fuse_lora = ops.quantize_w4a4_act_fuse_lora
123
+ fused_rms_norm_modulate = ops.fused_rms_norm_modulate
124
+ fused_affine_modulate = ops.fused_affine_modulate
125
+ fused_cross_head_qk_norm_rope = ops.fused_cross_head_qk_norm_rope
126
+ gemv_awq = ops.gemv_awq
127
+ attention_fp16 = ops.attention_fp16
128
+
129
+
130
+ def _ceil_divide(a: int, b: int) -> int:
131
+ return -(-a // b)
132
+
133
+
134
+ def svdq_gemm_w4a4_cuda(
135
+ act: torch.Tensor,
136
+ wgt: torch.Tensor,
137
+ out: torch.Tensor | None = None,
138
+ qout: torch.Tensor | None = None,
139
+ ascales: torch.Tensor | None = None,
140
+ wscales: torch.Tensor | None = None,
141
+ oscales: torch.Tensor | None = None,
142
+ poolout: torch.Tensor | None = None,
143
+ lora_act_in: torch.Tensor | None = None,
144
+ lora_up: torch.Tensor | None = None,
145
+ lora_down: torch.Tensor | None = None,
146
+ lora_act_out: torch.Tensor | None = None,
147
+ norm_q: torch.Tensor | None = None,
148
+ norm_k: torch.Tensor | None = None,
149
+ rotary_emb: torch.Tensor | None = None,
150
+ bias: torch.Tensor | None = None,
151
+ smooth_factor: torch.Tensor | None = None,
152
+ out_vk: torch.Tensor | None = None,
153
+ out_linearattn: torch.Tensor | None = None,
154
+ act_unsigned: bool = False,
155
+ lora_scales: list[float] | None = None,
156
+ fuse_silu: bool = False,
157
+ fp4: bool = False,
158
+ alpha: float | None = 1.0,
159
+ wcscales: torch.Tensor | None = None,
160
+ out_q: torch.Tensor | None = None,
161
+ out_k: torch.Tensor | None = None,
162
+ out_v: torch.Tensor | None = None,
163
+ attn_tokens: int = 0,
164
+ ) -> None:
165
+ if lora_scales is None:
166
+ if lora_up is None:
167
+ lora_scales = []
168
+ else:
169
+ rank = lora_up.shape[1]
170
+ lora_scales = [1.0] * math.ceil(rank / 16)
171
+ if isinstance(alpha, torch.Tensor):
172
+ alpha_arg = alpha
173
+ elif alpha is None or float(alpha) == 1.0:
174
+ alpha_arg = None
175
+ else:
176
+ alpha_arg = torch.tensor(float(alpha), device=act.device)
177
+ ops.gemm_w4a4(
178
+ act,
179
+ wgt,
180
+ out,
181
+ qout,
182
+ ascales,
183
+ wscales,
184
+ oscales,
185
+ poolout,
186
+ lora_act_in,
187
+ lora_up,
188
+ lora_down,
189
+ lora_act_out,
190
+ norm_q,
191
+ norm_k,
192
+ rotary_emb,
193
+ bias,
194
+ smooth_factor,
195
+ out_vk,
196
+ out_linearattn,
197
+ act_unsigned,
198
+ lora_scales,
199
+ fuse_silu,
200
+ fp4,
201
+ alpha_arg,
202
+ wcscales,
203
+ out_q,
204
+ out_k,
205
+ out_v,
206
+ attn_tokens,
207
+ )
208
+
209
+
210
+ def svdq_quantize_w4a4_act_fuse_lora_cuda(
211
+ input: torch.Tensor,
212
+ output: torch.Tensor | None = None,
213
+ oscales: torch.Tensor | None = None,
214
+ lora_down: torch.Tensor | None = None,
215
+ lora_act_out: torch.Tensor | None = None,
216
+ smooth: torch.Tensor | None = None,
217
+ fuse_glu: bool = False,
218
+ fp4: bool = False,
219
+ pad_size: int = 256,
220
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
221
+ if lora_down is None:
222
+ raise ValueError("lora_down is required")
223
+
224
+ batch_size, channels = input.shape
225
+ rank = lora_down.shape[1]
226
+ batch_size_pad = _ceil_divide(batch_size, pad_size) * pad_size
227
+ if output is None:
228
+ output = torch.empty(batch_size_pad, channels // 2, dtype=torch.uint8, device=input.device)
229
+ if oscales is None:
230
+ if fp4:
231
+ if channels % 16 != 0:
232
+ raise ValueError("NVFP4 activation channels must be divisible by 16")
233
+ oscales = torch.empty(channels // 16, batch_size_pad, dtype=torch.float8_e4m3fn, device=input.device)
234
+ else:
235
+ if channels % 64 != 0:
236
+ raise ValueError("INT4 activation channels must be divisible by 64")
237
+ oscales = torch.empty(channels // 64, batch_size_pad, dtype=input.dtype, device=input.device)
238
+ if lora_act_out is None:
239
+ lora_act_out = torch.empty(batch_size_pad, rank, dtype=torch.float32, device=input.device)
240
+
241
+ ops.quantize_w4a4_act_fuse_lora(input, output, oscales, lora_down, lora_act_out, smooth, fuse_glu, fp4)
242
+ return output, oscales, lora_act_out
243
+
244
+
245
+ def awq_gemm_w4a16_g128_int16(
246
+ in_feats: torch.Tensor,
247
+ kernel: torch.Tensor,
248
+ scaling_factors: torch.Tensor,
249
+ zeros: torch.Tensor,
250
+ ) -> torch.Tensor:
251
+ return ops.awq_gemm_w4a16_g128_int16(in_feats, kernel, scaling_factors, zeros)
252
+
253
+
254
+ def awq_gemm_w4a16_g64_int32(
255
+ in_feats: torch.Tensor,
256
+ kernel: torch.Tensor,
257
+ scaling_factors: torch.Tensor,
258
+ zeros: torch.Tensor,
259
+ ) -> torch.Tensor:
260
+ return ops.awq_gemm_w4a16_g64_int32(in_feats, kernel, scaling_factors, zeros)
261
+
262
+
263
+ def awq_gemv_w4a16_cuda(
264
+ in_feats: torch.Tensor,
265
+ kernel: torch.Tensor,
266
+ scaling_factors: torch.Tensor,
267
+ zeros: torch.Tensor,
268
+ m: int,
269
+ n: int,
270
+ k: int,
271
+ group_size: int = 64,
272
+ ) -> torch.Tensor:
273
+ return ops.gemv_awq(in_feats, kernel, scaling_factors, zeros, m, n, k, group_size)
274
+
275
+
276
+ def attention_fp16_cuda(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, o: torch.Tensor, scale: float) -> None:
277
+ ops.attention_fp16(q, k, v, o, scale)
278
+
279
+
280
+ __all__ = [
281
+ "attention_fp16",
282
+ "attention_fp16_cuda",
283
+ "awq_gemm_w4a16_g64_int32",
284
+ "awq_gemm_w4a16_g128_int16",
285
+ "awq_gemv_w4a16_cuda",
286
+ "fused_affine_modulate",
287
+ "fused_cross_head_qk_norm_rope",
288
+ "fused_rms_norm_modulate",
289
+ "gemm_w4a4",
290
+ "gemv_awq",
291
+ "ops",
292
+ "quantize_w4a4_act_fuse_lora",
293
+ "svdq_gemm_w4a4_cuda",
294
+ "svdq_quantize_w4a4_act_fuse_lora_cuda",
295
+ ]
build/torch213-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "nunchaku-lite-kernels",
3
+ "id": "_nunchaku_lite_kernels_cuda_c1b1dfd",
4
+ "version": 2,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0a",
11
+ "12.1a",
12
+ "7.5",
13
+ "8.0",
14
+ "8.6",
15
+ "8.9"
16
+ ]
17
+ },
18
+ "digest": {
19
+ "algorithm": "sha256",
20
+ "files": {
21
+ "__init__.py": "0eerIwJuv/RvYVPyuLYASiF7MseIk8ZxabQbfDR07Hs=",
22
+ "_nunchaku_lite_kernels_cuda_c1b1dfd.abi3.so": "fGlQnMnaLjj+ofRiL4E49SIhLyWCPzUxELHUNKGwwGM=",
23
+ "_ops.py": "yujtjMiZlmvBdrCs8mraeNJ94fjq/f6Gii1dF/pIvQw=",
24
+ "interface.py": "KydtqsOlHqYRN2QX9Nfu+cedcRxODHOj+G4UVmWQ1lk="
25
+ }
26
+ },
27
+ "provenance": {
28
+ "kernel-builder": {
29
+ "version": "0.17.0-dev0",
30
+ "sha": "1bbe76c612034e0f809716a166f1fa777cf40246",
31
+ "dirty": false
32
+ },
33
+ "kernel": {
34
+ "sha": "c1b1dfd6990a79cbbfc06b337561d3b44ae1ed04",
35
+ "dirty": false
36
+ }
37
+ }
38
+ }