flash-attention-turing/setup.py at main · systemEng-Learning/flash-attention-turing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from setuptools import setup
from torch.utils.cpp_extension import CUDAExtension, BuildExtension
import os
from pathlib import Path
import subprocess

this_dir = os.path.dirname(os.path.abspath(__file__))

if os.path.isdir(".git"):
    subprocess.run(["git", "submodule", "update", "--init", "csrc/cutlass"], check=True)

nvcc_flags = ["-std=c++17",
              "--expt-relaxed-constexpr",
              "-arch=sm_75",
              "-O3",
              "--use_fast_math",
              "--ptxas-options=-v",
              "-lineinfo"]

setup(
    name="flash_attn_turing",
    ext_modules=[
        CUDAExtension(
            name="flash_attn_turing",
            sources=["csrc/flash_attn/flash_api.cpp",
                     "csrc/flash_attn/src/flash_fwd_hdim64_fp16_sm75.cu",
                     "csrc/flash_attn/src/flash_fwd_hdim64_fp16_causal_sm75.cu",
                     "csrc/flash_attn/src/flash_fwd_hdim128_fp16_sm75.cu",
                     "csrc/flash_attn/src/flash_fwd_hdim128_fp16_causal_sm75.cu",
                     "csrc/flash_attn/src/flash_bwd_hdim64_fp16_sm75.cu",
                     "csrc/flash_attn/src/flash_bwd_hdim64_fp16_causal_sm75.cu",
                     "csrc/flash_attn/src/flash_bwd_hdim128_fp16_sm75.cu",
                     "csrc/flash_attn/src/flash_bwd_hdim128_fp16_causal_sm75.cu"
                     ],
            include_dirs=[
                Path(this_dir) / "csrc" / "flash_attn",
                Path(this_dir) / "csrc" / "flash_attn" / "src",
                Path(this_dir) / "csrc" / "cutlass" / "include",
                Path(this_dir) / "csrc" / "cutlass" / "tools/util/include"
                ],
            extra_compile_args={'nvcc': nvcc_flags}
        )
    ],
    install_requires=["torch"],
    cmdclass={"build_ext": BuildExtension}
)