mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-04-26 08:36:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

vulkan: Use fp16 for the flash attention P*V multiplication ()

Browse Source 
This is consistent with the ggml-cuda behavior and the mul_mat fallback.
This commit is contained in:
Jeff Bolz 2025-04-09 00:12:57 -05:00 committed by GitHub
parent 7538246e7c
commit 7ecd780b1a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions

6
ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp

@ -330,9 +330,11 @@ void main() {
// resize eM by using smear/reduce // resize eM by using smear/reduce
coopMatReduceNV(eMdiag, eM, gl_CooperativeMatrixReduceRowNV, smearReduce); coopMatReduceNV(eMdiag, eM, gl_CooperativeMatrixReduceRowNV, smearReduce);
O = eMdiag * O; // multiply with fp16 accumulation, then add to O.
coopmat<float16_t, gl_ScopeWorkgroup, Br, D, gl_MatrixUseAccumulator> PV = coopmat<float16_t, gl_ScopeWorkgroup, Br, D, gl_MatrixUseAccumulator>(0);
PV = coopMatMulAdd(P_A, V, PV);
O = coopMatMulAdd(P_A, V, O); O = eMdiag * O + coopmat<ACC_TYPE, gl_ScopeWorkgroup, Br, D, gl_MatrixUseAccumulator>(PV);
} }
// If there is split_k, then the split_k resolve shader does the final // If there is split_k, then the split_k resolve shader does the final