Skip to content

Commit 3de8daa

Browse files
jishnubdkarrasch
authored andcommitted
Fix GEMM dispatch for complex-real matmul (#1520)
This should fix #1519. The issue on master is that we have specialized dispatches for arrays of the same eltype, and the complex-real matmul ends up in `generic_matmatmul!`. This adds an extra method to ensure that the complex-real case also reaches BLAS. ```julia julia> A = ones(ComplexF64, 400, 400); B = ones(size(A)); C = similar(A); julia> @Btime mul!($C, $A, $B); 57.043 ms (0 allocations: 0 bytes) # master 1.608 ms (0 allocations: 0 bytes) # this PR ```
1 parent b17cd17 commit 3de8daa

File tree

1 file changed

+8
-10
lines changed

1 file changed

+8
-10
lines changed

src/matmul.jl

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ end
312312
BlasFlag.SYRK
313313
elseif (tA_uc == 'C' && tB_uc == 'N') || (tA_uc == 'N' && tB_uc == 'C')
314314
BlasFlag.HERK
315-
else isntc
315+
else
316316
BlasFlag.GEMM
317317
end
318318
else
@@ -494,7 +494,7 @@ function matmul2x2or3x3_nonzeroalpha!(C, tA, tB, A, B, α::Bool, β)
494494
return false
495495
end
496496

497-
# THE one big BLAS dispatch. This is split into two methods to improve latency
497+
# THE one big BLAS dispatch. This is split into syrk/herk/gemm and symm/hemm/none methods to improve latency
498498
Base.@constprop :aggressive function generic_matmatmul_wrapper!(C::StridedMatrix{T}, tA, tB, A::StridedVecOrMat{T}, B::StridedVecOrMat{T},
499499
α::Number, β::Number, val::BlasFlag.SyrkHerkGemm) where {T<:BlasFloat}
500500
mA, nA = lapack_size(tA, A)
@@ -507,6 +507,12 @@ Base.@constprop :aggressive function generic_matmatmul_wrapper!(C::StridedMatrix
507507
_syrk_herk_gemm_wrapper!(C, tA, tB, A, B, α, β, val)
508508
return C
509509
end
510+
511+
function generic_matmatmul_wrapper!(C::StridedVecOrMat{Complex{T}}, tA, tB, A::StridedVecOrMat{Complex{T}}, B::StridedVecOrMat{T},
512+
α::Number, β::Number, ::Val{BlasFlag.GEMM}) where {T<:BlasReal}
513+
gemm_wrapper!(C, tA, tB, A, B, α, β)
514+
end
515+
510516
Base.@constprop :aggressive function _syrk_herk_gemm_wrapper!(C, tA, tB, A, B, α, β, ::Val{BlasFlag.SYRK})
511517
if A === B
512518
tA_uc = uppercase(tA) # potentially strip a WrapperChar
@@ -583,14 +589,6 @@ Base.@constprop :aggressive generic_matmatmul!(C::StridedMatrix{T}, tA, tB, A::S
583589
_add::MulAddMul = MulAddMul()) where {T<:BlasFloat} =
584590
generic_matmatmul!(C, tA, tB, A, B, _add.alpha, _add.beta)
585591

586-
function generic_matmatmul_wrapper!(C::StridedVecOrMat{Complex{T}}, tA, tB, A::StridedVecOrMat{Complex{T}}, B::StridedVecOrMat{T},
587-
α::Number, β::Number, ::Val{true}) where {T<:BlasReal}
588-
gemm_wrapper!(C, tA, tB, A, B, α, β)
589-
end
590-
Base.@constprop :aggressive function generic_matmatmul_wrapper!(C::StridedVecOrMat{Complex{T}}, tA, tB, A::StridedVecOrMat{Complex{T}}, B::StridedVecOrMat{T},
591-
alpha::Number, beta::Number, ::Val{false}) where {T<:BlasReal}
592-
_generic_matmatmul!(C, wrap(A, tA), wrap(B, tB), alpha, beta)
593-
end
594592
# legacy method
595593
Base.@constprop :aggressive generic_matmatmul!(C::StridedVecOrMat{Complex{T}}, tA, tB, A::StridedVecOrMat{Complex{T}}, B::StridedVecOrMat{T},
596594
_add::MulAddMul = MulAddMul()) where {T<:BlasReal} =

0 commit comments

Comments
 (0)