From de961fd390894d1005574d82f6e8da0cb2835b4b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 19 Aug 2025 21:20:42 +0800 Subject: [PATCH 1/3] Precommit tests --- .../expand-vp-convert-evl.ll | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll new file mode 100644 index 0000000000000..191edcc9324c3 --- /dev/null +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll @@ -0,0 +1,119 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes=pre-isel-intrinsic-lowering -expandvp-override-evl-transform=Convert -expandvp-override-mask-transform=Legal -S < %s | FileCheck %s + +define @unknown_evl( %x, %y, %m, i32 %evl) { +; CHECK-LABEL: define @unknown_evl( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]], i32 [[EVL:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: ret [[ADD]] +; + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} + +define <2 x i64> @exact_evl_fixed(<2 x i64> %x, <2 x i64> %y, <2 x i1> %m) { +; CHECK-LABEL: define <2 x i64> @exact_evl_fixed( +; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i1> [[M:%.*]]) { +; CHECK-NEXT: [[ADD:%.*]] = call <2 x i64> @llvm.vp.add.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> [[M]], i32 2) +; CHECK-NEXT: ret <2 x i64> [[ADD]] +; + %add = call <2 x i64> @llvm.vp.add(<2 x i64> poison, <2 x i64> poison, <2 x i1> %m, i32 2) + ret <2 x i64> %add +} + +define @exact_evl_vscale_mul( %x, %y, %m) { +; CHECK-LABEL: define @exact_evl_vscale_mul( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[EVL:%.*]] = mul i32 [[VSCALE]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) +; CHECK-NEXT: ret [[ADD]] +; + %vscale = call i32 @llvm.vscale() + %evl = mul i32 %vscale, 2 + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} + +define @exact_evl_vscale_shl( %x, %y, %m) { +; CHECK-LABEL: define @exact_evl_vscale_shl( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { +; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[EVL:%.*]] = shl i32 [[VSCALE]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] +; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: ret [[ADD]] +; + %vscale = call i32 @llvm.vscale() + %evl = shl i32 %vscale, 1 + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} + +define @exact_evl_vscale_mul_trunc( %x, %y, %m) { +; CHECK-LABEL: define @exact_evl_vscale_mul_trunc( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[SHL:%.*]] = mul i64 [[VSCALE]], 2 +; CHECK-NEXT: [[EVL:%.*]] = trunc nuw i64 [[SHL]] to i32 +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] +; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: ret [[ADD]] +; + %vscale = call i64 @llvm.vscale() + %shl = mul i64 %vscale, 2 + %evl = trunc nuw i64 %shl to i32 + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} + + +define @exact_evl_vscale_shl_trunc( %x, %y, %m) { +; CHECK-LABEL: define @exact_evl_vscale_shl_trunc( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[SHL:%.*]] = shl i64 [[VSCALE]], 1 +; CHECK-NEXT: [[EVL:%.*]] = trunc nuw i64 [[SHL]] to i32 +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] +; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: ret [[ADD]] +; + %vscale = call i64 @llvm.vscale() + %shl = shl i64 %vscale, 1 + %evl = trunc nuw i64 %shl to i32 + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} + +define @exact_evl_vscale_shl_trunc_no_nuw( %x, %y, %m) { +; CHECK-LABEL: define @exact_evl_vscale_shl_trunc_no_nuw( +; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { +; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[SHL:%.*]] = shl i64 [[VSCALE]], 1 +; CHECK-NEXT: [[EVL:%.*]] = trunc i64 [[SHL]] to i32 +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) +; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] +; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: ret [[ADD]] +; + %vscale = call i64 @llvm.vscale() + %shl = shl i64 %vscale, 1 + %evl = trunc i64 %shl to i32 + %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) + ret %add +} From b13a35aa44d5cc4ea8c148ca523d3b7f553d1c4e Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 19 Aug 2025 21:23:35 +0800 Subject: [PATCH 2/3] [VP] Detect truncated and shifted EVLs during expansion --- llvm/include/llvm/IR/PatternMatch.h | 7 +++++++ llvm/lib/IR/IntrinsicInst.cpp | 6 +++++- .../expand-vp-convert-evl.ll | 18 +++--------------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 2ab652ca258c6..ec4dc787747ca 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -2193,6 +2193,13 @@ m_TruncOrSelf(const OpTy &Op) { return m_CombineOr(m_Trunc(Op), Op); } +template +inline match_combine_or, + OpTy> +m_NUWTruncOrSelf(const OpTy &Op) { + return m_CombineOr(m_NUWTrunc(Op), Op); +} + /// Matches SExt. template inline CastInst_match m_SExt(const OpTy &Op) { diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 23a4d1b5c615e..22c1479174e2c 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -613,8 +613,12 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const { if (EC.isScalable()) { // Compare vscale patterns uint64_t VScaleFactor; - if (match(VLParam, m_Mul(m_VScale(), m_ConstantInt(VScaleFactor)))) + if (match(VLParam, + m_NUWTruncOrSelf(m_Mul(m_VScale(), m_ConstantInt(VScaleFactor))))) return VScaleFactor >= EC.getKnownMinValue(); + if (match(VLParam, + m_NUWTruncOrSelf(m_Shl(m_VScale(), m_ConstantInt(VScaleFactor))))) + return 1 << VScaleFactor >= EC.getKnownMinValue(); return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale()); } diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll index 191edcc9324c3..8dd8ae153ae58 100644 --- a/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll @@ -44,11 +44,7 @@ define @exact_evl_vscale_shl( %x, [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[EVL:%.*]] = shl i32 [[VSCALE]], 1 -; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) -; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] -; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 -; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i32 @llvm.vscale() @@ -63,11 +59,7 @@ define @exact_evl_vscale_mul_trunc( %x, @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) -; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] -; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 -; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i64 @llvm.vscale() @@ -84,11 +76,7 @@ define @exact_evl_vscale_shl_trunc( %x, @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) -; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] -; CHECK-NEXT: [[VSCALE1:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[SCALABLE_SIZE:%.*]] = mul nuw i32 [[VSCALE1]], 2 -; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[TMP2]], i32 [[SCALABLE_SIZE]]) +; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i64 @llvm.vscale() From 0a61890e40f2f81d845e3e0f57155351b04e71a0 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 26 Aug 2025 10:59:52 +0800 Subject: [PATCH 3/3] Check mul/shl is nuw --- llvm/lib/IR/IntrinsicInst.cpp | 8 ++++---- .../expand-vp-convert-evl.ll | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index 22c1479174e2c..b0376c4803f20 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -613,11 +613,11 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const { if (EC.isScalable()) { // Compare vscale patterns uint64_t VScaleFactor; - if (match(VLParam, - m_NUWTruncOrSelf(m_Mul(m_VScale(), m_ConstantInt(VScaleFactor))))) + if (match(VLParam, m_NUWTruncOrSelf( + m_NUWMul(m_VScale(), m_ConstantInt(VScaleFactor))))) return VScaleFactor >= EC.getKnownMinValue(); - if (match(VLParam, - m_NUWTruncOrSelf(m_Shl(m_VScale(), m_ConstantInt(VScaleFactor))))) + if (match(VLParam, m_NUWTruncOrSelf( + m_NUWShl(m_VScale(), m_ConstantInt(VScaleFactor))))) return 1 << VScaleFactor >= EC.getKnownMinValue(); return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale()); } diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll index 8dd8ae153ae58..6ba5a589def7f 100644 --- a/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll +++ b/llvm/test/Transforms/PreISelIntrinsicLowering/expand-vp-convert-evl.ll @@ -29,12 +29,12 @@ define @exact_evl_vscale_mul( %x, @exact_evl_vscale_mul( ; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[EVL:%.*]] = mul i32 [[VSCALE]], 2 +; CHECK-NEXT: [[EVL:%.*]] = mul nuw i32 [[VSCALE]], 2 ; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i32 @llvm.vscale() - %evl = mul i32 %vscale, 2 + %evl = mul nuw i32 %vscale, 2 %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) ret %add } @@ -43,12 +43,12 @@ define @exact_evl_vscale_shl( %x, @exact_evl_vscale_shl( ; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[EVL:%.*]] = shl i32 [[VSCALE]], 1 +; CHECK-NEXT: [[EVL:%.*]] = shl nuw i32 [[VSCALE]], 1 ; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i32 @llvm.vscale() - %evl = shl i32 %vscale, 1 + %evl = shl nuw i32 %vscale, 1 %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) ret %add } @@ -57,13 +57,13 @@ define @exact_evl_vscale_mul_trunc( %x, @exact_evl_vscale_mul_trunc( ; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[SHL:%.*]] = mul i64 [[VSCALE]], 2 +; CHECK-NEXT: [[SHL:%.*]] = mul nuw i64 [[VSCALE]], 2 ; CHECK-NEXT: [[EVL:%.*]] = trunc nuw i64 [[SHL]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i64 @llvm.vscale() - %shl = mul i64 %vscale, 2 + %shl = mul nuw i64 %vscale, 2 %evl = trunc nuw i64 %shl to i32 %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) ret %add @@ -74,13 +74,13 @@ define @exact_evl_vscale_shl_trunc( %x, @exact_evl_vscale_shl_trunc( ; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[SHL:%.*]] = shl i64 [[VSCALE]], 1 +; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[VSCALE]], 1 ; CHECK-NEXT: [[EVL:%.*]] = trunc nuw i64 [[SHL]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = call @llvm.vp.add.nxv2i64( poison, poison, [[M]], i32 [[EVL]]) ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i64 @llvm.vscale() - %shl = shl i64 %vscale, 1 + %shl = shl nuw i64 %vscale, 1 %evl = trunc nuw i64 %shl to i32 %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) ret %add @@ -90,7 +90,7 @@ define @exact_evl_vscale_shl_trunc_no_nuw( ; CHECK-LABEL: define @exact_evl_vscale_shl_trunc_no_nuw( ; CHECK-SAME: [[X:%.*]], [[Y:%.*]], [[M:%.*]]) { ; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[SHL:%.*]] = shl i64 [[VSCALE]], 1 +; CHECK-NEXT: [[SHL:%.*]] = shl nuw i64 [[VSCALE]], 1 ; CHECK-NEXT: [[EVL:%.*]] = trunc i64 [[SHL]] to i32 ; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 0, i32 [[EVL]]) ; CHECK-NEXT: [[TMP2:%.*]] = and [[TMP1]], [[M]] @@ -100,7 +100,7 @@ define @exact_evl_vscale_shl_trunc_no_nuw( ; CHECK-NEXT: ret [[ADD]] ; %vscale = call i64 @llvm.vscale() - %shl = shl i64 %vscale, 1 + %shl = shl nuw i64 %vscale, 1 %evl = trunc i64 %shl to i32 %add = call @llvm.vp.add( poison, poison, %m, i32 %evl) ret %add