Skip to content

Commit f47f21f

Browse files
krystian-andrzejewskiigcbot
krystian-andrzejewski
authored and
igcbot
committed
Unification of pattern matching for imm global offset
This change is to unify the pattern matching case for imm global offset used in different functions. Moreover, this supports extra patterns with subtractionss in experimental code paths and resolves an issue with the immediate offsets in existing one.
1 parent 93ab844 commit f47f21f

File tree

7 files changed

+299
-108
lines changed

7 files changed

+299
-108
lines changed

‎IGC/Compiler/CISACodeGen/EmitVISAPass.cpp‎

Lines changed: 85 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -737,9 +737,8 @@ bool EmitPass::runOnFunction(llvm::Function &F) {
737737
m_encoder->SetIsCodePatchCandidate(false);
738738
}
739739

740-
if (m_pCtx->platform.supportDualSimd8PS() &&
741-
m_pCtx->platform.isCoreChildOf(IGFX_XE2_HPG_CORE) &&
742-
m_currShader->GetShaderType() == ShaderType::PIXEL_SHADER ) {
740+
if (m_pCtx->platform.supportDualSimd8PS() && m_pCtx->platform.isCoreChildOf(IGFX_XE2_HPG_CORE) &&
741+
m_currShader->GetShaderType() == ShaderType::PIXEL_SHADER) {
743742
m_encoder->SetIsCodePatchCandidate(false);
744743
}
745744
} else {
@@ -3992,7 +3991,8 @@ void EmitPass::emitVideoAnalyticGRF(llvm::GenIntrinsicInst *inst, const DWORD re
39923991

39933992
static bool isVectorTypeAllowed(Value *I) {
39943993
IGCLLVM::FixedVectorType *VecType = llvm::dyn_cast<IGCLLVM::FixedVectorType>(I->getType());
3995-
if (!VecType) return false;
3994+
if (!VecType)
3995+
return false;
39963996
auto ElType = VecType->getElementType();
39973997
return ElType->isIntegerTy(32) || ElType->isFloatTy();
39983998
}
@@ -4038,8 +4038,10 @@ void EmitPass::BinaryUnary(llvm::Instruction *inst, const SSource source[2], con
40384038
break;
40394039
case Instruction::FAdd:
40404040
case Instruction::Add:
4041-
if (inst->getType()->isVectorTy()) Add(source, modifier);
4042-
else EmitSimpleAlu(inst, source, modifier);
4041+
if (inst->getType()->isVectorTy())
4042+
Add(source, modifier);
4043+
else
4044+
EmitSimpleAlu(inst, source, modifier);
40434045
break;
40444046
case Instruction::Call:
40454047
EmitAluIntrinsic(cast<CallInst>(inst), source, modifier);
@@ -4238,7 +4240,6 @@ static unsigned getVectorSize(Value *I) {
42384240
return NumElements;
42394241
}
42404242

4241-
42424243
void EmitPass::FPTrunc(const SSource sources[2], const DstModifier &modifier) {
42434244

42444245
CVariable *src[2];
@@ -4344,13 +4345,18 @@ bool EmitPass::isVectorEmissionPossible(const SSource sources[2], CVariable *src
43444345
// any non ordinary state of source modifiers
43454346
// can be a sign of emission of a specific pattern,
43464347
// emission can interfere with
4347-
if (sources[0].elementOffset != 0) return false;
4348-
if (sources[1].elementOffset != 0) return false;
4349-
if (sources[0].SIMDOffset != 0) return false;
4350-
if (sources[1].SIMDOffset != 0) return false;
4348+
if (sources[0].elementOffset != 0)
4349+
return false;
4350+
if (sources[1].elementOffset != 0)
4351+
return false;
4352+
if (sources[0].SIMDOffset != 0)
4353+
return false;
4354+
if (sources[1].SIMDOffset != 0)
4355+
return false;
43514356

43524357
bool AllowedVectorTypes = isVectorTypeAllowed(sources[0].value) && isVectorTypeAllowed(sources[1].value);
4353-
if (!AllowedVectorTypes) return false;
4358+
if (!AllowedVectorTypes)
4359+
return false;
43544360

43554361
bool DestSizeEquals = false;
43564362
// it's possible have a case where two uniform vectors are
@@ -4725,7 +4731,8 @@ void EmitPass::VectorMad(const SSource sources[3], const DstModifier &modifier)
47254731
void EmitPass::FDiv(const SSource sources[2], const DstModifier &modifier) {
47264732

47274733
CVariable *src[2];
4728-
for (int i = 0; i < 2; ++i) src[i] = GetSrcVariable(sources[i]);
4734+
for (int i = 0; i < 2; ++i)
4735+
src[i] = GetSrcVariable(sources[i]);
47294736
bool IsPossible = isVectorEmissionPossible(sources, src);
47304737

47314738
if (IGC_IS_FLAG_ENABLED(EnableVectorEmitter) && IsPossible) {
@@ -5690,8 +5697,7 @@ void EmitPass::emitSimdShuffle(llvm::Instruction *inst) {
56905697
CVariable *tempCopy = m_currShader->GetNewVariable(simdChannel, "SanitizedIndexShuffleTmp");
56915698
m_encoder->And(tempCopy, simdChannel, m_currShader->ImmToVariable(maskOfValidLanes, ISA_TYPE_UW));
56925699
simdChannelUW = m_currShader->BitCast(tempCopy, ISA_TYPE_UW);
5693-
}
5694-
else {
5700+
} else {
56955701
m_encoder->SetSrcRegion(0, 2, 1, 0);
56965702
m_encoder->SetDstRegion(2);
56975703
m_encoder->And(simdChannelUW, simdChannelUW, m_currShader->ImmToVariable(maskOfValidLanes, ISA_TYPE_UW));
@@ -8734,12 +8740,13 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst *inst) {
87348740
break;
87358741
case GenISAIntrinsic::GenISA_ldrawvector_indexed:
87368742
case GenISAIntrinsic::GenISA_ldraw_indexed:
8737-
emitLoadRawIndexed(cast<LdRawIntrinsic>(inst), cast<LdRawIntrinsic>(inst)->getOffsetValue(), nullptr, nullptr);
8743+
emitLoadRawIndexed(cast<LdRawIntrinsic>(inst), cast<LdRawIntrinsic>(inst)->getOffsetValue(), nullptr, nullptr,
8744+
false /*flipVarOffsetSign*/);
87388745
break;
87398746
case GenISAIntrinsic::GenISA_storerawvector_indexed:
87408747
case GenISAIntrinsic::GenISA_storeraw_indexed:
87418748
emitStoreRawIndexed(cast<StoreRawIntrinsic>(inst), cast<StoreRawIntrinsic>(inst)->getOffsetValue(), nullptr,
8742-
nullptr);
8749+
nullptr, false /*flipVarOffsetSign*/);
87438750
break;
87448751
case GenISAIntrinsic::GenISA_GetBufferPtr:
87458752
emitGetBufferPtr(inst);
@@ -9204,6 +9211,7 @@ void EmitPass::EmitIntrinsicMessage(llvm::IntrinsicInst *inst) {
92049211
case Intrinsic::fabs:
92059212
case Intrinsic::trap:
92069213
case Intrinsic::experimental_noalias_scope_decl:
9214+
case Intrinsic::assume:
92079215
// do nothing
92089216
break;
92099217
case Intrinsic::stacksave:
@@ -10425,8 +10433,8 @@ void EmitPass::setRovCacheCtrl(GenIntrinsicInst *inst) {
1042510433
inst->setMetadata("lsc.cache.ctrl", node);
1042610434
}
1042710435

10428-
void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale,
10429-
ConstantInt *immOffset) {
10436+
void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale, ConstantInt *immOffset,
10437+
bool flipVarOffsetSign) {
1043010438
Value *bufPtrv = inst->getResourceValue();
1043110439

1043210440
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
@@ -10437,7 +10445,7 @@ void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, Consta
1043710445
}
1043810446
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, true);
1043910447
emitLSCVectorLoad(inst, bufPtrv,
10440-
varOffset, immOffset, immScale, cacheOpts, addrSpace
10448+
varOffset, immOffset, immScale, flipVarOffsetSign, cacheOpts, addrSpace
1044110449
);
1044210450
return;
1044310451
}
@@ -10696,7 +10704,8 @@ void EmitPass::emitLoad3DInner(LdRawIntrinsic *inst, ResourceDescriptor &resourc
1069610704
}
1069710705
}
1069810706

10699-
void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, ConstantInt *immScale
10707+
void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, ConstantInt *immScale,
10708+
bool flipVarOffsetSign
1070010709
) {
1070110710
if (m_pCtx->getModuleMetaData()->isHDCFastClearShader) {
1070210711
emitFastClear(inst);
@@ -10708,13 +10717,13 @@ void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, C
1070810717
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1070910718

1071010719
emitLSCVectorLoad(inst, inst->getPointerOperand(),
10711-
offset, immOffset, immScale, cacheOpts, addrSpace
10720+
offset, immOffset, immScale, flipVarOffsetSign, cacheOpts, addrSpace
1071210721
);
1071310722
return;
1071410723
}
1071510724
IGC_ASSERT_MESSAGE(immScale ? immScale->getSExtValue() == 1 : true, "Immediate Scale not supported on non-LSC path!");
1071610725

10717-
emitVectorLoad(inst, offset, immOffset);
10726+
emitVectorLoad(inst, offset, immOffset, flipVarOffsetSign);
1071810727
}
1071910728

1072010729
void EmitPass::emitPredicatedLoad(Instruction *inst) {
@@ -10729,7 +10738,7 @@ void EmitPass::emitPredicatedLoad(Instruction *inst) {
1072910738
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1073010739

1073110740
emitLSCVectorLoad(inst, offset,
10732-
offset, nullptr, nullptr, cacheOpts, addrSpace
10741+
offset, nullptr, nullptr, false, cacheOpts, addrSpace
1073310742
);
1073410743
}
1073510744

@@ -11766,7 +11775,7 @@ void EmitPass::emitSymbolRelocation(Function &F) {
1176611775
}
1176711776

1176811777
void EmitPass::emitStoreRawIndexed(StoreRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale,
11769-
ConstantInt *immOffset) {
11778+
ConstantInt *immOffset, bool flipVarOffsetSign) {
1177011779
Value *pBufPtr = inst->getResourceValue();
1177111780
Value *pValToStore = inst->getStoreValue();
1177211781

@@ -11777,8 +11786,8 @@ void EmitPass::emitStoreRawIndexed(StoreRawIntrinsic *inst, Value *varOffset, Co
1177711786

1177811787
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1177911788
emitLSCVectorStore(pBufPtr,
11780-
varOffset, immOffset, immScale, pValToStore, inst->getParent(), cacheOpts, inst->getAlignment(),
11781-
false, addrSpace
11789+
varOffset, immOffset, immScale, flipVarOffsetSign, pValToStore, inst->getParent(), cacheOpts,
11790+
inst->getAlignment(), false, addrSpace
1178211791
);
1178311792
return;
1178411793
}
@@ -11903,20 +11912,21 @@ void EmitPass::emitStore3DInner(Value *pllValToStore, Value *pllDstPtr, Value *p
1190311912
}
1190411913
}
1190511914

11906-
void EmitPass::emitStore(StoreInst *inst, Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale
11915+
void EmitPass::emitStore(StoreInst *inst, Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
11916+
bool flipVarOffsetSign
1190711917
) {
1190811918
if (shouldGenerateLSC(inst)) {
1190911919
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1191011920

1191111921
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1191211922
emitLSCVectorStore(inst->getPointerOperand(),
11913-
varOffset, immOffset, immScale, inst->getValueOperand(), inst->getParent(), cacheOpts,
11914-
IGCLLVM::getAlignmentValue(inst), inst->getMetadata("enable.vmask"), addrSpace
11923+
varOffset, immOffset, immScale, flipVarOffsetSign, inst->getValueOperand(), inst->getParent(),
11924+
cacheOpts, IGCLLVM::getAlignmentValue(inst), inst->getMetadata("enable.vmask"), addrSpace
1191511925
);
1191611926
return;
1191711927
}
1191811928
IGC_ASSERT_MESSAGE(immScale ? immScale->getSExtValue() == 1 : true, "Immediate Scale not supported on non-LSC path!");
11919-
emitVectorStore(inst, varOffset, immOffset);
11929+
emitVectorStore(inst, varOffset, immOffset, flipVarOffsetSign);
1192011930
}
1192111931

1192211932
void EmitPass::emitPredicatedStore(Instruction *inst) {
@@ -11929,7 +11939,8 @@ void EmitPass::emitPredicatedStore(Instruction *inst) {
1192911939
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1193011940

1193111941
emitLSCVectorStore(inst->getOperand(0),
11932-
inst->getOperand(0), nullptr, nullptr, inst->getOperand(1), inst->getParent(), cacheOpts,
11942+
inst->getOperand(0), nullptr, nullptr, false /*flipVarOffsetSign*/, inst->getOperand(1),
11943+
inst->getParent(), cacheOpts,
1193311944
cast<ConstantInt>(inst->getOperand(2))->getZExtValue(), // alignment
1193411945
inst->getMetadata("enable.vmask"), addrSpace
1193511946
,
@@ -14609,7 +14620,7 @@ CVariable *EmitPass::UnpackOrBroadcastIfUniform(CVariable *pVar) {
1460914620
}
1461014621

1461114622
void EmitPass::emitAtomicRaw(llvm::GenIntrinsicInst *pInst, Value *dstAddr, ConstantInt *immOffset,
14612-
ConstantInt *immScale
14623+
ConstantInt *immScale, bool flipVarOffsetSign
1461314624
) {
1461414625
ForceDMask();
1461514626
// Currently, Dword Atomics can be called by matching 2 intrinsics. One is the
@@ -14683,6 +14694,13 @@ void EmitPass::emitAtomicRaw(llvm::GenIntrinsicInst *pInst, Value *dstAddr, Cons
1468314694

1468414695
// Dst address in bytes.
1468514696
CVariable *pDstAddr = GetSymbol(dstAddr);
14697+
if (flipVarOffsetSign) {
14698+
CVariable *eNegBase = m_currShader->GetNewVariable(pDstAddr, CName(pDstAddr->getName(), "Neg"));
14699+
m_encoder->SetSrcModifier(0, EMOD_NEG);
14700+
m_encoder->Copy(eNegBase, pDstAddr); // MOV with -src modifier
14701+
m_encoder->Push();
14702+
pDstAddr = eNegBase;
14703+
}
1468614704

1468714705

1468814706
PointerType *PtrTy = dyn_cast<PointerType>(dstAddr->getType());
@@ -16604,7 +16622,7 @@ void EmitPass::emitFastClearSend(llvm::Instruction *pInst) {
1660416622
m_encoder->Push();
1660516623
}
1660616624

16607-
void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset) {
16625+
void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, bool flipVarOffsetSign) {
1660816626
int immOffsetInt = 0;
1660916627
if (immOffset)
1661016628
immOffsetInt = static_cast<int>(immOffset->getSExtValue());
@@ -16618,6 +16636,13 @@ void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOff
1661816636
// eOffset is in bytes
1661916637
// offset corresponds to Int2Ptr operand obtained during pattern matching
1662016638
CVariable *eOffset = GetSymbol(immOffset ? offset : Ptr);
16639+
if (flipVarOffsetSign) {
16640+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
16641+
m_encoder->SetSrcModifier(0, EMOD_NEG);
16642+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
16643+
m_encoder->Push();
16644+
eOffset = eNegOffset;
16645+
}
1662116646
if (useA32) {
1662216647
eOffset = TruncatePointer(eOffset);
1662316648
}
@@ -17056,7 +17081,7 @@ void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOff
1705617081
}
1705717082
}
1705817083

17059-
void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immOffset) {
17084+
void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immOffset, bool flipVarOffsetSign) {
1706017085
int immOffsetInt = 0;
1706117086
if (immOffset)
1706217087
immOffsetInt = static_cast<int>(immOffset->getSExtValue());
@@ -17073,6 +17098,13 @@ void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immO
1707317098
// eOffset is in bytes
1707417099
// offset corresponds to Int2Ptr operand obtained during pattern matching
1707517100
CVariable *eOffset = GetSymbol(immOffset ? offset : Ptr);
17101+
if (flipVarOffsetSign) {
17102+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
17103+
m_encoder->SetSrcModifier(0, EMOD_NEG);
17104+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
17105+
m_encoder->Push();
17106+
eOffset = eNegOffset;
17107+
}
1707617108
bool useA32 = !isA64Ptr(ptrType, m_currShader->GetContext());
1707717109
if (useA32) {
1707817110
eOffset = TruncatePointer(eOffset);
@@ -17747,7 +17779,7 @@ void EmitPass::emitLSCVectorLoad_uniform(LSC_CACHE_OPTS CacheOpts, bool UseA32,
1774717779

1774817780
void EmitPass::emitLSCVectorLoad(Instruction *inst, Value *Ptr,
1774917781
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
17750-
LSC_CACHE_OPTS cacheOpts, LSC_DOC_ADDR_SPACE addrSpace
17782+
bool flipVarOffsetSign, LSC_CACHE_OPTS cacheOpts, LSC_DOC_ADDR_SPACE addrSpace
1775117783
) {
1775217784

1775317785
bool predicatedLoad = false;
@@ -17777,6 +17809,13 @@ void EmitPass::emitLSCVectorLoad(Instruction *inst, Value *Ptr,
1777717809
// eOffset is in bytes
1777817810
// offset corresponds to Int2Ptr operand obtained during pattern matching
1777917811
CVariable *eOffset = GetSymbol(varOffset);
17812+
if (flipVarOffsetSign) {
17813+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
17814+
m_encoder->SetSrcModifier(0, EMOD_NEG);
17815+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
17816+
m_encoder->Push();
17817+
eOffset = eNegOffset;
17818+
}
1778017819
if (useA32) {
1778117820
eOffset = TruncatePointer(eOffset);
1778217821
}
@@ -18128,9 +18167,9 @@ void EmitPass::emitLSCVectorStore_uniform(LSC_CACHE_OPTS CacheOpts, bool UseA32,
1812818167
}
1812918168

1813018169
void EmitPass::emitLSCVectorStore(Value *Ptr,
18131-
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale, Value *storedVal,
18132-
BasicBlock *BB, LSC_CACHE_OPTS cacheOpts, alignment_t align, bool dontForceDmask,
18133-
LSC_DOC_ADDR_SPACE addrSpace
18170+
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
18171+
bool flipVarOffsetSign, Value *storedVal, BasicBlock *BB, LSC_CACHE_OPTS cacheOpts,
18172+
alignment_t align, bool dontForceDmask, LSC_DOC_ADDR_SPACE addrSpace
1813418173
,
1813518174
Value *predicate) {
1813618175

@@ -18151,6 +18190,13 @@ void EmitPass::emitLSCVectorStore(Value *Ptr,
1815118190

1815218191
// offset corresponds to Int2Ptr operand obtained during pattern matching
1815318192
CVariable *eOffset = GetSymbol(varOffset);
18193+
if (flipVarOffsetSign) {
18194+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
18195+
m_encoder->SetSrcModifier(0, EMOD_NEG);
18196+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
18197+
m_encoder->Push();
18198+
eOffset = eNegOffset;
18199+
}
1815418200
bool useA32 = !isA64Ptr(ptrType, m_currShader->GetContext());
1815518201
LSC_ADDR_SIZE addrSize = useA32 ? LSC_ADDR_SIZE_32b : LSC_ADDR_SIZE_64b;
1815618202
if (useA32) {

0 commit comments

Comments
 (0)