Refactor logic around movi.

Extract it to new common function, use it to prevent hoisting in few edge cases.
Add experimental support for extended cases.

Author: bgajdaINTC

IGC/Compiler/CISACodeGen/CISABuilder.cpp

@@ -4842,6 +4842,11 @@ namespace IGC
             SaveOption(vISA_ALTMode, true);
         }
 
+        if (IGC_GET_FLAG_VALUE(EnableEmitMoreMoviCases))
+        {
+            SaveOption(vISA_emitMoreMoviCases, true);
+        }
+
         //
         // Setting number of GRF and threads per EU is restricted to OCL only
         // Number of threads can be set by:

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -5829,13 +5829,31 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
         // bit set. Inactive lanes contain garbage data and may cause an
         // out-of-bounds register access.
         bool laneIdCanBeOOB = !m_currShader->m_DriverInfo->needsRegisterAccessBoundsChecks();
-        if (!m_currShader->m_Platform->supportsOutOfBoundsGrfAccess() &&
-            !channelUniform &&
-            (m_encoder->IsSubSpanDestination() || laneIdCanBeOOB))
+
+        bool defaultConditions =
+            (!m_currShader->m_Platform->supportsOutOfBoundsGrfAccess() &&
+             !channelUniform &&
+             (m_encoder->IsSubSpanDestination() || laneIdCanBeOOB));
+
+        // Enabling movi requires that first lane even inactive will be within bounds of register we want.
+        // It also is limited to accessing single GRF.
+        // For uniform channel which will be simd1 there's probably no gain in movi.
+        bool isSingleGrf = data->GetSize() <= (unsigned)getGRFSize();
+        bool platformMoviTypeCheck = m_currShader->m_Platform->allowsMoviForType(data->GetType());
+        bool forcePreventOOB = isSingleGrf && platformMoviTypeCheck && !channelUniform;
+
+        if (defaultConditions || forcePreventOOB)
         {
             uint maskOfValidLanes = numLanes(m_currShader->m_State.m_dispatchSize) - 1;
             m_encoder->SetSrcRegion(0, 2, 1, 0);
             m_encoder->SetDstRegion(2);
+
+            // To support conversion to movi we need to make all calculations (shl, addr_add) of address NoMask.
+            // to avoid random data from previous execution in divergent CF.
+            if (forcePreventOOB)
+            {
+                m_encoder->SetNoMask();
+            }
             m_encoder->And(simdChannelUW, simdChannelUW,
                 m_currShader->ImmToVariable(maskOfValidLanes, ISA_TYPE_UW));
             m_encoder->Push();
@@ -5851,6 +5869,10 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
         {
             m_encoder->SetSrcRegion(0, 16, 8, 2);
         }
+        if (forcePreventOOB)
+        {
+            m_encoder->SetNoMask();
+        }
         m_encoder->Shl(pSrcElm, simdChannelUW,
             m_currShader->ImmToVariable(shtAmt, ISA_TYPE_UW));
         m_encoder->Push();
@@ -5889,6 +5911,10 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
                 else // !channelUniform
                 {
                     m_encoder->SetSimdSize(SIMDMode::SIMD16);
+                    if (forcePreventOOB)
+                    {
+                        m_encoder->SetNoMask();
+                    }
 
                     m_encoder->AddrAdd(pDstArrElm, src, pSrcElm);
                     m_encoder->Push();
@@ -5924,6 +5950,10 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
                     m_encoder->SetMask(EMASK_H2);
                     m_encoder->SetSrcSubReg(0, 16);
                     m_encoder->SetSrcSubReg(1, 16);
+                    if (forcePreventOOB)
+                    {
+                        m_encoder->SetNoMask();
+                    }
                     m_encoder->AddrAdd(pDstArrElm, src, pSrcElm);
                     m_encoder->Push();
 
@@ -5994,6 +6024,10 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
             {
                 // also calculate the second half of address
                 m_encoder->SetSrcRegion(0, 16, 8, 2);
+                if (forcePreventOOB)
+                {
+                    m_encoder->SetNoMask();
+                }
                 m_encoder->Shl(pSrcElm, simdChannelUW,
                     m_currShader->ImmToVariable(shtAmt, ISA_TYPE_UW));
                 m_encoder->Push();
@@ -6016,6 +6050,11 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
             true,
             m_destination->getName());
 
+        if (forcePreventOOB)
+        {
+            m_encoder->SetNoMask();
+        }
+
         m_encoder->AddrAdd(pDstArrElm, src, pSrcElm);
         m_encoder->Push();
 
@@ -6025,6 +6064,10 @@ void EmitPass::emitSimdShuffle(llvm::Instruction* inst)
         if (isSimd32)
         {
             m_encoder->SetSecondHalf(true);
+            if (forcePreventOOB)
+            {
+                m_encoder->SetNoMask();
+            }
             m_encoder->AddrAdd(pDstArrElm, src, pSrcElm);
             m_encoder->Push();
             m_encoder->Copy(m_destination, pDstArrElm);

IGC/Compiler/CISACodeGen/Platform.hpp

@@ -1910,5 +1910,9 @@ bool allowProceedBasedApproachForRayQueryDynamicRayManagementMechanism() const
     return IGC_IS_FLAG_DISABLED(DisableProceedBasedApproachForRayQueryDynamicRayManagementMechanism);
 }
 
+bool allowsMoviForType(VISA_Type type) const {
+  return IGC_GET_FLAG_VALUE(EnableEmitMoreMoviCases) && (type == ISA_TYPE_UD || type == ISA_TYPE_D);
+}
+
 };
 }//namespace IGC

IGC/Compiler/CustomSafeOptPass.cpp

@@ -1951,7 +1951,7 @@ void CustomSafeOptPass::visitTruncInst(TruncInst& I)
     To:
     %335 = call i16 @llvm.genx.GenISA.WaveShuffleIndex.i16(i16 %orig, i32 %333, i32 0)
     */
-    if( I.getSrcTy()->isIntegerTy( 32 ) && I.getDestTy()->isIntegerTy( 16 ) )
+    if( I.getSrcTy()->isIntegerTy( 32 ) && I.getDestTy()->isIntegerTy( 16 ) && IGC_IS_FLAG_DISABLED(EnableEmitMoreMoviCases))
     {
         // We know all variants of shuffle from zext are safe to demote. (unlike WaveAll which might not be)
         if( auto* genIntr = dyn_cast<GenIntrinsicInst>( I.getOperand( 0 ) ); genIntr && isSubGroupShuffleVariant( genIntr ) && genIntr->hasOneUse() )

IGC/common/igc_flags.h

@@ -684,6 +684,7 @@ DECLARE_IGC_REGKEY_ENUM(LscStoreCacheControlOverride, 0, "Overrides cache-contro
 DECLARE_IGC_REGKEY_ENUM(TgmLoadCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC tgm loads.", LSC_CACHE_CTRL_OPTIONS, true)
 DECLARE_IGC_REGKEY_ENUM(TgmStoreCacheControlOverride, 0, "Overrides cache-control options for non-intrinsic LSC tgm stores.", LSC_CACHE_CTRL_OPTIONS, true)
 DECLARE_IGC_REGKEY(bool, LscForceSpillNonStackcall, false, "Non-stack call kernels that spill will use LSC on DG2+", true)
+DECLARE_IGC_REGKEY(bool, EnableEmitMoreMoviCases, false, "Enables emitting movi for waveShuffle cases using And to keep index within single register.", true)
 DECLARE_IGC_REGKEY_ENUM(ForceRegisterAccessBoundsChecks, -1, \
     "Controls the behavior of RegisterAccessBoundsChecks, the pass that adds runtime bounds-checks for vector-indexing instructions." \
     "-1 - default behavior, the pass is enabled based on the API type or AILs" \

visa/BuildIR.h

@@ -2423,9 +2423,12 @@ class IR_Builder {
 
   void materializeGlobalImm(G4_BB *entryBB); // why is in FlowGraph.cpp???
 
+  bool canPromoteToMovi(G4_INST *);
+
 #include "HWCaps.inc"
 
 private:
+  G4_INST* getSingleDefInst(G4_INST* UI, Gen4_Operand_Number OpndNum) const;
   G4_SrcRegRegion *createBindlessExDesc(uint32_t exdesc);
   uint32_t createSamplerMsgDesc(VISASampler3DSubOpCode samplerOp,
                                 bool isNativeSIMDSize, bool isFP16Return,