drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /* To compile this assembly code:
  24  * PROJECT=greenland ./sp3 cwsr_trap_handler_gfx9.asm -hex tmp.hex
  25  */
  26
  27 var ACK_SQC_STORE                   =   1                   //workaround for suspected SQC store bug causing incorrect stores under concurrency
  28 var SAVE_AFTER_XNACK_ERROR          =   1                   //workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger
  29 var SINGLE_STEP_MISSED_WORKAROUND   =   1                   //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
  30
  31 /**************************************************************************/
  32 /*                      variables                                         */
  33 /**************************************************************************/
  34 var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
  35 var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
  36 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
  37 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
  38 var SQ_WAVE_STATUS_HALT_MASK       = 0x2000
  39 var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT   = 0
  40 var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE    = 1
  41 var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT  = 3
  42 var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE   = 29
  43 var SQ_WAVE_STATUS_ALLOW_REPLAY_MASK    = 0x400000
  44
  45 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT    = 12
  46 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE     = 9
  47 var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT   = 8
  48 var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE    = 6
  49 var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT   = 24
  50 var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE    = 3                     //FIXME  sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
  51
  52 var SQ_WAVE_TRAPSTS_SAVECTX_MASK    =   0x400
  53 var SQ_WAVE_TRAPSTS_EXCE_MASK       =   0x1FF                   // Exception mask
  54 var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT   =   10
  55 var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK   =   0x100
  56 var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT  =   8
  57 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK    =   0x3FF
  58 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT   =   0x0
  59 var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE    =   10
  60 var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK   =   0xFFFFF800
  61 var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT  =   11
  62 var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE   =   21
  63 var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK   =   0x800
  64 var SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK    =   0x10000000
  65
  66 var SQ_WAVE_IB_STS_RCNT_SHIFT           =   16                  //FIXME
  67 var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT   =   15                  //FIXME
  68 var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK       = 0x1F8000
  69 var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG   = 0x00007FFF    //FIXME
  70
  71 var SQ_WAVE_MODE_DEBUG_EN_MASK          =   0x800
  72
  73 var SQ_BUF_RSRC_WORD1_ATC_SHIFT     =   24
  74 var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT   =   27
  75
  76 var TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT =   26                  // bits [31:26] unused by SPI debug data
  77 var TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK  =   0xFC000000
  78
  79 /*      Save        */
  80 var S_SAVE_BUF_RSRC_WORD1_STRIDE        =   0x00040000          //stride is 4 bytes
  81 var S_SAVE_BUF_RSRC_WORD3_MISC          =   0x00807FAC          //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
  82
  83 var S_SAVE_SPI_INIT_ATC_MASK            =   0x08000000          //bit[27]: ATC bit
  84 var S_SAVE_SPI_INIT_ATC_SHIFT           =   27
  85 var S_SAVE_SPI_INIT_MTYPE_MASK          =   0x70000000          //bit[30:28]: Mtype
  86 var S_SAVE_SPI_INIT_MTYPE_SHIFT         =   28
  87 var S_SAVE_SPI_INIT_FIRST_WAVE_MASK     =   0x04000000          //bit[26]: FirstWaveInTG
  88 var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT    =   26
  89
  90 var S_SAVE_PC_HI_RCNT_SHIFT             =   27                  //FIXME  check with Brian to ensure all fields other than PC[47:0] can be used
  91 var S_SAVE_PC_HI_RCNT_MASK              =   0xF8000000          //FIXME
  92 var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT     =   26                  //FIXME
  93 var S_SAVE_PC_HI_FIRST_REPLAY_MASK      =   0x04000000          //FIXME
  94
  95 var s_save_spi_init_lo              =   exec_lo
  96 var s_save_spi_init_hi              =   exec_hi
  97
  98 var s_save_pc_lo            =   ttmp0           //{TTMP1, TTMP0} = {3'h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
  99 var s_save_pc_hi            =   ttmp1
 100 var s_save_exec_lo          =   ttmp2
 101 var s_save_exec_hi          =   ttmp3
 102 var s_save_tmp              =   ttmp14
 103 var s_save_trapsts          =   ttmp15          //not really used until the end of the SAVE routine
 104 var s_save_xnack_mask_lo    =   ttmp6
 105 var s_save_xnack_mask_hi    =   ttmp7
 106 var s_save_buf_rsrc0        =   ttmp8
 107 var s_save_buf_rsrc1        =   ttmp9
 108 var s_save_buf_rsrc2        =   ttmp10
 109 var s_save_buf_rsrc3        =   ttmp11
 110 var s_save_status           =   ttmp12
 111 var s_save_mem_offset       =   ttmp4
 112 var s_save_alloc_size       =   s_save_trapsts          //conflict
 113 var s_save_m0               =   ttmp5
 114 var s_save_ttmps_lo         =   s_save_tmp              //no conflict
 115 var s_save_ttmps_hi         =   s_save_trapsts          //no conflict
 116
 117 /*      Restore     */
 118 var S_RESTORE_BUF_RSRC_WORD1_STRIDE         =   S_SAVE_BUF_RSRC_WORD1_STRIDE
 119 var S_RESTORE_BUF_RSRC_WORD3_MISC           =   S_SAVE_BUF_RSRC_WORD3_MISC
 120
 121 var S_RESTORE_SPI_INIT_ATC_MASK             =   0x08000000          //bit[27]: ATC bit
 122 var S_RESTORE_SPI_INIT_ATC_SHIFT            =   27
 123 var S_RESTORE_SPI_INIT_MTYPE_MASK           =   0x70000000          //bit[30:28]: Mtype
 124 var S_RESTORE_SPI_INIT_MTYPE_SHIFT          =   28
 125 var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK      =   0x04000000          //bit[26]: FirstWaveInTG
 126 var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT     =   26
 127
 128 var S_RESTORE_PC_HI_RCNT_SHIFT              =   S_SAVE_PC_HI_RCNT_SHIFT
 129 var S_RESTORE_PC_HI_RCNT_MASK               =   S_SAVE_PC_HI_RCNT_MASK
 130 var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT      =   S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
 131 var S_RESTORE_PC_HI_FIRST_REPLAY_MASK       =   S_SAVE_PC_HI_FIRST_REPLAY_MASK
 132
 133 var s_restore_spi_init_lo                   =   exec_lo
 134 var s_restore_spi_init_hi                   =   exec_hi
 135
 136 var s_restore_mem_offset        =   ttmp12
 137 var s_restore_accvgpr_offset    =   ttmp13
 138 var s_restore_alloc_size        =   ttmp3
 139 var s_restore_tmp               =   ttmp2
 140 var s_restore_mem_offset_save   =   s_restore_tmp       //no conflict
 141 var s_restore_accvgpr_offset_save = ttmp7
 142
 143 var s_restore_m0            =   s_restore_alloc_size    //no conflict
 144
 145 var s_restore_mode          =   s_restore_accvgpr_offset_save
 146
 147 var s_restore_pc_lo         =   ttmp0
 148 var s_restore_pc_hi         =   ttmp1
 149 var s_restore_exec_lo       =   ttmp4
 150 var s_restore_exec_hi       =   ttmp5
 151 var s_restore_status        =   ttmp14
 152 var s_restore_trapsts       =   ttmp15
 153 var s_restore_xnack_mask_lo =   xnack_mask_lo
 154 var s_restore_xnack_mask_hi =   xnack_mask_hi
 155 var s_restore_buf_rsrc0     =   ttmp8
 156 var s_restore_buf_rsrc1     =   ttmp9
 157 var s_restore_buf_rsrc2     =   ttmp10
 158 var s_restore_buf_rsrc3     =   ttmp11
 159 var s_restore_ttmps_lo      =   s_restore_tmp           //no conflict
 160 var s_restore_ttmps_hi      =   s_restore_alloc_size    //no conflict
 161
 162 /**************************************************************************/
 163 /*                      trap handler entry points                         */
 164 /**************************************************************************/
 165 /* Shader Main*/
 166
 167 shader main
 168   asic(DEFAULT)
 169   type(CS)
 170
 171
 172         s_branch L_SKIP_RESTORE                                     //NOT restore. might be a regular trap or save
 173
 174 L_JUMP_TO_RESTORE:
 175     s_branch L_RESTORE                                              //restore
 176
 177 L_SKIP_RESTORE:
 178
 179     s_getreg_b32    s_save_status, hwreg(HW_REG_STATUS)                             //save STATUS since we will change SCC
 180     s_andn2_b32     s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK      //check whether this is for save
 181
 182 if SINGLE_STEP_MISSED_WORKAROUND
 183     // No single step exceptions if MODE.DEBUG_EN=0.
 184     s_getreg_b32    ttmp2, hwreg(HW_REG_MODE)
 185     s_and_b32       ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK
 186     s_cbranch_scc0  L_NO_SINGLE_STEP_WORKAROUND
 187
 188     // Second-level trap already handled exception if STATUS.HALT=1.
 189     s_and_b32       ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 190
 191     // Prioritize single step exception over context save.
 192     // Second-level trap will halt wave and RFE, re-entering for SAVECTX.
 193     s_cbranch_scc0  L_FETCH_2ND_TRAP
 194
 195 L_NO_SINGLE_STEP_WORKAROUND:
 196 end
 197
 198     s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 199     s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK    //check whether this is for save
 200     s_cbranch_scc1  L_SAVE                                      //this is the operation for save
 201
 202     // *********    Handle non-CWSR traps       *******************
 203
 204     // Illegal instruction is a non-maskable exception which blocks context save.
 205     // Halt the wavefront and return from the trap.
 206     s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK
 207     s_cbranch_scc1  L_HALT_WAVE
 208
 209     // If STATUS.MEM_VIOL is asserted then we cannot fetch from the TMA.
 210     // Instead, halt the wavefront and return from the trap.
 211     s_and_b32       ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
 212     s_cbranch_scc0  L_FETCH_2ND_TRAP
 213
 214 L_HALT_WAVE:
 215     // If STATUS.HALT is set then this fault must come from SQC instruction fetch.
 216     // We cannot prevent further faults. Spin wait until context saved.
 217     s_and_b32       ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 218     s_cbranch_scc0  L_NOT_ALREADY_HALTED
 219
 220 L_WAIT_CTX_SAVE:
 221     s_sleep         0x10
 222     s_getreg_b32    ttmp2, hwreg(HW_REG_TRAPSTS)
 223     s_and_b32       ttmp2, ttmp2, SQ_WAVE_TRAPSTS_SAVECTX_MASK
 224     s_cbranch_scc0  L_WAIT_CTX_SAVE
 225
 226 L_NOT_ALREADY_HALTED:
 227     s_or_b32        s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 228
 229     // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
 230     // Rewind the PC to prevent this from occurring. The debugger compensates for this.
 231     s_sub_u32       ttmp0, ttmp0, 0x8
 232     s_subb_u32      ttmp1, ttmp1, 0x0
 233
 234 L_FETCH_2ND_TRAP:
 235     // Preserve and clear scalar XNACK state before issuing scalar reads.
 236     // Save IB_STS.FIRST_REPLAY[15] and IB_STS.RCNT[20:16] into unused space ttmp11[31:26].
 237     s_getreg_b32    ttmp2, hwreg(HW_REG_IB_STS)
 238     s_and_b32       ttmp3, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
 239     s_lshl_b32      ttmp3, ttmp3, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
 240     s_andn2_b32     ttmp11, ttmp11, TTMP11_SAVE_RCNT_FIRST_REPLAY_MASK
 241     s_or_b32        ttmp11, ttmp11, ttmp3
 242
 243     s_andn2_b32     ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
 244     s_setreg_b32    hwreg(HW_REG_IB_STS), ttmp2
 245
 246     // Read second-level TBA/TMA from first-level TMA and jump if available.
 247     // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
 248     // ttmp12 holds SQ_WAVE_STATUS
 249     s_getreg_b32    ttmp14, hwreg(HW_REG_SQ_SHADER_TMA_LO)
 250     s_getreg_b32    ttmp15, hwreg(HW_REG_SQ_SHADER_TMA_HI)
 251     s_lshl_b64      [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
 252     s_load_dwordx2  [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA
 253     s_waitcnt       lgkmcnt(0)
 254     s_load_dwordx2  [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1 // second-level TMA
 255     s_waitcnt       lgkmcnt(0)
 256     s_and_b64       [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]
 257     s_cbranch_scc0  L_NO_NEXT_TRAP // second-level trap handler not been set
 258     s_setpc_b64     [ttmp2, ttmp3] // jump to second-level trap handler
 259
 260 L_NO_NEXT_TRAP:
 261     s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 262     s_and_b32       s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
 263     s_cbranch_scc1  L_EXCP_CASE   // Exception, jump back to the shader program directly.
 264     s_add_u32       ttmp0, ttmp0, 4   // S_TRAP case, add 4 to ttmp0
 265     s_addc_u32  ttmp1, ttmp1, 0
 266 L_EXCP_CASE:
 267     s_and_b32   ttmp1, ttmp1, 0xFFFF
 268
 269     // Restore SQ_WAVE_IB_STS.
 270     s_lshr_b32      ttmp2, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
 271     s_and_b32       ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
 272     s_setreg_b32    hwreg(HW_REG_IB_STS), ttmp2
 273
 274     // Restore SQ_WAVE_STATUS.
 275     s_and_b64       exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
 276     s_and_b64       vcc, vcc, vcc    // Restore STATUS.VCCZ, not writable by s_setreg_b32
 277     set_status_without_spi_prio(s_save_status, ttmp2)
 278
 279     s_rfe_b64       [ttmp0, ttmp1]
 280
 281     // *********        End handling of non-CWSR traps   *******************
 282
 283 /**************************************************************************/
 284 /*                      save routine                                      */
 285 /**************************************************************************/
 286
 287 L_SAVE:
 288     s_and_b32       s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
 289
 290     s_mov_b32       s_save_tmp, 0                                                           //clear saveCtx bit
 291     s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp     //clear saveCtx bit
 292
 293     s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE)                   //save RCNT
 294     s_lshl_b32      s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
 295     s_or_b32        s_save_pc_hi, s_save_pc_hi, s_save_tmp
 296     s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE)   //save FIRST_REPLAY
 297     s_lshl_b32      s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
 298     s_or_b32        s_save_pc_hi, s_save_pc_hi, s_save_tmp
 299     s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS)                                        //clear RCNT and FIRST_REPLAY in IB_STS
 300     s_and_b32       s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
 301
 302     s_setreg_b32    hwreg(HW_REG_IB_STS), s_save_tmp
 303
 304     /*      inform SPI the readiness and wait for SPI's go signal */
 305     s_mov_b32       s_save_exec_lo, exec_lo                                                 //save EXEC and use EXEC for the go signal from SPI
 306     s_mov_b32       s_save_exec_hi, exec_hi
 307     s_mov_b64       exec,   0x0                                                             //clear EXEC to get ready to receive
 308
 309         s_sendmsg   sendmsg(MSG_SAVEWAVE)  //send SPI a message and wait for SPI's write to EXEC
 310
 311     // Set SPI_PRIO=2 to avoid starving instruction fetch in the waves we're waiting for.
 312     s_or_b32 s_save_tmp, s_save_status, (2 << SQ_WAVE_STATUS_SPI_PRIO_SHIFT)
 313     s_setreg_b32 hwreg(HW_REG_STATUS), s_save_tmp
 314
 315   L_SLEEP:
 316     s_sleep 0x2                // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
 317
 318         s_cbranch_execz L_SLEEP
 319
 320     // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
 321     // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
 322     get_vgpr_size_bytes(s_save_ttmps_lo)
 323     get_sgpr_size_bytes(s_save_ttmps_hi)
 324     s_add_u32       s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
 325     s_add_u32       s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo
 326     s_addc_u32      s_save_ttmps_hi, s_save_spi_init_hi, 0x0
 327     s_and_b32       s_save_ttmps_hi, s_save_ttmps_hi, 0xFFFF
 328     s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1
 329     ack_sqc_store_workaround()
 330     s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1
 331     ack_sqc_store_workaround()
 332     s_store_dword   ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1
 333     ack_sqc_store_workaround()
 334
 335     /*      setup Resource Contants    */
 336     s_mov_b32       s_save_buf_rsrc0,   s_save_spi_init_lo                                                      //base_addr_lo
 337     s_and_b32       s_save_buf_rsrc1,   s_save_spi_init_hi, 0x0000FFFF                                          //base_addr_hi
 338     s_or_b32        s_save_buf_rsrc1,   s_save_buf_rsrc1,  S_SAVE_BUF_RSRC_WORD1_STRIDE
 339     s_mov_b32       s_save_buf_rsrc2,   0                                                                       //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
 340     s_mov_b32       s_save_buf_rsrc3,   S_SAVE_BUF_RSRC_WORD3_MISC
 341     s_and_b32       s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
 342     s_lshr_b32      s_save_tmp,         s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)         //get ATC bit into position
 343     s_or_b32        s_save_buf_rsrc3,   s_save_buf_rsrc3,  s_save_tmp                                           //or ATC
 344     s_and_b32       s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
 345     s_lshr_b32      s_save_tmp,         s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)     //get MTYPE bits into position
 346     s_or_b32        s_save_buf_rsrc3,   s_save_buf_rsrc3,  s_save_tmp                                           //or MTYPE
 347
 348     //FIXME  right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi  (might need to save them before using them?)
 349     s_mov_b32       s_save_m0,          m0                                                                  //save M0
 350
 351     /*      global mem offset           */
 352     s_mov_b32       s_save_mem_offset,  0x0                                                                     //mem offset initial value = 0
 353
 354
 355
 356
 357     /*      save HW registers   */
 358     //////////////////////////////
 359
 360   L_SAVE_HWREG:
 361         // HWREG SR memory offset : size(VGPR)+size(SGPR)
 362        get_vgpr_size_bytes(s_save_mem_offset)
 363        get_sgpr_size_bytes(s_save_tmp)
 364        s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
 365
 366
 367     s_mov_b32       s_save_buf_rsrc2, 0x4                               //NUM_RECORDS   in bytes
 368         s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
 369
 370
 371     write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)                  //M0
 372     write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)                   //PC
 373     write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
 374     write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)             //EXEC
 375     write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
 376     write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)              //STATUS
 377
 378     //s_save_trapsts conflicts with s_save_alloc_size
 379     s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 380     write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset)             //TRAPSTS
 381
 382     write_hwreg_to_mem(xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset)          //XNACK_MASK_LO
 383     write_hwreg_to_mem(xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset)          //XNACK_MASK_HI
 384
 385     //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
 386     s_getreg_b32    s_save_m0, hwreg(HW_REG_MODE)                                                   //MODE
 387     write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
 388
 389
 390
 391     /*      the first wave in the threadgroup    */
 392     s_and_b32       s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK     // extract fisrt wave bit
 393     s_mov_b32        s_save_exec_hi, 0x0
 394     s_or_b32         s_save_exec_hi, s_save_tmp, s_save_exec_hi                          // save first wave bit to s_save_exec_hi.bits[26]
 395
 396
 397     /*          save SGPRs      */
 398         // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
 399     //////////////////////////////
 400
 401     // SGPR SR memory offset : size(VGPR)
 402     get_vgpr_size_bytes(s_save_mem_offset)
 403     // TODO, change RSRC word to rearrange memory layout for SGPRS
 404
 405     s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)               //spgr_size
 406     s_add_u32       s_save_alloc_size, s_save_alloc_size, 1
 407     s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 4                         //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
 408
 409         s_lshl_b32      s_save_buf_rsrc2,   s_save_alloc_size, 2                    //NUM_RECORDS in bytes
 410
 411         s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
 412
 413
 414     // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
 415     //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
 416     s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
 417     s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
 418     s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
 419
 420     s_mov_b32       m0, 0x0                         //SGPR initial index value =0
 421     s_nop           0x0                             //Manually inserted wait states
 422   L_SAVE_SGPR_LOOP:
 423     // SGPR is allocated in 16 SGPR granularity
 424     s_movrels_b64   s0, s0     //s0 = s[0+m0], s1 = s[1+m0]
 425     s_movrels_b64   s2, s2     //s2 = s[2+m0], s3 = s[3+m0]
 426     s_movrels_b64   s4, s4     //s4 = s[4+m0], s5 = s[5+m0]
 427     s_movrels_b64   s6, s6     //s6 = s[6+m0], s7 = s[7+m0]
 428     s_movrels_b64   s8, s8     //s8 = s[8+m0], s9 = s[9+m0]
 429     s_movrels_b64   s10, s10   //s10 = s[10+m0], s11 = s[11+m0]
 430     s_movrels_b64   s12, s12   //s12 = s[12+m0], s13 = s[13+m0]
 431     s_movrels_b64   s14, s14   //s14 = s[14+m0], s15 = s[15+m0]
 432
 433     write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
 434     s_add_u32       m0, m0, 16                                                      //next sgpr index
 435     s_cmp_lt_u32    m0, s_save_alloc_size                                           //scc = (m0 < s_save_alloc_size) ? 1 : 0
 436     s_cbranch_scc1  L_SAVE_SGPR_LOOP                                    //SGPR save is complete?
 437     // restore s_save_buf_rsrc0,1
 438     //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
 439     s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
 440
 441
 442
 443
 444     /*          save first 4 VGPR, then LDS save could use   */
 445         // each wave will alloc 4 vgprs at least...
 446     /////////////////////////////////////////////////////////////////////////////////////
 447
 448     s_mov_b32       s_save_mem_offset, 0
 449     s_mov_b32       exec_lo, 0xFFFFFFFF                                             //need every thread from now on
 450     s_mov_b32       exec_hi, 0xFFFFFFFF
 451     s_mov_b32       xnack_mask_lo, 0x0
 452     s_mov_b32       xnack_mask_hi, 0x0
 453
 454         s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
 455
 456
 457     // VGPR Allocated in 4-GPR granularity
 458
 459 if SAVE_AFTER_XNACK_ERROR
 460         check_if_tcp_store_ok()
 461         s_cbranch_scc1 L_SAVE_FIRST_VGPRS_WITH_TCP
 462
 463         write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 464         s_branch L_SAVE_LDS
 465
 466 L_SAVE_FIRST_VGPRS_WITH_TCP:
 467 end
 468
 469         buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 470         buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
 471         buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
 472         buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
 473
 474
 475
 476     /*          save LDS        */
 477     //////////////////////////////
 478
 479   L_SAVE_LDS:
 480
 481         // Change EXEC to all threads...
 482     s_mov_b32       exec_lo, 0xFFFFFFFF   //need every thread from now on
 483     s_mov_b32       exec_hi, 0xFFFFFFFF
 484
 485     s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)             //lds_size
 486     s_and_b32       s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF                //lds_size is zero?
 487     s_cbranch_scc0  L_SAVE_LDS_DONE                                                                            //no lds used? jump to L_SAVE_DONE
 488
 489     s_barrier               //LDS is used? wait for other waves in the same TG
 490     s_and_b32       s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK                //exec is still used here
 491     s_cbranch_scc0  L_SAVE_LDS_DONE
 492
 493         // first wave do LDS save;
 494
 495     s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 6                         //LDS size in dwords = lds_size * 64dw
 496     s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 2                         //LDS size in bytes
 497     s_mov_b32       s_save_buf_rsrc2,  s_save_alloc_size                            //NUM_RECORDS in bytes
 498
 499     // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
 500     //
 501     get_vgpr_size_bytes(s_save_mem_offset)
 502     get_sgpr_size_bytes(s_save_tmp)
 503     s_add_u32  s_save_mem_offset, s_save_mem_offset, s_save_tmp
 504     s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
 505
 506
 507         s_mov_b32       s_save_buf_rsrc2,  0x1000000                  //NUM_RECORDS in bytes
 508
 509     s_mov_b32       m0, 0x0                                               //lds_offset initial value = 0
 510
 511
 512       v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
 513       v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2     // tid
 514
 515 if SAVE_AFTER_XNACK_ERROR
 516         check_if_tcp_store_ok()
 517         s_cbranch_scc1 L_SAVE_LDS_WITH_TCP
 518
 519         v_lshlrev_b32 v2, 2, v3
 520 L_SAVE_LDS_LOOP_SQC:
 521         ds_read2_b32 v[0:1], v2 offset0:0 offset1:0x40
 522         s_waitcnt lgkmcnt(0)
 523
 524         write_vgprs_to_mem_with_sqc(v0, 2, s_save_buf_rsrc0, s_save_mem_offset)
 525
 526         v_add_u32 v2, 0x200, v2
 527         v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
 528         s_cbranch_vccnz L_SAVE_LDS_LOOP_SQC
 529
 530         s_branch L_SAVE_LDS_DONE
 531
 532 L_SAVE_LDS_WITH_TCP:
 533 end
 534
 535       v_mul_i32_i24 v2, v3, 8   // tid*8
 536       v_mov_b32 v3, 256*2
 537       s_mov_b32 m0, 0x10000
 538       s_mov_b32 s0, s_save_buf_rsrc3
 539       s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF    // disable add_tid
 540       s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000   //DFMT
 541
 542 L_SAVE_LDS_LOOP_VECTOR:
 543       ds_read_b64 v[0:1], v2    //x =LDS[a], byte address
 544       s_waitcnt lgkmcnt(0)
 545       buffer_store_dwordx2  v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1  glc:1  slc:1
 546 //      s_waitcnt vmcnt(0)
 547 //      v_add_u32 v2, vcc[0:1], v2, v3
 548       v_add_u32 v2, v2, v3
 549       v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
 550       s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
 551
 552       // restore rsrc3
 553       s_mov_b32 s_save_buf_rsrc3, s0
 554
 555 L_SAVE_LDS_DONE:
 556
 557
 558     /*          save VGPRs  - set the Rest VGPRs        */
 559     //////////////////////////////////////////////////////////////////////////////////////
 560   L_SAVE_VGPR:
 561     // VGPR SR memory offset: 0
 562     // TODO rearrange the RSRC words to use swizzle for VGPR save...
 563
 564     s_mov_b32       s_save_mem_offset, (0+256*4)                                    // for the rest VGPRs
 565     s_mov_b32       exec_lo, 0xFFFFFFFF                                             //need every thread from now on
 566     s_mov_b32       exec_hi, 0xFFFFFFFF
 567
 568     s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)                   //vpgr_size
 569     s_add_u32       s_save_alloc_size, s_save_alloc_size, 1
 570     s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 2                         //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)   //FIXME for GFX, zero is possible
 571     s_lshl_b32      s_save_buf_rsrc2,  s_save_alloc_size, 8                         //NUM_RECORDS in bytes (64 threads*4)
 572         s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
 573
 574
 575     // VGPR store using dw burst
 576     s_mov_b32         m0, 0x4   //VGPR initial index value =0
 577     s_cmp_lt_u32      m0, s_save_alloc_size
 578     s_cbranch_scc0    L_SAVE_VGPR_END
 579
 580
 581     s_set_gpr_idx_on    m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
 582     s_add_u32       s_save_alloc_size, s_save_alloc_size, 0x1000                    //add 0x1000 since we compare m0 against it later
 583
 584 if SAVE_AFTER_XNACK_ERROR
 585         check_if_tcp_store_ok()
 586         s_cbranch_scc1 L_SAVE_VGPR_LOOP
 587
 588 L_SAVE_VGPR_LOOP_SQC:
 589         write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 590
 591         s_add_u32 m0, m0, 4
 592         s_cmp_lt_u32 m0, s_save_alloc_size
 593         s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC
 594
 595         s_set_gpr_idx_off
 596         s_branch L_SAVE_VGPR_END
 597 end
 598
 599   L_SAVE_VGPR_LOOP:
 600     v_mov_b32       v0, v0              //v0 = v[0+m0]
 601     v_mov_b32       v1, v1              //v0 = v[0+m0]
 602     v_mov_b32       v2, v2              //v0 = v[0+m0]
 603     v_mov_b32       v3, v3              //v0 = v[0+m0]
 604
 605         buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 606         buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
 607         buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
 608         buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
 609
 610     s_add_u32       m0, m0, 4                                                       //next vgpr index
 611     s_add_u32       s_save_mem_offset, s_save_mem_offset, 256*4                     //every buffer_store_dword does 256 bytes
 612     s_cmp_lt_u32    m0, s_save_alloc_size                                           //scc = (m0 < s_save_alloc_size) ? 1 : 0
 613     s_cbranch_scc1  L_SAVE_VGPR_LOOP                                                //VGPR save is complete?
 614     s_set_gpr_idx_off
 615
 616 L_SAVE_VGPR_END:
 617
 618 if ASIC_TARGET_ARCTURUS
 619     // Save ACC VGPRs
 620     s_mov_b32 m0, 0x0 //VGPR initial index value =0
 621     s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
 622
 623 if SAVE_AFTER_XNACK_ERROR
 624     check_if_tcp_store_ok()
 625     s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP
 626
 627 L_SAVE_ACCVGPR_LOOP_SQC:
 628     for var vgpr = 0; vgpr < 4; ++ vgpr
 629         v_accvgpr_read v[vgpr], acc[vgpr]  // v[N] = acc[N+m0]
 630     end
 631
 632     write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)
 633
 634     s_add_u32 m0, m0, 4
 635     s_cmp_lt_u32 m0, s_save_alloc_size
 636     s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP_SQC
 637
 638     s_set_gpr_idx_off
 639     s_branch L_SAVE_ACCVGPR_END
 640 end
 641
 642 L_SAVE_ACCVGPR_LOOP:
 643     for var vgpr = 0; vgpr < 4; ++ vgpr
 644         v_accvgpr_read v[vgpr], acc[vgpr]  // v[N] = acc[N+m0]
 645     end
 646
 647     buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
 648     buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256
 649     buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*2
 650     buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1 offset:256*3
 651
 652     s_add_u32 m0, m0, 4
 653     s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4
 654     s_cmp_lt_u32 m0, s_save_alloc_size
 655     s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP
 656     s_set_gpr_idx_off
 657
 658 L_SAVE_ACCVGPR_END:
 659 end
 660
 661     s_branch    L_END_PGM
 662
 663
 664
 665 /**************************************************************************/
 666 /*                      restore routine                                   */
 667 /**************************************************************************/
 668
 669 L_RESTORE:
 670     /*      Setup Resource Contants    */
 671     s_mov_b32       s_restore_buf_rsrc0,    s_restore_spi_init_lo                                                           //base_addr_lo
 672     s_and_b32       s_restore_buf_rsrc1,    s_restore_spi_init_hi, 0x0000FFFF                                               //base_addr_hi
 673     s_or_b32        s_restore_buf_rsrc1,    s_restore_buf_rsrc1,  S_RESTORE_BUF_RSRC_WORD1_STRIDE
 674     s_mov_b32       s_restore_buf_rsrc2,    0                                                                               //NUM_RECORDS initial value = 0 (in bytes)
 675     s_mov_b32       s_restore_buf_rsrc3,    S_RESTORE_BUF_RSRC_WORD3_MISC
 676     s_and_b32       s_restore_tmp,          s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
 677     s_lshr_b32      s_restore_tmp,          s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)       //get ATC bit into position
 678     s_or_b32        s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp                                             //or ATC
 679     s_and_b32       s_restore_tmp,          s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
 680     s_lshr_b32      s_restore_tmp,          s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)   //get MTYPE bits into position
 681     s_or_b32        s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp                                             //or MTYPE
 682
 683     /*      global mem offset           */
 684 //  s_mov_b32       s_restore_mem_offset, 0x0                               //mem offset initial value = 0
 685
 686     /*      the first wave in the threadgroup    */
 687     s_and_b32       s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
 688     s_cbranch_scc0  L_RESTORE_VGPR
 689
 690     /*          restore LDS     */
 691     //////////////////////////////
 692   L_RESTORE_LDS:
 693
 694     s_mov_b32       exec_lo, 0xFFFFFFFF                                                     //need every thread from now on   //be consistent with SAVE although can be moved ahead
 695     s_mov_b32       exec_hi, 0xFFFFFFFF
 696
 697     s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)              //lds_size
 698     s_and_b32       s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF                  //lds_size is zero?
 699     s_cbranch_scc0  L_RESTORE_VGPR                                                          //no lds used? jump to L_RESTORE_VGPR
 700     s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 6                           //LDS size in dwords = lds_size * 64dw
 701     s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 2                           //LDS size in bytes
 702     s_mov_b32       s_restore_buf_rsrc2,    s_restore_alloc_size                            //NUM_RECORDS in bytes
 703
 704     // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
 705     //
 706     get_vgpr_size_bytes(s_restore_mem_offset)
 707     get_sgpr_size_bytes(s_restore_tmp)
 708     s_add_u32  s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
 709     s_add_u32  s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()            //FIXME, Check if offset overflow???
 710
 711
 712         s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
 713     s_mov_b32       m0, 0x0                                                                 //lds_offset initial value = 0
 714
 715   L_RESTORE_LDS_LOOP:
 716         buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1                    // first 64DW
 717         buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256         // second 64DW
 718     s_add_u32       m0, m0, 256*2                                               // 128 DW
 719     s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*2           //mem offset increased by 128DW
 720     s_cmp_lt_u32    m0, s_restore_alloc_size                                    //scc=(m0 < s_restore_alloc_size) ? 1 : 0
 721     s_cbranch_scc1  L_RESTORE_LDS_LOOP                                                      //LDS restore is complete?
 722
 723
 724     /*          restore VGPRs       */
 725     //////////////////////////////
 726   L_RESTORE_VGPR:
 727         // VGPR SR memory offset : 0
 728     s_mov_b32       s_restore_mem_offset, 0x0
 729     s_mov_b32       exec_lo, 0xFFFFFFFF                                                     //need every thread from now on   //be consistent with SAVE although can be moved ahead
 730     s_mov_b32       exec_hi, 0xFFFFFFFF
 731
 732     s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)    //vpgr_size
 733     s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 1
 734     s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 2                           //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
 735     s_lshl_b32      s_restore_buf_rsrc2,  s_restore_alloc_size, 8                           //NUM_RECORDS in bytes (64 threads*4)
 736
 737 if ASIC_TARGET_ARCTURUS
 738     s_mov_b32       s_restore_accvgpr_offset, s_restore_buf_rsrc2                           //ACC VGPRs at end of VGPRs
 739 end
 740
 741         s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
 742
 743     // VGPR load using dw burst
 744     s_mov_b32       s_restore_mem_offset_save, s_restore_mem_offset     // restore start with v1, v0 will be the last
 745     s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*4
 746 if ASIC_TARGET_ARCTURUS
 747     s_mov_b32       s_restore_accvgpr_offset_save, s_restore_accvgpr_offset
 748     s_add_u32       s_restore_accvgpr_offset, s_restore_accvgpr_offset, 256*4
 749 end
 750     s_mov_b32       m0, 4                               //VGPR initial index value = 1
 751     s_set_gpr_idx_on  m0, 0x8                       //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
 752     s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 0x8000                      //add 0x8000 since we compare m0 against it later
 753
 754   L_RESTORE_VGPR_LOOP:
 755
 756 if ASIC_TARGET_ARCTURUS
 757         buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset slc:1 glc:1
 758         buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset slc:1 glc:1 offset:256
 759         buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset slc:1 glc:1 offset:256*2
 760         buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset slc:1 glc:1 offset:256*3
 761         s_add_u32 s_restore_accvgpr_offset, s_restore_accvgpr_offset, 256*4
 762         s_waitcnt vmcnt(0)
 763
 764         for var vgpr = 0; vgpr < 4; ++ vgpr
 765                 v_accvgpr_write acc[vgpr], v[vgpr]
 766         end
 767 end
 768
 769         buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
 770         buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
 771         buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
 772         buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
 773     s_waitcnt       vmcnt(0)                                                                //ensure data ready
 774     v_mov_b32       v0, v0                                                                  //v[0+m0] = v0
 775     v_mov_b32       v1, v1
 776     v_mov_b32       v2, v2
 777     v_mov_b32       v3, v3
 778     s_add_u32       m0, m0, 4                                                               //next vgpr index
 779     s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*4                           //every buffer_load_dword does 256 bytes
 780     s_cmp_lt_u32    m0, s_restore_alloc_size                                                //scc = (m0 < s_restore_alloc_size) ? 1 : 0
 781     s_cbranch_scc1  L_RESTORE_VGPR_LOOP                                                     //VGPR restore (except v0) is complete?
 782     s_set_gpr_idx_off
 783                                                                                             /* VGPR restore on v0 */
 784 if ASIC_TARGET_ARCTURUS
 785         buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset_save slc:1 glc:1
 786         buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset_save slc:1 glc:1 offset:256
 787         buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset_save slc:1 glc:1 offset:256*2
 788         buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_accvgpr_offset_save slc:1 glc:1 offset:256*3
 789         s_waitcnt vmcnt(0)
 790
 791         for var vgpr = 0; vgpr < 4; ++ vgpr
 792                 v_accvgpr_write acc[vgpr], v[vgpr]
 793         end
 794 end
 795
 796         buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1
 797         buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256
 798         buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*2
 799         buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*3
 800
 801     /*          restore SGPRs       */
 802     //////////////////////////////
 803
 804     // SGPR SR memory offset : size(VGPR)
 805     get_vgpr_size_bytes(s_restore_mem_offset)
 806     get_sgpr_size_bytes(s_restore_tmp)
 807     s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
 808     s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4     // restore SGPR from S[n] to S[0], by 16 sgprs group
 809     // TODO, change RSRC word to rearrange memory layout for SGPRS
 810
 811     s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)                //spgr_size
 812     s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 1
 813     s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 4                           //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
 814
 815         s_lshl_b32      s_restore_buf_rsrc2,    s_restore_alloc_size, 2                     //NUM_RECORDS in bytes
 816         s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
 817
 818     s_mov_b32 m0, s_restore_alloc_size
 819
 820  L_RESTORE_SGPR_LOOP:
 821     read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)  //PV: further performance improvement can be made
 822     s_waitcnt       lgkmcnt(0)                                                              //ensure data ready
 823
 824     s_sub_u32 m0, m0, 16    // Restore from S[n] to S[0]
 825     s_nop 0 // hazard SALU M0=> S_MOVREL
 826
 827     s_movreld_b64   s0, s0      //s[0+m0] = s0
 828     s_movreld_b64   s2, s2
 829     s_movreld_b64   s4, s4
 830     s_movreld_b64   s6, s6
 831     s_movreld_b64   s8, s8
 832     s_movreld_b64   s10, s10
 833     s_movreld_b64   s12, s12
 834     s_movreld_b64   s14, s14
 835
 836     s_cmp_eq_u32    m0, 0               //scc = (m0 < s_restore_alloc_size) ? 1 : 0
 837     s_cbranch_scc0  L_RESTORE_SGPR_LOOP             //SGPR restore (except s0) is complete?
 838
 839     /*      restore HW registers    */
 840     //////////////////////////////
 841   L_RESTORE_HWREG:
 842
 843
 844     // HWREG SR memory offset : size(VGPR)+size(SGPR)
 845     get_vgpr_size_bytes(s_restore_mem_offset)
 846     get_sgpr_size_bytes(s_restore_tmp)
 847     s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
 848
 849
 850     s_mov_b32       s_restore_buf_rsrc2, 0x4                                                //NUM_RECORDS   in bytes
 851         s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
 852
 853     read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)                    //M0
 854     read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)             //PC
 855     read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
 856     read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)               //EXEC
 857     read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
 858     read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)                //STATUS
 859     read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)               //TRAPSTS
 860     read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset)                   //XNACK_MASK_LO
 861     read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset)                   //XNACK_MASK_HI
 862     read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)              //MODE
 863
 864     s_waitcnt       lgkmcnt(0)                                                                                      //from now on, it is safe to restore STATUS and IB_STS
 865
 866     s_mov_b32       m0,         s_restore_m0
 867     s_mov_b32       exec_lo,    s_restore_exec_lo
 868     s_mov_b32       exec_hi,    s_restore_exec_hi
 869
 870     s_and_b32       s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
 871     s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
 872     s_and_b32       s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
 873     s_lshr_b32      s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
 874     s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
 875     //s_setreg_b32  hwreg(HW_REG_TRAPSTS),  s_restore_trapsts      //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
 876     s_setreg_b32    hwreg(HW_REG_MODE),     s_restore_mode
 877
 878     // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
 879     // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40
 880     get_vgpr_size_bytes(s_restore_ttmps_lo)
 881     get_sgpr_size_bytes(s_restore_ttmps_hi)
 882     s_add_u32       s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi
 883     s_add_u32       s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0
 884     s_addc_u32      s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0
 885     s_and_b32       s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF
 886     s_load_dwordx4  [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1
 887     s_load_dwordx4  [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1
 888     s_load_dword    ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1
 889     s_waitcnt       lgkmcnt(0)
 890
 891     //reuse s_restore_m0 as a temp register
 892     s_and_b32       s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
 893     s_lshr_b32      s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
 894     s_lshl_b32      s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
 895     s_mov_b32       s_restore_tmp, 0x0                                                                              //IB_STS is zero
 896     s_or_b32        s_restore_tmp, s_restore_tmp, s_restore_m0
 897     s_and_b32       s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
 898     s_lshr_b32      s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
 899     s_lshl_b32      s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
 900     s_or_b32        s_restore_tmp, s_restore_tmp, s_restore_m0
 901     s_and_b32       s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
 902     s_lshr_b32      s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
 903     s_setreg_b32    hwreg(HW_REG_IB_STS),   s_restore_tmp
 904
 905     s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
 906     s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
 907     s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
 908     set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
 909
 910     s_barrier                                                   //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
 911
 912 //  s_rfe_b64 s_restore_pc_lo                                   //Return to the main shader program and resume execution
 913     s_rfe_restore_b64  s_restore_pc_lo, s_restore_m0            // s_restore_m0[0] is used to set STATUS.inst_atc
 914
 915
 916 /**************************************************************************/
 917 /*                      the END                                           */
 918 /**************************************************************************/
 919 L_END_PGM:
 920     s_endpgm
 921
 922 end
 923
 924
 925 /**************************************************************************/
 926 /*                      the helper functions                              */
 927 /**************************************************************************/
 928
 929 //Only for save hwreg to mem
 930 function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
 931         s_mov_b32 exec_lo, m0                   //assuming exec_lo is not needed anymore from this point on
 932         s_mov_b32 m0, s_mem_offset
 933         s_buffer_store_dword s, s_rsrc, m0      glc:1
 934         ack_sqc_store_workaround()
 935         s_add_u32       s_mem_offset, s_mem_offset, 4
 936         s_mov_b32   m0, exec_lo
 937 end
 938
 939
 940 // HWREG are saved before SGPRs, so all HWREG could be use.
 941 function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
 942
 943         s_buffer_store_dwordx4 s[0], s_rsrc, 0  glc:1
 944         ack_sqc_store_workaround()
 945         s_buffer_store_dwordx4 s[4], s_rsrc, 16  glc:1
 946         ack_sqc_store_workaround()
 947         s_buffer_store_dwordx4 s[8], s_rsrc, 32  glc:1
 948         ack_sqc_store_workaround()
 949         s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
 950         ack_sqc_store_workaround()
 951         s_add_u32       s_rsrc[0], s_rsrc[0], 4*16
 952         s_addc_u32      s_rsrc[1], s_rsrc[1], 0x0             // +scc
 953 end
 954
 955
 956 function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
 957     s_buffer_load_dword s, s_rsrc, s_mem_offset     glc:1
 958     s_add_u32       s_mem_offset, s_mem_offset, 4
 959 end
 960
 961 function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
 962     s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset      glc:1
 963     s_sub_u32       s_mem_offset, s_mem_offset, 4*16
 964 end
 965
 966 function check_if_tcp_store_ok
 967         // If STATUS.ALLOW_REPLAY=0 and TRAPSTS.XNACK_ERROR=1 then TCP stores will fail.
 968         s_and_b32 s_save_tmp, s_save_status, SQ_WAVE_STATUS_ALLOW_REPLAY_MASK
 969         s_cbranch_scc1 L_TCP_STORE_CHECK_DONE
 970
 971         s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)
 972         s_andn2_b32 s_save_tmp, SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK, s_save_tmp
 973
 974 L_TCP_STORE_CHECK_DONE:
 975 end
 976
 977 function write_vgpr_to_mem_with_sqc(v, s_rsrc, s_mem_offset)
 978         s_mov_b32 s4, 0
 979
 980 L_WRITE_VGPR_LANE_LOOP:
 981         for var lane = 0; lane < 4; ++ lane
 982                 v_readlane_b32 s[lane], v, s4
 983                 s_add_u32 s4, s4, 1
 984         end
 985
 986         s_buffer_store_dwordx4 s[0:3], s_rsrc, s_mem_offset glc:1
 987         ack_sqc_store_workaround()
 988
 989         s_add_u32 s_mem_offset, s_mem_offset, 0x10
 990         s_cmp_eq_u32 s4, 0x40
 991         s_cbranch_scc0 L_WRITE_VGPR_LANE_LOOP
 992 end
 993
 994 function write_vgprs_to_mem_with_sqc(v, n_vgprs, s_rsrc, s_mem_offset)
 995         for var vgpr = 0; vgpr < n_vgprs; ++ vgpr
 996                 write_vgpr_to_mem_with_sqc(v[vgpr], s_rsrc, s_mem_offset)
 997         end
 998 end
 999
1000 function get_lds_size_bytes(s_lds_size_byte)
1001     // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW
1002     s_getreg_b32   s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)          // lds_size
1003     s_lshl_b32     s_lds_size_byte, s_lds_size_byte, 8                      //LDS size in dwords = lds_size * 64 *4Bytes    // granularity 64DW
1004 end
1005
1006 function get_vgpr_size_bytes(s_vgpr_size_byte)
1007     s_getreg_b32   s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)  //vpgr_size
1008     s_add_u32      s_vgpr_size_byte, s_vgpr_size_byte, 1
1009     s_lshl_b32     s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4   (non-zero value)   //FIXME for GFX, zero is possible
1010
1011 if ASIC_TARGET_ARCTURUS
1012     s_lshl_b32     s_vgpr_size_byte, s_vgpr_size_byte, 1  // Double size for ACC VGPRs
1013 end
1014 end
1015
1016 function get_sgpr_size_bytes(s_sgpr_size_byte)
1017     s_getreg_b32   s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)  //spgr_size
1018     s_add_u32      s_sgpr_size_byte, s_sgpr_size_byte, 1
1019     s_lshl_b32     s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4   (non-zero value)
1020 end
1021
1022 function get_hwreg_size_bytes
1023     return 128 //HWREG size 128 bytes
1024 end
1025
1026 function ack_sqc_store_workaround
1027     if ACK_SQC_STORE
1028         s_waitcnt lgkmcnt(0)
1029     end
1030 end
1031
1032 function set_status_without_spi_prio(status, tmp)
1033     // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
1034     s_lshr_b32      tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
1035     s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
1036     s_nop           0x2 // avoid S_SETREG => S_SETREG hazard
1037     s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
1038 end