[LoongArch64] Modify several atomic instructions in nativeaot. (dotne…

…t#108377)

src/coreclr/nativeaot/Runtime/loongarch64/AllocFast.S

@@ -45,7 +45,7 @@
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
         ld.d   $t4, $a1, OFFSETOF__Thread__m_alloc_context__alloc_limit
-        bltu  $t4, $a2, RhpNewFast_RarePath
+        bltu  $t4, $a2, LOCAL_LABEL(RhpNewFast_RarePath)
 
         // Update the alloc pointer to account for the allocation.
         st.d  $a2, $a1, OFFSETOF__Thread__m_alloc_context__alloc_ptr
@@ -56,7 +56,7 @@
         ori  $a0, $t3, 0
         jirl  $r0, $ra, 0
 
-RhpNewFast_RarePath:
+LOCAL_LABEL(RhpNewFast_RarePath):
         ori  $a1, $zero, 0
         b  RhpNewObject
     LEAF_END RhpNewFast, _TEXT
@@ -87,14 +87,14 @@ RhpNewFast_RarePath:
         bl  C_FUNC(RhpGcAlloc)
 
         // Set the new objects MethodTable pointer on success.
-        beq  $a0, $zero, NewOutOfMemory
+        beq  $a0, $zero, LOCAL_LABEL(NewOutOfMemory)
 
         .cfi_remember_state
         POP_COOP_PINVOKE_FRAME
         EPILOG_RETURN
 
         .cfi_restore_state
-NewOutOfMemory:
+LOCAL_LABEL(NewOutOfMemory):
         // This is the OOM failure path. We are going to tail-call to a managed helper that will throw
         // an out of memory exception that the caller of this allocator understands.
 
@@ -113,7 +113,7 @@ NewOutOfMemory:
         // Make sure computing the overall allocation size wont overflow
         lu12i.w  $a2, ((MAX_STRING_LENGTH >> 12) & 0xFFFFF)
         ori  $a2, $a2, (MAX_STRING_LENGTH & 0xFFF)
-        bltu  $a2, $a1, StringSizeOverflow
+        bltu  $a2, $a1, LOCAL_LABEL(StringSizeOverflow)
 
         // Compute overall allocation size (align(base size + (element size * elements), 8)).
         ori  $a2, $zero, STRING_COMPONENT_SIZE
@@ -138,7 +138,7 @@ NewOutOfMemory:
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
         ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_limit
-        bltu  $t3, $a2, RhNewString_Rare
+        bltu  $t3, $a2, LOCAL_LABEL(RhNewString_Rare)
 
         // Reload new object address into r12.
         ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_ptr
@@ -155,7 +155,7 @@ NewOutOfMemory:
 
         jirl  $r0, $ra, 0
 
-StringSizeOverflow:
+LOCAL_LABEL(StringSizeOverflow):
         // We get here if the length of the final string object can not be represented as an unsigned
         // 32-bit value. We are going to tail-call to a managed helper that will throw
         // an OOM exception that the caller of this allocator understands.
@@ -164,7 +164,7 @@ StringSizeOverflow:
         ori  $a1, $zero, 1                  // Indicate that we should throw OverflowException
         b  C_FUNC(RhExceptionHandling_FailedAllocation)
 
-RhNewString_Rare:
+LOCAL_LABEL(RhNewString_Rare):
         b  C_FUNC(RhpNewArrayRare)
     LEAF_END    RhNewString, _Text
 
@@ -179,7 +179,7 @@ RhNewString_Rare:
         // case (32 dimensional MdArray) is less than 0xffff, and thus the product fits in 64 bits.
         lu12i.w  $a2, 0x7ffff
         ori  $a2, $a2, 0xfff
-        bltu  $a2, $a1, ArraySizeOverflow
+        bltu  $a2, $a1, LOCAL_LABEL(ArraySizeOverflow)
 
         ld.h  $a2, $a0, OFFSETOF__MethodTable__m_usComponentSize
         mulw.d.w  $a2, $a1, $a2
@@ -200,7 +200,7 @@ RhNewString_Rare:
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
         ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_limit
-        bltu  $t3, $a2, RhpNewArray_Rare
+        bltu  $t3, $a2, LOCAL_LABEL(RhpNewArray_Rare)
 
         // Reload new object address into t3.
         ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_ptr
@@ -217,7 +217,7 @@ RhNewString_Rare:
 
         jirl  $r0, $ra, 0
 
-ArraySizeOverflow:
+LOCAL_LABEL(ArraySizeOverflow):
         // We get here if the size of the final array object can not be represented as an unsigned
         // 32-bit value. We are going to tail-call to a managed helper that will throw
         // an overflow exception that the caller of this allocator understands.
@@ -226,7 +226,7 @@ ArraySizeOverflow:
         ori  $a1, $zero, 1 // Indicate that we should throw OverflowException
         b  C_FUNC(RhExceptionHandling_FailedAllocation)
 
-RhpNewArray_Rare:
+LOCAL_LABEL(RhpNewArray_Rare):
         b  C_FUNC(RhpNewArrayRare)
     LEAF_END    RhpNewArray, _TEXT
 
@@ -253,14 +253,14 @@ RhpNewArray_Rare:
         bl  C_FUNC(RhpGcAlloc)
 
         // Set the new objects MethodTable pointer and length on success.
-        beq  $a0, $zero, ArrayOutOfMemory
+        beq  $a0, $zero, LOCAL_LABEL(ArrayOutOfMemory)
 
         .cfi_remember_state
         POP_COOP_PINVOKE_FRAME
         EPILOG_RETURN
 
         .cfi_restore_state
-ArrayOutOfMemory:
+LOCAL_LABEL(ArrayOutOfMemory):
         // This is the OOM failure path. We are going to tail-call to a managed helper that will throw
         // an out of memory exception that the caller of this allocator understands.
 

src/coreclr/nativeaot/Runtime/loongarch64/ExceptionHandling.S

@@ -288,7 +288,7 @@
         // where the tail-calling thread had saved RA, which may not match where we have saved RA.
 
         ld.d  $a1, $a2, OFFSETOF__Thread__m_pvHijackedReturnAddress
-        beq  $a1, $zero, NotHijacked
+        beq  $a1, $zero, LOCAL_LABEL(NotHijacked)
 
         ld.d  $a3, $a2, OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation
 
@@ -298,13 +298,13 @@
         // a3: hijacked return address location
 
         addi.d  $t3, $sp, (STACKSIZEOF_ExInfo + SIZEOF__PAL_LIMITED_CONTEXT) // re-compute SP at callsite
-        bltu  $a3, $t3, TailCallWasHijacked // if (m_ppvHijackedReturnAddressLocation < SP at callsite)
+        bltu  $a3, $t3, LOCAL_LABEL(TailCallWasHijacked) // if (m_ppvHijackedReturnAddressLocation < SP at callsite)
 
         // normal case where a valid return address location is hijacked
         st.d  $a1, $a3, 0
-        b  ClearThreadState
+        b  LOCAL_LABEL(ClearThreadState)
 
-TailCallWasHijacked:
+LOCAL_LABEL(TailCallWasHijacked):
 
         // Abnormal case where the return address location is now invalid because we ended up here via a tail
         // call.  In this case, our hijacked return address should be the correct caller of this method.
@@ -314,13 +314,13 @@ TailCallWasHijacked:
         st.d  $ra, $sp, (rsp_offsetof_Context + OFFSETOF__PAL_LIMITED_CONTEXT__RA)
         st.d  $ra, $sp, (rsp_offsetof_Context + OFFSETOF__PAL_LIMITED_CONTEXT__IP)
 
-ClearThreadState:
+LOCAL_LABEL(ClearThreadState):
 
         // clear the Thread's hijack state
         st.d  $zero, $a2, OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation
         st.d  $zero, $a2, OFFSETOF__Thread__m_pvHijackedReturnAddress
 
-NotHijacked:
+LOCAL_LABEL(NotHijacked):
 
         addi.d  $a1, $sp, rsp_offsetof_ExInfo            // a1 <- ExInfo*
         st.d  $zero, $a1, OFFSETOF__ExInfo__m_exception  // pExInfo->m_exception = null
@@ -502,21 +502,21 @@ NotHijacked:
         ld.d  $a3, $sp, rsp_offset_a3                           // a3 <- current ExInfo*
         ld.d  $a2, $a2, OFFSETOF__REGDISPLAY__SP                // a2 <- resume SP value
 
-PopExInfoLoop:
+LOCAL_LABEL(PopExInfoLoop):
         ld.d  $a3, $a3, OFFSETOF__ExInfo__m_pPrevExInfo         // a3 <- next ExInfo
-        beq  $a3, $zero, DonePopping                            // if (pExInfo == null) { we're done }
-        blt  $a3, $a2, PopExInfoLoop                            // if (pExInfo < resume SP} { keep going }
+        beq  $a3, $zero, LOCAL_LABEL(DonePopping)               // if (pExInfo == null) { we're done }
+        blt  $a3, $a2, LOCAL_LABEL(PopExInfoLoop)               // if (pExInfo < resume SP} { keep going }
 
-DonePopping:
+LOCAL_LABEL(DonePopping):
         st.d  $a3, $a1, OFFSETOF__Thread__m_pExInfoStackHead    // store the new head on the Thread
 
         PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, $a3
 
         andi  $t7, $a3, TrapThreadsFlags_AbortInProgress_Bit
-        beq  $t7, $zero, NoAbort
+        beq  $t7, $zero, LOCAL_LABEL(NoAbort)
 
         ld.d  $a3, $sp, rsp_offset_is_not_handling_thread_abort
-        bne  $a3, $zero, NoAbort
+        bne  $a3, $zero, LOCAL_LABEL(NoAbort)
 
         // It was the ThreadAbortException, so rethrow it
         // reset SP
@@ -525,7 +525,7 @@ DonePopping:
         ori  $sp, $a2, 0
         b  C_FUNC(RhpThrowHwEx)
 
-NoAbort:
+LOCAL_LABEL(NoAbort):
         // reset SP and jump to continuation address
         ori  $sp, $a2, 0
         jirl  $r0, $a0, 0
@@ -618,8 +618,8 @@ NoAbort:
         ld.d  $a2, $sp, rsp_FinallyFunclet_offset_thread
 
         addi.d  $t3, $a2, OFFSETOF__Thread__m_ThreadStateFlags
-        addi.w  $a3, $zero, -17     // $a3 = $a3 & ~TSF_DoNotTriggerGc, TSF_DoNotTriggerGc=0x10.
-        amand_db.w  $a1, $a3, $t3
+        addi.w  $a3, $zero, 16     // $a3 = $a3 | TSF_DoNotTriggerGc, TSF_DoNotTriggerGc=0x10.
+        amor_db.w  $a1, $a3, $t3
 
         fld.d  $f24, $sp, 0x00
         fld.d  $f25, $sp, 0x08
@@ -771,12 +771,12 @@ NoAbort:
         ld.d  $a3, $sp, rsp_offset_a3                    // a3 <- current ExInfo*
         ld.d  $a2, $a2, OFFSETOF__REGDISPLAY__SP         // a2 <- resume SP value
 
-Propagate_PopExInfoLoop:
-        ld.d  $a3, $a3, OFFSETOF__ExInfo__m_pPrevExInfo  // a3 <- next ExInfo
-        beq  $a3, $zero, Propagate_DonePopping           // if (pExInfo == null) { we're done }
-        blt  $a3, $a2, Propagate_PopExInfoLoop           // if (pExInfo < resume SP} { keep going }
+LOCAL_LABEL(Propagate_PopExInfoLoop):
+        ld.d  $a3, $a3, OFFSETOF__ExInfo__m_pPrevExInfo     // a3 <- next ExInfo
+        beq  $a3, $zero, LOCAL_LABEL(Propagate_DonePopping) // if (pExInfo == null) { we're done }
+        blt  $a3, $a2, LOCAL_LABEL(Propagate_PopExInfoLoop) // if (pExInfo < resume SP} { keep going }
 
-Propagate_DonePopping:
+LOCAL_LABEL(Propagate_DonePopping):
         st.d  $a3, $a1, OFFSETOF__Thread__m_pExInfoStackHead // store the new head on the Thread
 
         // restore preemptive mode

src/coreclr/nativeaot/Runtime/loongarch64/GcProbe.S

@@ -135,10 +135,10 @@ NESTED_ENTRY RhpGcProbeHijack, _TEXT, NoHandler
 
     PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, $a3
     andi  $t8, $a3, TrapThreadsFlags_TrapThreads_Bit
-    bne  $t8, $zero, WaitForGC
+    bne  $t8, $zero, LOCAL_LABEL(WaitForGC)
     jirl  $r0, $ra, 0
 
-WaitForGC:
+LOCAL_LABEL(WaitForGC):
     lu12i.w  $t7, ((DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_R4 + PTFF_SAVE_R5) >> 12) & 0xfffff
     ori  $t7, $t7, (DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_R4 + PTFF_SAVE_R5) & 0xfff
     or  $t3, $t3, $t7
@@ -155,14 +155,14 @@ NESTED_ENTRY RhpWaitForGC, _TEXT, NoHandler
 
     ld.d  $a2,$sp, OFFSETOF__PInvokeTransitionFrame__m_Flags
     andi  $t8, $a2, PTFF_THREAD_ABORT_BIT
-    bne  $t8, $zero, ThrowThreadAbort
+    bne  $t8, $zero, LOCAL_LABEL(ThrowThreadAbort)
 
     .cfi_remember_state
     POP_PROBE_FRAME
     EPILOG_RETURN
 
     .cfi_restore_state
-ThrowThreadAbort:
+LOCAL_LABEL(ThrowThreadAbort):
     POP_PROBE_FRAME
     addi.w  $a0, $zero, STATUS_REDHAWK_THREAD_ABORT
     ori  $a1, $ra, 0 // return address as exception PC

src/coreclr/nativeaot/Runtime/loongarch64/WriteBarriers.S

@@ -231,7 +231,7 @@ LEAF_END RhpByRefAssignRef, _TEXT
         or  $t4, $t0, $t4
         beq  $t4, $zero, C_FUNC(RhpAssignRefLoongArch64)
 
-NotInHeap:
+LOCAL_LABEL(NotInHeap):
     ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
         st.d  $t7, $t6, 0
         addi.d $t6, $t6, 8
@@ -293,26 +293,28 @@ LEAF_END RhpAssignRef, _TEXT
 //
     LEAF_ENTRY RhpCheckedLockCmpXchg
 
-        ori  $t1, $a2, 0
-        ld.d  $t0, $a0, 0
-        beq  $t0, $t1, 12
-        ori  $t1, $t0, 0
-        b  EndOfExchange
-        st.d  $a1, $a0, 0
+LOCAL_LABEL(RetryLoop):
+        ll.d  $t3, $a0, 0
+        ori  $t1, $a1, 0
+        bne  $t3, $a2, LOCAL_LABEL(EndOfExchange)
+        sc.d  $t1, $a0, 0
+        beqz  $t1, LOCAL_LABEL(RetryLoop)
+        b  LOCAL_LABEL(DoCardsCmpXchg)
 
-EndOfExchange:
-        bne  $a2, $t1, CmpXchgNoUpdate
+LOCAL_LABEL(EndOfExchange):
+        dbar 0x700
+        b  LOCAL_LABEL(CmpXchgNoUpdate)
 
-DoCardsCmpXchg:
+LOCAL_LABEL(DoCardsCmpXchg):
         // We have successfully updated the value of the objectref so now we need a GC write barrier.
         // The following barrier code takes the destination in $a0 and the value in $a1 so the arguments are
         // already correctly set up.
 
         INSERT_CHECKED_WRITE_BARRIER_CORE  $a0, $a1
 
-CmpXchgNoUpdate:
-        // t1 still contains the original value.
-        ori  $a0, $t1, 0
+LOCAL_LABEL(CmpXchgNoUpdate):
+        // a2 still contains the original value.
+        ori  $a0, $a2, 0
 
         jirl  $r0, $ra, 0
 
@@ -337,10 +339,8 @@ CmpXchgNoUpdate:
 //
     LEAF_ENTRY RhpCheckedXchg, _TEXT
 
-        ld.d  $t1, $a0, 0
-        st.d  $a1, $a0, 0
+        amswap_db.d  $t1, $a1, $a0      // exchange
 
-DoCardsXchg:
         // We have successfully updated the value of the objectref so now we need a GC write barrier.
         // The following barrier code takes the destination in $a0 and the value in $a1 so the arguments are
         // already correctly set up.